fix for retention logic
This commit is contained in:
onysd
parent
70c0ba44f0
commit
e6bfe2d444
35 changed files with 2108 additions and 132 deletions
21
.env.example
21
.env.example
|
|
@ -319,6 +319,27 @@ TELESRV_STORAGE_RETENTION_MODE=off
|
||||||
# check on the shared TELESRV_RETENTION_INTERVAL/TELESRV_RETENTION_BATCH
|
# check on the shared TELESRV_RETENTION_INTERVAL/TELESRV_RETENTION_BATCH
|
||||||
# cadence (Advanced section below).
|
# cadence (Advanced section below).
|
||||||
TELESRV_STORAGE_RETENTION_MAX_AGE=720h
|
TELESRV_STORAGE_RETENTION_MAX_AGE=720h
|
||||||
|
# Optional per-category overrides of the shared age above (Photo/Video/Round
|
||||||
|
# Video/Gif/Music/Voice/File/Avatar) -- each empty/unset value falls back to
|
||||||
|
# TELESRV_STORAGE_RETENTION_MAX_AGE. The mode switch above still applies to
|
||||||
|
# all of them; these only let one category expire sooner or later than the
|
||||||
|
# rest (e.g. purge voice notes after a week but keep files for a year).
|
||||||
|
TELESRV_STORAGE_RETENTION_MAX_AGE_PHOTO=
|
||||||
|
TELESRV_STORAGE_RETENTION_MAX_AGE_VIDEO=
|
||||||
|
TELESRV_STORAGE_RETENTION_MAX_AGE_ROUND_VIDEO=
|
||||||
|
TELESRV_STORAGE_RETENTION_MAX_AGE_GIF=
|
||||||
|
TELESRV_STORAGE_RETENTION_MAX_AGE_MUSIC=
|
||||||
|
TELESRV_STORAGE_RETENTION_MAX_AGE_VOICE=
|
||||||
|
TELESRV_STORAGE_RETENTION_MAX_AGE_FILE=
|
||||||
|
TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR=
|
||||||
|
# Once enabled, actively reclaims space once total physical storage exceeds
|
||||||
|
# TELESRV_STORAGE_MAX_TOTAL_BYTES above: the oldest files (regardless of
|
||||||
|
# category/age) are purged the same way "hard" retention mode purges blob
|
||||||
|
# bytes, until back under budget. Independent of the retention mode switch
|
||||||
|
# above -- can run even when that's "off". Default false: TELESRV_STORAGE_MAX_
|
||||||
|
# TOTAL_BYTES otherwise only ever blocks new uploads, never reclaims from
|
||||||
|
# existing ones.
|
||||||
|
TELESRV_STORAGE_EVICTION_ENABLE=false
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
2
cmd/telesrv-admin/web/dist/index.html
vendored
2
cmd/telesrv-admin/web/dist/index.html
vendored
|
|
@ -23,7 +23,7 @@
|
||||||
})();
|
})();
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<script type="module" crossorigin src="/assets/index-IZM8WmmY.js"></script>
|
<script type="module" crossorigin src="/assets/index-By5gK7SA.js"></script>
|
||||||
<link rel="stylesheet" crossorigin href="/assets/index-B7hI8ol7.css">
|
<link rel="stylesheet" crossorigin href="/assets/index-B7hI8ol7.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, Loader2, RefreshCw, Search } from "lucide-react";
|
import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, Loader2, RefreshCw, Search, Settings2, X } from "lucide-react";
|
||||||
import { useEffect, useMemo, useState } from "react";
|
import { useEffect, useMemo, useState } from "react";
|
||||||
|
import { createPortal } from "react-dom";
|
||||||
import { api, errorMessage } from "../api";
|
import { api, errorMessage } from "../api";
|
||||||
import { ActionButton } from "../components/ActionButton";
|
import { ActionButton } from "../components/ActionButton";
|
||||||
import { Alert, EmptyRow, Metric, PageFrame, QueryPanel, SectionHead } from "../components/ui";
|
import { Alert, EmptyRow, Metric, PageFrame, QueryPanel, SectionHead } from "../components/ui";
|
||||||
|
|
@ -200,13 +201,28 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
|
||||||
// GB inputs for byte budgets, a mode dropdown + day count for retention.
|
// GB inputs for byte budgets, a mode dropdown + day count for retention.
|
||||||
|
|
||||||
const STORAGE_ENV_KEYS = {
|
const STORAGE_ENV_KEYS = {
|
||||||
|
blobBackend: "TELESRV_BLOB_BACKEND",
|
||||||
maxTotal: "TELESRV_STORAGE_MAX_TOTAL_BYTES",
|
maxTotal: "TELESRV_STORAGE_MAX_TOTAL_BYTES",
|
||||||
minFree: "TELESRV_STORAGE_MIN_FREE_BYTES",
|
minFree: "TELESRV_STORAGE_MIN_FREE_BYTES",
|
||||||
maxUploadFile: "TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES",
|
maxUploadFile: "TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES",
|
||||||
retentionMode: "TELESRV_STORAGE_RETENTION_MODE",
|
retentionMode: "TELESRV_STORAGE_RETENTION_MODE",
|
||||||
retentionMaxAge: "TELESRV_STORAGE_RETENTION_MAX_AGE"
|
retentionMaxAge: "TELESRV_STORAGE_RETENTION_MAX_AGE",
|
||||||
|
evictionEnable: "TELESRV_STORAGE_EVICTION_ENABLE"
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
|
// Per-category retention age overrides -- each optional, empty/0 means
|
||||||
|
// "inherit the shared Retention age" above. Order matches the admin UI list.
|
||||||
|
const CATEGORY_AGE_FIELDS: { key: string; envKey: string; label: string }[] = [
|
||||||
|
{ key: "photo", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_PHOTO", label: "Photo" },
|
||||||
|
{ key: "video", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_VIDEO", label: "Video" },
|
||||||
|
{ key: "round_video", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_ROUND_VIDEO", label: "Round video (video message)" },
|
||||||
|
{ key: "gif", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_GIF", label: "GIF" },
|
||||||
|
{ key: "music", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_MUSIC", label: "Music" },
|
||||||
|
{ key: "voice", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_VOICE", label: "Voice message" },
|
||||||
|
{ key: "file", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_FILE", label: "File" },
|
||||||
|
{ key: "avatar", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR", label: "Avatar" }
|
||||||
|
];
|
||||||
|
|
||||||
// Mirrors internal/app/files.MaxUploadPartBytes * MaxUploadParts -- the
|
// Mirrors internal/app/files.MaxUploadPartBytes * MaxUploadParts -- the
|
||||||
// protocol's own upload-part-count ceiling that TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES
|
// protocol's own upload-part-count ceiling that TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES
|
||||||
// can never legally exceed (internal/config validates this server-side too;
|
// can never legally exceed (internal/config validates this server-side too;
|
||||||
|
|
@ -272,11 +288,23 @@ function DurationField({
|
||||||
const totalMinutes = Number(minutes || "0");
|
const totalMinutes = Number(minutes || "0");
|
||||||
const [unitLabel, setUnitLabel] = useState(() => bestDurationUnit(totalMinutes).label);
|
const [unitLabel, setUnitLabel] = useState(() => bestDurationUnit(totalMinutes).label);
|
||||||
const unit = DURATION_UNITS.find((u) => u.label === unitLabel) ?? DURATION_UNITS[2];
|
const unit = DURATION_UNITS.find((u) => u.label === unitLabel) ?? DURATION_UNITS[2];
|
||||||
const amount = totalMinutes > 0 ? totalMinutes / unit.minutes : NaN;
|
// 0 is a real, distinct value here (e.g. the shared Retention age
|
||||||
|
// deliberately set to 0 to disable the default sweep) -- unlike
|
||||||
|
// ByteSizeField's "0/empty = Unlimited" convention, this must not collapse
|
||||||
|
// to a blank input, or there'd be no way to tell "0" from "not typed yet".
|
||||||
|
const amount = Number.isFinite(totalMinutes) ? totalMinutes / unit.minutes : NaN;
|
||||||
|
|
||||||
function handleAmountChange(raw: string) {
|
function handleAmountChange(raw: string) {
|
||||||
|
if (!raw.trim()) {
|
||||||
|
onChange("0");
|
||||||
|
return;
|
||||||
|
}
|
||||||
const parsed = Number(raw);
|
const parsed = Number(raw);
|
||||||
if (!raw.trim() || Number.isNaN(parsed) || parsed <= 0) {
|
if (Number.isNaN(parsed) || parsed < 0) {
|
||||||
|
onChange("0");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (parsed === 0) {
|
||||||
onChange("0");
|
onChange("0");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -292,6 +320,7 @@ function DurationField({
|
||||||
min="0"
|
min="0"
|
||||||
step="any"
|
step="any"
|
||||||
value={Number.isNaN(amount) ? "" : amount}
|
value={Number.isNaN(amount) ? "" : amount}
|
||||||
|
placeholder={"0"}
|
||||||
disabled={disabled}
|
disabled={disabled}
|
||||||
onChange={(event) => handleAmountChange(event.target.value)}
|
onChange={(event) => handleAmountChange(event.target.value)}
|
||||||
/>
|
/>
|
||||||
|
|
@ -357,15 +386,71 @@ function ByteSizeField({
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CategoryRetentionModal is a focused editor for the per-category age
|
||||||
|
// overrides -- pulled out of the main Limits & Retention flow (which was
|
||||||
|
// getting crowded with 8 extra duration fields) into its own dialog, following
|
||||||
|
// the same modal-backdrop/command-modal pattern as MintCollectibleUsernameModal.
|
||||||
|
// It edits the same categoryAgeMinutes state the parent already owns; there's
|
||||||
|
// no separate save here, just Close -- the one shared "Save limits &
|
||||||
|
// retention settings" button below still covers these values too.
|
||||||
|
function CategoryRetentionModal({
|
||||||
|
categoryAgeMinutes,
|
||||||
|
onChange,
|
||||||
|
disabled,
|
||||||
|
onClose
|
||||||
|
}: {
|
||||||
|
categoryAgeMinutes: Record<string, string>;
|
||||||
|
onChange: (key: string, minutes: string) => void;
|
||||||
|
disabled: boolean;
|
||||||
|
onClose: () => void;
|
||||||
|
}) {
|
||||||
|
return createPortal(
|
||||||
|
<div className="modal-backdrop" role="presentation">
|
||||||
|
<section className="modal command-modal" role="dialog" aria-modal="true" aria-label={"Per-category retention overrides"}>
|
||||||
|
<div className="modal-head">
|
||||||
|
<div>
|
||||||
|
<div className="eyebrow">{"Limits & Retention"}</div>
|
||||||
|
<h2>{"Per-category overrides"}</h2>
|
||||||
|
</div>
|
||||||
|
<button className="icon-btn" type="button" onClick={onClose} aria-label={"Close"}><X size={15} /></button>
|
||||||
|
</div>
|
||||||
|
<div className="command-body">
|
||||||
|
<p className="env-field-desc">
|
||||||
|
{"Leave a category at 0 to inherit the shared Retention age. The mode switch still applies to all of them -- these only change how old that one category's media must be."}
|
||||||
|
</p>
|
||||||
|
{CATEGORY_AGE_FIELDS.map((field) => (
|
||||||
|
<DurationField
|
||||||
|
key={field.key}
|
||||||
|
label={field.label}
|
||||||
|
help={"Inherits the shared Retention age when left at 0."}
|
||||||
|
minutes={categoryAgeMinutes[field.key] || "0"}
|
||||||
|
disabled={disabled}
|
||||||
|
onChange={(minutes) => onChange(field.key, minutes)}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
<div className="modal-actions">
|
||||||
|
<button className="btn primary" type="button" onClick={onClose}>{"Close"}</button>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</div>,
|
||||||
|
document.body
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function LimitsRetentionSection() {
|
function LimitsRetentionSection() {
|
||||||
const [loaded, setLoaded] = useState(false);
|
const [loaded, setLoaded] = useState(false);
|
||||||
const [error, setError] = useState("");
|
const [error, setError] = useState("");
|
||||||
const [initial, setInitial] = useState<Record<string, string>>({});
|
const [initial, setInitial] = useState<Record<string, string>>({});
|
||||||
|
const [blobBackend, setBlobBackend] = useState("s3");
|
||||||
const [maxTotalBytes, setMaxTotalBytes] = useState("0");
|
const [maxTotalBytes, setMaxTotalBytes] = useState("0");
|
||||||
const [minFreeBytes, setMinFreeBytes] = useState("0");
|
const [minFreeBytes, setMinFreeBytes] = useState("0");
|
||||||
const [maxUploadFileBytes, setMaxUploadFileBytes] = useState("0");
|
const [maxUploadFileBytes, setMaxUploadFileBytes] = useState("0");
|
||||||
const [retentionMode, setRetentionMode] = useState("off");
|
const [retentionMode, setRetentionMode] = useState("off");
|
||||||
const [retentionAgeMinutes, setRetentionAgeMinutes] = useState("43200");
|
const [retentionAgeMinutes, setRetentionAgeMinutes] = useState("43200");
|
||||||
|
const [categoryAgeMinutes, setCategoryAgeMinutes] = useState<Record<string, string>>({});
|
||||||
|
const [categoryModalOpen, setCategoryModalOpen] = useState(false);
|
||||||
|
const [evictionEnable, setEvictionEnable] = useState(false);
|
||||||
|
|
||||||
async function load() {
|
async function load() {
|
||||||
setError("");
|
setError("");
|
||||||
|
|
@ -378,6 +463,7 @@ function LimitsRetentionSection() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
setInitial(values);
|
setInitial(values);
|
||||||
|
setBlobBackend((values[STORAGE_ENV_KEYS.blobBackend] || "s3").trim().toLowerCase());
|
||||||
setMaxTotalBytes(values[STORAGE_ENV_KEYS.maxTotal] || "0");
|
setMaxTotalBytes(values[STORAGE_ENV_KEYS.maxTotal] || "0");
|
||||||
setMinFreeBytes(values[STORAGE_ENV_KEYS.minFree] || "0");
|
setMinFreeBytes(values[STORAGE_ENV_KEYS.minFree] || "0");
|
||||||
setMaxUploadFileBytes(values[STORAGE_ENV_KEYS.maxUploadFile] || "0");
|
setMaxUploadFileBytes(values[STORAGE_ENV_KEYS.maxUploadFile] || "0");
|
||||||
|
|
@ -385,6 +471,14 @@ function LimitsRetentionSection() {
|
||||||
setRetentionMode(mode === "orphan" || mode === "hard" ? mode : "off");
|
setRetentionMode(mode === "orphan" || mode === "hard" ? mode : "off");
|
||||||
const mins = parseDurationMinutes(values[STORAGE_ENV_KEYS.retentionMaxAge] || "720h");
|
const mins = parseDurationMinutes(values[STORAGE_ENV_KEYS.retentionMaxAge] || "720h");
|
||||||
setRetentionAgeMinutes(mins > 0 ? String(Math.max(1, Math.round(mins))) : "43200");
|
setRetentionAgeMinutes(mins > 0 ? String(Math.max(1, Math.round(mins))) : "43200");
|
||||||
|
const nextCategoryAges: Record<string, string> = {};
|
||||||
|
for (const field of CATEGORY_AGE_FIELDS) {
|
||||||
|
const raw = values[field.envKey] || "";
|
||||||
|
const categoryMins = parseDurationMinutes(raw);
|
||||||
|
nextCategoryAges[field.key] = categoryMins > 0 ? String(Math.round(categoryMins)) : "0";
|
||||||
|
}
|
||||||
|
setCategoryAgeMinutes(nextCategoryAges);
|
||||||
|
setEvictionEnable((values[STORAGE_ENV_KEYS.evictionEnable] || "false").trim().toLowerCase() === "true");
|
||||||
setLoaded(true);
|
setLoaded(true);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(errorMessage(err));
|
setError(errorMessage(err));
|
||||||
|
|
@ -404,14 +498,26 @@ function LimitsRetentionSection() {
|
||||||
[STORAGE_ENV_KEYS.retentionMode]: retentionMode,
|
[STORAGE_ENV_KEYS.retentionMode]: retentionMode,
|
||||||
[STORAGE_ENV_KEYS.retentionMaxAge]: retentionMode === "off"
|
[STORAGE_ENV_KEYS.retentionMaxAge]: retentionMode === "off"
|
||||||
? (initial[STORAGE_ENV_KEYS.retentionMaxAge] || "720h")
|
? (initial[STORAGE_ENV_KEYS.retentionMaxAge] || "720h")
|
||||||
: `${Math.max(1, Math.round(Number(retentionAgeMinutes || "0")))}m`
|
// 0 is a legitimate value here: "no sweep by default", left for
|
||||||
|
// per-category overrides below to opt specific categories in. Don't
|
||||||
|
// clamp it up to a minimum of 1 -- that used to make it impossible to
|
||||||
|
// ever save a 0 global age.
|
||||||
|
: `${Math.max(0, Math.round(Number(retentionAgeMinutes || "0")))}m`,
|
||||||
|
[STORAGE_ENV_KEYS.evictionEnable]: evictionEnable ? "true" : "false"
|
||||||
};
|
};
|
||||||
|
// Per-category overrides: a field left at "inherit global" (0) saves as
|
||||||
|
// empty rather than a redundant explicit duration, matching the "unset
|
||||||
|
// means inherit" contract on the server side.
|
||||||
|
for (const field of CATEGORY_AGE_FIELDS) {
|
||||||
|
const mins = Math.round(Number(categoryAgeMinutes[field.key] || "0"));
|
||||||
|
next[field.envKey] = mins > 0 ? `${mins}m` : "";
|
||||||
|
}
|
||||||
const changed: Record<string, string> = {};
|
const changed: Record<string, string> = {};
|
||||||
for (const [key, value] of Object.entries(next)) {
|
for (const [key, value] of Object.entries(next)) {
|
||||||
if ((initial[key] ?? "") !== value) changed[key] = value;
|
if ((initial[key] ?? "") !== value) changed[key] = value;
|
||||||
}
|
}
|
||||||
return changed;
|
return changed;
|
||||||
}, [maxTotalBytes, minFreeBytes, maxUploadFileBytes, retentionMode, retentionAgeMinutes, initial]);
|
}, [maxTotalBytes, minFreeBytes, maxUploadFileBytes, retentionMode, retentionAgeMinutes, categoryAgeMinutes, evictionEnable, initial]);
|
||||||
|
|
||||||
const hasChanges = Object.keys(pendingValues).length > 0;
|
const hasChanges = Object.keys(pendingValues).length > 0;
|
||||||
const uploadCeilingExceeded = Number(maxUploadFileBytes || "0") > PROTOCOL_UPLOAD_CEILING_BYTES;
|
const uploadCeilingExceeded = Number(maxUploadFileBytes || "0") > PROTOCOL_UPLOAD_CEILING_BYTES;
|
||||||
|
|
@ -434,12 +540,14 @@ function LimitsRetentionSection() {
|
||||||
bytes={maxTotalBytes}
|
bytes={maxTotalBytes}
|
||||||
onChange={setMaxTotalBytes}
|
onChange={setMaxTotalBytes}
|
||||||
/>
|
/>
|
||||||
|
{blobBackend === "localfs" && (
|
||||||
<ByteSizeField
|
<ByteSizeField
|
||||||
label={"Min free space guard"}
|
label={"Min free space guard"}
|
||||||
help={"localfs backend only: reject new uploads once real free disk space falls below this. Empty/0 disables the check."}
|
help={"localfs backend only: reject new uploads once real free disk space falls below this. Empty/0 disables the check."}
|
||||||
bytes={minFreeBytes}
|
bytes={minFreeBytes}
|
||||||
onChange={setMinFreeBytes}
|
onChange={setMinFreeBytes}
|
||||||
/>
|
/>
|
||||||
|
)}
|
||||||
<ByteSizeField
|
<ByteSizeField
|
||||||
label={"Max single file size"}
|
label={"Max single file size"}
|
||||||
help={"Reject a single upload once its total assembled size exceeds this. Empty/0 = unlimited, bounded only by the protocol's own ~4GB per-file ceiling."}
|
help={"Reject a single upload once its total assembled size exceeds this. Empty/0 = unlimited, bounded only by the protocol's own ~4GB per-file ceiling."}
|
||||||
|
|
@ -471,9 +579,10 @@ function LimitsRetentionSection() {
|
||||||
<DurationField
|
<DurationField
|
||||||
label={"Retention age"}
|
label={"Retention age"}
|
||||||
help={
|
help={
|
||||||
retentionMode === "hard"
|
(retentionMode === "hard"
|
||||||
? "How old the media itself must be, counted from when it was uploaded, before its bytes are purged."
|
? "How old the media itself must be, counted from when it was uploaded, before its bytes are purged."
|
||||||
: "How long a document or photo must have had zero references before its file is deleted."
|
: "How long a document or photo must have had zero references before its file is deleted.") +
|
||||||
|
" Set to 0 to disable the sweep by default and only clean up categories you explicitly override below."
|
||||||
}
|
}
|
||||||
minutes={retentionAgeMinutes}
|
minutes={retentionAgeMinutes}
|
||||||
disabled={retentionMode === "off"}
|
disabled={retentionMode === "off"}
|
||||||
|
|
@ -481,6 +590,37 @@ function LimitsRetentionSection() {
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div className="attr-block" style={{ marginTop: "1.5em" }}>
|
||||||
|
<button className="btn icon-text" type="button" onClick={() => setCategoryModalOpen(true)}>
|
||||||
|
<Settings2 size={15} /> {"Per-category overrides..."}
|
||||||
|
</button>
|
||||||
|
<span className="env-field-desc">
|
||||||
|
{"Optional -- give Photo/Video/GIF/Music/Voice/File/Avatar their own retention age instead of the shared one above."}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
{categoryModalOpen && (
|
||||||
|
<CategoryRetentionModal
|
||||||
|
categoryAgeMinutes={categoryAgeMinutes}
|
||||||
|
onChange={(key, minutes) => setCategoryAgeMinutes((prev) => ({ ...prev, [key]: minutes }))}
|
||||||
|
disabled={retentionMode === "off"}
|
||||||
|
onClose={() => setCategoryModalOpen(false)}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div className="attr-block" style={{ marginTop: "1.5em" }}>
|
||||||
|
<label className="checkline">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={evictionEnable}
|
||||||
|
onChange={(event) => setEvictionEnable(event.target.checked)}
|
||||||
|
/>
|
||||||
|
{" "}{"Actively reclaim space once over budget"}
|
||||||
|
</label>
|
||||||
|
<p className="env-field-desc">
|
||||||
|
{"Once total physical storage exceeds the Max total storage budget above, actively delete the oldest files (regardless of category or age) until back under budget -- the same way \"hard\" retention mode purges files. Independent of the retention mode above: this can run even when that's Off. Off by default, since this changes the storage budget from block-new-uploads-only to also reclaiming from existing files."}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div className="gift-table-actions env-save-row">
|
<div className="gift-table-actions env-save-row">
|
||||||
<ActionButton
|
<ActionButton
|
||||||
tone="warn"
|
tone="warn"
|
||||||
|
|
|
||||||
|
|
@ -550,6 +550,25 @@ func webPagePreviewOption(cfg config.Config) filesapp.Option {
|
||||||
return filesapp.WithWebPagePreview(cfg.WebPagePreviewMaxBytes, cfg.WebPagePreviewRatePerMin)
|
return filesapp.WithWebPagePreview(cfg.WebPagePreviewMaxBytes, cfg.WebPagePreviewRatePerMin)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// fastestPositiveDuration returns the smaller of a and b, treating a
|
||||||
|
// non-positive value as "not configured" rather than as the smallest
|
||||||
|
// possible duration -- 0 only when both are non-positive (nothing
|
||||||
|
// configured). Used to derive the storage retention sweep's ticker cadence
|
||||||
|
// from whichever of the shared default age and its per-category overrides
|
||||||
|
// asks to run soonest.
|
||||||
|
func fastestPositiveDuration(a, b time.Duration) time.Duration {
|
||||||
|
if a <= 0 {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
if b <= 0 {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
if a < b {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
func run(logger *zap.Logger) error {
|
func run(logger *zap.Logger) error {
|
||||||
cfg, err := config.Load()
|
cfg, err := config.Load()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -933,6 +952,8 @@ func run(logger *zap.Logger) error {
|
||||||
filesapp.WithAdditionalBlobBackend(additionalBlobBackend),
|
filesapp.WithAdditionalBlobBackend(additionalBlobBackend),
|
||||||
filesapp.WithSpaceGuard(spaceGuard),
|
filesapp.WithSpaceGuard(spaceGuard),
|
||||||
filesapp.WithMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes),
|
filesapp.WithMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes),
|
||||||
|
filesapp.WithStorageRetentionAges(cfg.StorageRetentionMaxAge, cfg.StorageRetentionMaxAgeByCategory, cfg.StorageRetentionMaxAgeAvatar),
|
||||||
|
filesapp.WithStorageMaxTotalBytes(cfg.StorageMaxTotalBytes),
|
||||||
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
|
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
|
||||||
externalMediaOption(cfg),
|
externalMediaOption(cfg),
|
||||||
webPagePreviewOption(cfg),
|
webPagePreviewOption(cfg),
|
||||||
|
|
@ -1081,12 +1102,31 @@ func run(logger *zap.Logger) error {
|
||||||
// TELESRV_STORAGE_RETENTION_MODE is a single 3-way switch: at most one of
|
// TELESRV_STORAGE_RETENTION_MODE is a single 3-way switch: at most one of
|
||||||
// the orphan-only (safe) and hard (age-based, ignores live references)
|
// the orphan-only (safe) and hard (age-based, ignores live references)
|
||||||
// media sweeps is ever wired in, matching "off"/"orphan"/"hard".
|
// media sweeps is ever wired in, matching "off"/"orphan"/"hard".
|
||||||
|
// The worker's own maxAge parameter only drives how often the sweep
|
||||||
|
// ticks (internal/app/maintenance.RetentionWorker.mediaRetentionInterval)
|
||||||
|
// -- the real per-category cutoff math lives entirely in
|
||||||
|
// files.Service (WithStorageRetentionAges above). Passing the raw shared
|
||||||
|
// default here would tick as slowly as a 30-day default even when a
|
||||||
|
// TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY> override asks for a much
|
||||||
|
// shorter age (or, if the shared default is 0 -- "disabled by default,
|
||||||
|
// only specific categories opt in" -- would disable the sweep outright,
|
||||||
|
// since a 0 maxAge here used to gate the whole sweep off). Use the
|
||||||
|
// fastest positive age across the shared default and every override
|
||||||
|
// instead, so the ticker -- and the sweep-enabled gate -- reflect
|
||||||
|
// whatever is actually configured to run soonest.
|
||||||
|
fastestRetentionAge := fastestPositiveDuration(cfg.StorageRetentionMaxAge, cfg.StorageRetentionMaxAgeAvatar)
|
||||||
|
for _, age := range cfg.StorageRetentionMaxAgeByCategory {
|
||||||
|
fastestRetentionAge = fastestPositiveDuration(fastestRetentionAge, age)
|
||||||
|
}
|
||||||
switch cfg.StorageRetentionMode {
|
switch cfg.StorageRetentionMode {
|
||||||
case config.StorageRetentionModeOrphan:
|
case config.StorageRetentionModeOrphan:
|
||||||
retentionWorker = retentionWorker.WithOrphanedMediaRetention(filesService, cfg.StorageRetentionMaxAge)
|
retentionWorker = retentionWorker.WithOrphanedMediaRetention(filesService, fastestRetentionAge)
|
||||||
case config.StorageRetentionModeHard:
|
case config.StorageRetentionModeHard:
|
||||||
retentionWorker = retentionWorker.WithHardMediaRetention(filesService, cfg.StorageRetentionMaxAge)
|
retentionWorker = retentionWorker.WithHardMediaRetention(filesService, fastestRetentionAge)
|
||||||
}
|
}
|
||||||
|
// Active eviction is independent of TELESRV_STORAGE_RETENTION_MODE (can
|
||||||
|
// run even when that's "off") and reuses the same media sweep ticker.
|
||||||
|
retentionWorker = retentionWorker.WithStorageEviction(filesService, cfg.StorageEvictionEnable)
|
||||||
go retentionWorker.Run(ctx)
|
go retentionWorker.Run(ctx)
|
||||||
go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"),
|
go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"),
|
||||||
cfg.UploadPartTTL,
|
cfg.UploadPartTTL,
|
||||||
|
|
@ -1383,6 +1423,13 @@ func run(logger *zap.Logger) error {
|
||||||
messageapp.WithSendPermissionChecker(adminService),
|
messageapp.WithSendPermissionChecker(adminService),
|
||||||
messageapp.WithBusinessAutomation(passwordStore, businessAutomationOptions...),
|
messageapp.WithBusinessAutomation(passwordStore, businessAutomationOptions...),
|
||||||
)
|
)
|
||||||
|
// Wires the storage retention sweep's purge-notice capability now that
|
||||||
|
// both edit-capable app services exist -- filesService (and the
|
||||||
|
// background retentionWorker goroutine reading it) was constructed
|
||||||
|
// earlier, before either was available. A sweep tick that races ahead of
|
||||||
|
// this call simply finds no notifier yet and skips the notice for that
|
||||||
|
// tick (best-effort, see files.SetRetentionPurgeNotifier).
|
||||||
|
filesService.SetRetentionPurgeNotifier(messagesService, channelsService)
|
||||||
moderationService := moderationapp.NewService(
|
moderationService := moderationapp.NewService(
|
||||||
moderationReportStore,
|
moderationReportStore,
|
||||||
moderationapp.WithMessageReaders(messagesService, channelsService),
|
moderationapp.WithMessageReaders(messagesService, channelsService),
|
||||||
|
|
|
||||||
|
|
@ -2,10 +2,43 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
telegramloginapp "telesrv/internal/app/telegramlogin"
|
telegramloginapp "telesrv/internal/app/telegramlogin"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// TestFastestPositiveDurationIgnoresNonPositiveValues guards a real reported
|
||||||
|
// bug: the storage retention sweep's ticker cadence used to be driven purely
|
||||||
|
// by the shared TELESRV_STORAGE_RETENTION_MAX_AGE default (e.g. 30 days),
|
||||||
|
// even when a much shorter TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY>
|
||||||
|
// override was configured -- so a category set to "1 minute" would still
|
||||||
|
// only actually get swept on the shared default's own slow cadence (or, if
|
||||||
|
// the shared default was 0, would disable the sweep outright). The worker
|
||||||
|
// must be handed the fastest positive age across the default and every
|
||||||
|
// override instead.
|
||||||
|
func TestFastestPositiveDurationIgnoresNonPositiveValues(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
a, b time.Duration
|
||||||
|
want time.Duration
|
||||||
|
}{
|
||||||
|
{"both positive, a smaller", 30 * 24 * time.Hour, time.Minute, time.Minute},
|
||||||
|
{"both positive, b smaller", time.Minute, 30 * 24 * time.Hour, time.Minute},
|
||||||
|
{"a zero (disabled), b positive", 0, time.Minute, time.Minute},
|
||||||
|
{"a positive, b zero (disabled)", time.Minute, 0, time.Minute},
|
||||||
|
{"a negative, b positive", -time.Hour, time.Minute, time.Minute},
|
||||||
|
{"both zero (nothing configured)", 0, 0, 0},
|
||||||
|
{"both negative", -time.Hour, -time.Minute, -time.Minute},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
if got := fastestPositiveDuration(c.a, c.b); got != c.want {
|
||||||
|
t.Fatalf("fastestPositiveDuration(%v, %v) = %v, want %v", c.a, c.b, got, c.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestTelegramLoginRPCDependencyPreservesDisabledNil(t *testing.T) {
|
func TestTelegramLoginRPCDependencyPreservesDisabledNil(t *testing.T) {
|
||||||
var disabled *telegramloginapp.Service
|
var disabled *telegramloginapp.Service
|
||||||
if dependency := telegramLoginRPCDependency(disabled); dependency != nil {
|
if dependency := telegramLoginRPCDependency(disabled); dependency != nil {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
DROP INDEX IF EXISTS public.documents_category_created_at_idx;
|
||||||
|
ALTER TABLE public.documents
|
||||||
|
DROP COLUMN IF EXISTS category;
|
||||||
18
deploy/migrations/20260902000001_documents_category.up.sql
Normal file
18
deploy/migrations/20260902000001_documents_category.up.sql
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
-- Adds a per-document media category column so the storage retention sweep
|
||||||
|
-- can use a different retention age per category (Photo/Video/RoundVideo/
|
||||||
|
-- Gif/Music/Voice/File/Avatar -- see TELESRV_STORAGE_RETENTION_MAX_AGE_*)
|
||||||
|
-- instead of one shared age for every document regardless of kind.
|
||||||
|
--
|
||||||
|
-- The value mirrors domain.MediaCategory (int16): 0 = None/unclassified
|
||||||
|
-- (stickers and anything else classifyDocumentCategory doesn't tag -- these
|
||||||
|
-- always fall back to the shared global age, there is no per-category
|
||||||
|
-- override for this bucket), 2 = Video, 3 = Gif, 4 = File, 5 = Music,
|
||||||
|
-- 6 = Voice, 7 = RoundVideo. Deliberately NOT backfilled for existing rows:
|
||||||
|
-- they stay at the default 0 (global age) until they are next
|
||||||
|
-- created/edited, which is an acceptable one-time transitional behavior --
|
||||||
|
-- backfilling would require re-deriving the category from each document's
|
||||||
|
-- already-stored attributes for potentially every row in the table.
|
||||||
|
ALTER TABLE public.documents
|
||||||
|
ADD COLUMN category smallint NOT NULL DEFAULT 0;
|
||||||
|
|
||||||
|
CREATE INDEX documents_category_created_at_idx ON public.documents (category, created_at);
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
ALTER TABLE public.account_settings
|
||||||
|
DROP COLUMN IF EXISTS disallow_unlimited_stargifts,
|
||||||
|
DROP COLUMN IF EXISTS disallow_limited_stargifts,
|
||||||
|
DROP COLUMN IF EXISTS disallow_unique_stargifts,
|
||||||
|
DROP COLUMN IF EXISTS disallow_premium_gifts,
|
||||||
|
DROP COLUMN IF EXISTS disallow_stargifts_from_channels;
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
-- account_settings gained global gift-reception switches (Layer 228
|
||||||
|
-- disallowedGifts) in code (internal/domain/account.go's DisallowedGifts,
|
||||||
|
-- read/written by internal/store/postgres/account.go's GetAccountSettings/
|
||||||
|
-- GetAccountSettingsBatch/SaveAccountSettings) without a matching migration
|
||||||
|
-- ever being added -- every account_settings read/write has been failing
|
||||||
|
-- with "column does not exist" on any database that never got these columns,
|
||||||
|
-- which breaks not just gift settings but every RPC that loads cached
|
||||||
|
-- account settings (privacy, account TTL, contact sign-up notification,
|
||||||
|
-- sendMessage's privacy check, etc).
|
||||||
|
ALTER TABLE public.account_settings
|
||||||
|
ADD COLUMN IF NOT EXISTS disallow_unlimited_stargifts boolean NOT NULL DEFAULT false,
|
||||||
|
ADD COLUMN IF NOT EXISTS disallow_limited_stargifts boolean NOT NULL DEFAULT false,
|
||||||
|
ADD COLUMN IF NOT EXISTS disallow_unique_stargifts boolean NOT NULL DEFAULT false,
|
||||||
|
ADD COLUMN IF NOT EXISTS disallow_premium_gifts boolean NOT NULL DEFAULT false,
|
||||||
|
ADD COLUMN IF NOT EXISTS disallow_stargifts_from_channels boolean NOT NULL DEFAULT false;
|
||||||
|
|
@ -1514,6 +1514,22 @@ func (s *Service) EditMessage(ctx context.Context, userID int64, req domain.Edit
|
||||||
return s.channels.EditChannelMessage(ctx, req)
|
return s.channels.EditChannelMessage(ctx, req)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EditChannelMessageInternal performs a server-internal channel message edit
|
||||||
|
// with no acting user -- currently only the storage retention sweep
|
||||||
|
// (internal/app/files.notifyRetentionPurge), which needs to turn a purged
|
||||||
|
// message into a visible notice but has no RPC caller/channel member behind
|
||||||
|
// it. Bypasses the userID==0 gate EditMessage enforces for ordinary
|
||||||
|
// RPC-driven edits; the store's own RetentionPurge bypass (see
|
||||||
|
// EditChannelMessageRequest.RetentionPurge) enforces the actual permission
|
||||||
|
// bypass semantics, so this refuses anything that isn't actually a retention
|
||||||
|
// purge request rather than becoming a second general-purpose edit path.
|
||||||
|
func (s *Service) EditChannelMessageInternal(ctx context.Context, req domain.EditChannelMessageRequest) (domain.EditChannelMessageResult, error) {
|
||||||
|
if s == nil || s.channels == nil || !req.RetentionPurge || req.RetentionPurgeAction == nil || req.ChannelID == 0 || req.ID <= 0 {
|
||||||
|
return domain.EditChannelMessageResult{}, domain.ErrChannelInvalid
|
||||||
|
}
|
||||||
|
return s.channels.EditChannelMessage(ctx, req)
|
||||||
|
}
|
||||||
|
|
||||||
// GetInlineBotMessage returns one live channel message addressed by a signed inline id.
|
// GetInlineBotMessage returns one live channel message addressed by a signed inline id.
|
||||||
func (s *Service) GetInlineBotMessage(ctx context.Context, botID, channelID int64, id int) (domain.Channel, domain.ChannelMessage, bool, error) {
|
func (s *Service) GetInlineBotMessage(ctx context.Context, botID, channelID int64, id int) (domain.Channel, domain.ChannelMessage, bool, error) {
|
||||||
if s == nil || s.channels == nil || botID == 0 || channelID == 0 || id <= 0 || id > domain.MaxMessageBoxID {
|
if s == nil || s.channels == nil || botID == 0 || channelID == 0 || id <= 0 || id > domain.MaxMessageBoxID {
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ package files
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
|
@ -10,14 +11,52 @@ import (
|
||||||
"telesrv/internal/domain"
|
"telesrv/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// hardRetentionDocumentCategories enumerates every documents.category bucket
|
||||||
|
// the per-category retention sweep loops over. MediaCategoryNone covers
|
||||||
|
// unclassified documents (stickers and anything else classifyDocumentCategory
|
||||||
|
// doesn't tag) and always uses the shared global age -- there is no
|
||||||
|
// per-category override for that bucket. Photo/Avatar are not in this list:
|
||||||
|
// photos have no category column and are instead split by a profile_photos
|
||||||
|
// join (see categoryRetentionAge/avatarRetentionAge below and the dedicated
|
||||||
|
// photo/avatar query variants).
|
||||||
|
var hardRetentionDocumentCategories = []domain.MediaCategory{
|
||||||
|
domain.MediaCategoryNone,
|
||||||
|
domain.MediaCategoryVideo,
|
||||||
|
domain.MediaCategoryGif,
|
||||||
|
domain.MediaCategoryFile,
|
||||||
|
domain.MediaCategoryMusic,
|
||||||
|
domain.MediaCategoryVoice,
|
||||||
|
domain.MediaCategoryRoundVideo,
|
||||||
|
}
|
||||||
|
|
||||||
|
// categoryRetentionAge returns the effective retention age for a document
|
||||||
|
// media category: its configured override if positive, otherwise the shared
|
||||||
|
// global age.
|
||||||
|
func (s *Service) categoryRetentionAge(category domain.MediaCategory) time.Duration {
|
||||||
|
if age, ok := s.storageRetentionCategoryAges[category]; ok && age > 0 {
|
||||||
|
return age
|
||||||
|
}
|
||||||
|
return s.storageRetentionGlobalMaxAge
|
||||||
|
}
|
||||||
|
|
||||||
|
// avatarRetentionAge is the Photo category's counterpart for photos currently
|
||||||
|
// active as someone's avatar -- see categoryRetentionAge.
|
||||||
|
func (s *Service) avatarRetentionAge() time.Duration {
|
||||||
|
if s.storageRetentionAvatarMaxAge > 0 {
|
||||||
|
return s.storageRetentionAvatarMaxAge
|
||||||
|
}
|
||||||
|
return s.storageRetentionGlobalMaxAge
|
||||||
|
}
|
||||||
|
|
||||||
// mediaRetentionStore is implemented by store.MediaStore backends that
|
// mediaRetentionStore is implemented by store.MediaStore backends that
|
||||||
// support the storage retention sweep (currently only the Postgres store).
|
// support the storage retention sweep (currently only the Postgres store).
|
||||||
// A type assertion, not a MediaStore interface method, keeps these
|
// A type assertion, not a MediaStore interface method, keeps these
|
||||||
// admin/maintenance-only queries out of the hot RPC-facing interface --
|
// admin/maintenance-only queries out of the hot RPC-facing interface --
|
||||||
// same convention as photoBatchStore above.
|
// same convention as photoBatchStore above.
|
||||||
type mediaRetentionStore interface {
|
type mediaRetentionStore interface {
|
||||||
ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
ListOrphanedDocumentIDsOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error)
|
||||||
ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||||
|
ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||||
CountFileBlobRefs(ctx context.Context, backend, objectKey string) (int, error)
|
CountFileBlobRefs(ctx context.Context, backend, objectKey string) (int, error)
|
||||||
DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||||
DeletePhotoAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
DeletePhotoAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||||
|
|
@ -31,28 +70,51 @@ type mediaRetentionStore interface {
|
||||||
// orphaned one once it's old enough. The delete methods physically
|
// orphaned one once it's old enough. The delete methods physically
|
||||||
// remove only the file_blobs row(s)/bytes, never the document/photo
|
// remove only the file_blobs row(s)/bytes, never the document/photo
|
||||||
// metadata row -- see DeleteFileBlobsForDocument's doc comment.
|
// metadata row -- see DeleteFileBlobsForDocument's doc comment.
|
||||||
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error)
|
||||||
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||||
|
ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||||
DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||||
DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||||
|
|
||||||
|
// -- active eviction (TELESRV_STORAGE_EVICTION_ENABLE) --
|
||||||
|
SumFileBlobBytes(ctx context.Context) (int64, error)
|
||||||
|
ListOldestMediaForEviction(ctx context.Context, limit int) ([]domain.EvictionCandidate, error)
|
||||||
|
|
||||||
|
// -- retention purge notice (see retention_purge.go) --
|
||||||
|
ListMediaReferences(ctx context.Context, kind domain.MediaKind, mediaID int64) ([]domain.MediaReference, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
|
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
|
||||||
// permanently deletes documents/photos that have had no live reference
|
// permanently deletes documents/photos that have had no live reference
|
||||||
// (message/profile-photo/sticker-set, see media_references) since at least
|
// (message/profile-photo/sticker-set, see media_references) since at least
|
||||||
// cutoff, along with their blob(s) -- but only physically removes bytes
|
// the per-category cutoff derived from now, along with their blob(s) -- but
|
||||||
// from the backend once confirming no other file_blobs row still needs the
|
// only physically removes bytes from the backend once confirming no other
|
||||||
// object, since content-addressed storage means the same bytes can be
|
// file_blobs row still needs the object, since content-addressed storage
|
||||||
// shared across documents/photos.
|
// means the same bytes can be shared across documents/photos. Loops one
|
||||||
func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
|
// query per document category (each with its own effective age, see
|
||||||
|
// categoryRetentionAge) plus a regular/avatar split for photos; limit applies
|
||||||
|
// per category per tick, not as one shared budget across all of them --
|
||||||
|
// simplest correct behavior, revisit only if one category starves another in
|
||||||
|
// practice.
|
||||||
|
func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, now time.Time, limit int) (int, error) {
|
||||||
store, ok := s.media.(mediaRetentionStore)
|
store, ok := s.media.(mediaRetentionStore)
|
||||||
if !ok || limit <= 0 {
|
if !ok || limit <= 0 {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
deleted := 0
|
deleted := 0
|
||||||
docIDs, err := store.ListOrphanedDocumentIDsOlderThan(ctx, cutoff, limit)
|
for _, cat := range hardRetentionDocumentCategories {
|
||||||
|
age := s.categoryRetentionAge(cat)
|
||||||
|
if age <= 0 {
|
||||||
|
// Global age is 0 (retention only enabled for other, explicitly
|
||||||
|
// overridden categories) and this category has no override of its
|
||||||
|
// own -- skip it entirely rather than treating age<=0 as an
|
||||||
|
// immediate "everything is older than now" cutoff.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cutoff := now.Add(-age)
|
||||||
|
docIDs, err := store.ListOrphanedDocumentIDsOlderThan(ctx, cat, cutoff, limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return deleted, fmt.Errorf("list orphaned documents: %w", err)
|
return deleted, fmt.Errorf("list orphaned documents (category %d): %w", cat, err)
|
||||||
}
|
}
|
||||||
for _, id := range docIDs {
|
for _, id := range docIDs {
|
||||||
blobs, err := store.DeleteDocumentAndBlobs(ctx, id)
|
blobs, err := store.DeleteDocumentAndBlobs(ctx, id)
|
||||||
|
|
@ -63,7 +125,10 @@ func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time,
|
||||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||||
deleted++
|
deleted++
|
||||||
}
|
}
|
||||||
photoIDs, err := store.ListOrphanedPhotoIDsOlderThan(ctx, cutoff, limit)
|
}
|
||||||
|
if age := s.categoryRetentionAge(domain.MediaCategoryPhoto); age > 0 {
|
||||||
|
photoCutoff := now.Add(-age)
|
||||||
|
photoIDs, err := store.ListOrphanedPhotoIDsOlderThan(ctx, photoCutoff, limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return deleted, fmt.Errorf("list orphaned photos: %w", err)
|
return deleted, fmt.Errorf("list orphaned photos: %w", err)
|
||||||
}
|
}
|
||||||
|
|
@ -76,30 +141,61 @@ func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time,
|
||||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||||
deleted++
|
deleted++
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
if age := s.avatarRetentionAge(); age > 0 {
|
||||||
|
avatarCutoff := now.Add(-age)
|
||||||
|
avatarIDs, err := store.ListAvatarOrphanedPhotoIDsOlderThan(ctx, avatarCutoff, limit)
|
||||||
|
if err != nil {
|
||||||
|
return deleted, fmt.Errorf("list orphaned avatar photos: %w", err)
|
||||||
|
}
|
||||||
|
for _, id := range avatarIDs {
|
||||||
|
blobs, err := store.DeletePhotoAndBlobs(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
s.log.Warn("delete orphaned avatar photo failed", zap.Int64("photo_id", id), zap.Error(err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||||
|
deleted++
|
||||||
|
}
|
||||||
|
}
|
||||||
return deleted, nil
|
return deleted, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteBlobBytesForMediaOlderThan implements
|
// DeleteBlobBytesForMediaOlderThan implements
|
||||||
// maintenance.HardMediaRetentionStore ("hard" retention mode): for
|
// maintenance.HardMediaRetentionStore ("hard" retention mode): for
|
||||||
// documents/photos whose upload/created_at is older than cutoff, physically
|
// documents/photos whose upload/created_at is older than the per-category
|
||||||
// deletes their blob bytes (main body + thumbnail/rendition variants) from
|
// cutoff derived from now (see categoryRetentionAge/avatarRetentionAge),
|
||||||
// the backend and removes their file_blobs rows -- REGARDLESS of whether a
|
// physically deletes their blob bytes (main body + thumbnail/rendition
|
||||||
// live message/profile-photo/sticker-set still references them. It
|
// variants) from the backend and removes their file_blobs rows --
|
||||||
// deliberately never touches the documents/photos metadata row itself: a
|
// REGARDLESS of whether a live message/profile-photo/sticker-set still
|
||||||
// message must still be able to render "here was a photo/document"
|
// references them. It deliberately never touches the documents/photos
|
||||||
// (dimensions, mime type, filename) after its bytes are gone, rather than
|
// metadata row itself: a message must still be able to render "here was a
|
||||||
// the message breaking outright. A subsequent upload.getFile for the same
|
// photo/document" (dimensions, mime type, filename) after its bytes are
|
||||||
// location key finds no file_blobs row and returns LOCATION_INVALID, which
|
// gone, rather than the message breaking outright. A subsequent
|
||||||
// stock clients already render as a "media unavailable" placeholder.
|
// upload.getFile for the same location key finds no file_blobs row and
|
||||||
func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
|
// returns LOCATION_INVALID, which stock clients already render as a "media
|
||||||
|
// unavailable" placeholder -- and every message that still embeds the purged
|
||||||
|
// media additionally gets turned into a visible retention-purge notice (see
|
||||||
|
// notifyRetentionPurge in retention_purge.go).
|
||||||
|
func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time.Time, limit int) (int, error) {
|
||||||
store, ok := s.media.(mediaRetentionStore)
|
store, ok := s.media.(mediaRetentionStore)
|
||||||
if !ok || limit <= 0 {
|
if !ok || limit <= 0 {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
purged := 0
|
purged := 0
|
||||||
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, limit)
|
for _, cat := range hardRetentionDocumentCategories {
|
||||||
|
age := s.categoryRetentionAge(cat)
|
||||||
|
if age <= 0 {
|
||||||
|
// See DeleteOrphanedOlderThan's identical guard: age<=0 means this
|
||||||
|
// category has no override and the global age is 0 (retention
|
||||||
|
// enabled only for other categories) -- skip, don't treat it as
|
||||||
|
// "everything is older than now".
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cutoff := now.Add(-age)
|
||||||
|
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, cutoff, limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return purged, fmt.Errorf("list documents for hard retention: %w", err)
|
return purged, fmt.Errorf("list documents for hard retention (category %d): %w", cat, err)
|
||||||
}
|
}
|
||||||
for _, id := range docIDs {
|
for _, id := range docIDs {
|
||||||
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
|
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
|
||||||
|
|
@ -111,13 +207,13 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff t
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||||
|
s.notifyRetentionPurge(ctx, domain.MediaKindDocument, id)
|
||||||
purged++
|
purged++
|
||||||
}
|
}
|
||||||
remaining := limit - len(docIDs)
|
|
||||||
if remaining <= 0 {
|
|
||||||
return purged, nil
|
|
||||||
}
|
}
|
||||||
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, cutoff, remaining)
|
if age := s.categoryRetentionAge(domain.MediaCategoryPhoto); age > 0 {
|
||||||
|
photoCutoff := now.Add(-age)
|
||||||
|
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, photoCutoff, limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return purged, fmt.Errorf("list photos for hard retention: %w", err)
|
return purged, fmt.Errorf("list photos for hard retention: %w", err)
|
||||||
}
|
}
|
||||||
|
|
@ -131,11 +227,93 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff t
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||||
|
s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
|
||||||
purged++
|
purged++
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
if age := s.avatarRetentionAge(); age > 0 {
|
||||||
|
avatarCutoff := now.Add(-age)
|
||||||
|
avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, avatarCutoff, limit)
|
||||||
|
if err != nil {
|
||||||
|
return purged, fmt.Errorf("list avatar photos for hard retention: %w", err)
|
||||||
|
}
|
||||||
|
for _, id := range avatarIDs {
|
||||||
|
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
s.log.Warn("hard-delete avatar photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(blobs) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||||
|
s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
|
||||||
|
purged++
|
||||||
|
}
|
||||||
|
}
|
||||||
return purged, nil
|
return purged, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EvictOldestMediaOverBudget implements maintenance.StorageEvictionStore
|
||||||
|
// (TELESRV_STORAGE_EVICTION_ENABLE): once total physical blob bytes
|
||||||
|
// (SumFileBlobBytes) exceed TELESRV_STORAGE_MAX_TOTAL_BYTES, purges the
|
||||||
|
// oldest documents/photos overall -- interleaved by created_at across both
|
||||||
|
// tables, regardless of category or age -- reusing the exact same blob-purge
|
||||||
|
// (DeleteFileBlobsForDocument/DeleteFileBlobsForPhoto) and retention-purge
|
||||||
|
// notice primitive as "hard" mode. Stops once the running total (tracked
|
||||||
|
// locally from each purge's returned blob sizes, avoiding a re-query per
|
||||||
|
// item) is back under budget, or limit purges have happened this tick,
|
||||||
|
// whichever comes first -- bounding how much one tick can reclaim at once.
|
||||||
|
func (s *Service) EvictOldestMediaOverBudget(ctx context.Context, limit int) (int, error) {
|
||||||
|
store, ok := s.media.(mediaRetentionStore)
|
||||||
|
if !ok || limit <= 0 || s.storageMaxTotalBytes <= 0 {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
total, err := store.SumFileBlobBytes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("sum file blob bytes: %w", err)
|
||||||
|
}
|
||||||
|
if total <= s.storageMaxTotalBytes {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
candidates, err := store.ListOldestMediaForEviction(ctx, limit)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("list oldest media for eviction: %w", err)
|
||||||
|
}
|
||||||
|
sort.Slice(candidates, func(i, j int) bool { return candidates[i].CreatedAt.Before(candidates[j].CreatedAt) })
|
||||||
|
evicted := 0
|
||||||
|
for _, c := range candidates {
|
||||||
|
if evicted >= limit || total <= s.storageMaxTotalBytes {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
var blobs []domain.FileBlob
|
||||||
|
var delErr error
|
||||||
|
switch c.Kind {
|
||||||
|
case domain.MediaKindDocument:
|
||||||
|
blobs, delErr = store.DeleteFileBlobsForDocument(ctx, c.MediaID)
|
||||||
|
case domain.MediaKindPhoto:
|
||||||
|
blobs, delErr = store.DeleteFileBlobsForPhoto(ctx, c.MediaID)
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if delErr != nil {
|
||||||
|
s.log.Warn("active storage eviction blob purge failed",
|
||||||
|
zap.String("media_kind", string(c.Kind)), zap.Int64("media_id", c.MediaID), zap.Error(delErr))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(blobs) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||||
|
for _, b := range blobs {
|
||||||
|
total -= b.Size
|
||||||
|
}
|
||||||
|
s.notifyRetentionPurge(ctx, c.Kind, c.MediaID)
|
||||||
|
evicted++
|
||||||
|
}
|
||||||
|
return evicted, nil
|
||||||
|
}
|
||||||
|
|
||||||
// deleteDocumentNowIfUnreferenced is the immediate counterpart to the
|
// deleteDocumentNowIfUnreferenced is the immediate counterpart to the
|
||||||
// age-based sweep DeleteOrphanedOlderThan runs in the background: orphans
|
// age-based sweep DeleteOrphanedOlderThan runs in the background: orphans
|
||||||
// id right now (skipping the grace period) and, only if that succeeds --
|
// id right now (skipping the grace period) and, only if that succeeds --
|
||||||
|
|
|
||||||
153
internal/app/files/retention_category_zero_test.go
Normal file
153
internal/app/files/retention_category_zero_test.go
Normal file
|
|
@ -0,0 +1,153 @@
|
||||||
|
package files
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.uber.org/zap"
|
||||||
|
|
||||||
|
"telesrv/internal/domain"
|
||||||
|
"telesrv/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeCategorySweepStore records which category/photo/avatar queries
|
||||||
|
// DeleteBlobBytesForMediaOlderThan/DeleteOrphanedOlderThan actually issue, so
|
||||||
|
// the zero-age skip guard can be checked without a live database. Embeds a
|
||||||
|
// nil store.MediaStore so it satisfies that (large) interface for free via
|
||||||
|
// promoted methods that must never actually be called here -- only the
|
||||||
|
// mediaRetentionStore subset overridden below is exercised.
|
||||||
|
type fakeCategorySweepStore struct {
|
||||||
|
store.MediaStore
|
||||||
|
queriedDocCategories []domain.MediaCategory
|
||||||
|
queriedPhotos bool
|
||||||
|
queriedAvatars bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeCategorySweepStore) ListDocumentIDsForHardRetentionOlderThan(_ context.Context, category domain.MediaCategory, _ time.Time, _ int) ([]int64, error) {
|
||||||
|
f.queriedDocCategories = append(f.queriedDocCategories, category)
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeCategorySweepStore) ListPhotoIDsForHardRetentionOlderThan(context.Context, time.Time, int) ([]int64, error) {
|
||||||
|
f.queriedPhotos = true
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeCategorySweepStore) ListAvatarPhotoIDsForHardRetentionOlderThan(context.Context, time.Time, int) ([]int64, error) {
|
||||||
|
f.queriedAvatars = true
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeCategorySweepStore) ListOrphanedDocumentIDsOlderThan(_ context.Context, category domain.MediaCategory, _ time.Time, _ int) ([]int64, error) {
|
||||||
|
f.queriedDocCategories = append(f.queriedDocCategories, category)
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeCategorySweepStore) ListOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
|
||||||
|
f.queriedPhotos = true
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeCategorySweepStore) ListAvatarOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
|
||||||
|
f.queriedAvatars = true
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// The rest of mediaRetentionStore's methods aren't part of store.MediaStore
|
||||||
|
// (see that interface's own doc comment -- they're kept out of the hot
|
||||||
|
// RPC-facing interface on purpose), so embedding store.MediaStore alone
|
||||||
|
// doesn't provide them. Stub them out; this test never exercises them since
|
||||||
|
// every List* above returns no candidates.
|
||||||
|
func (f *fakeCategorySweepStore) CountFileBlobRefs(context.Context, string, string) (int, error) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
func (f *fakeCategorySweepStore) DeleteDocumentAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (f *fakeCategorySweepStore) DeletePhotoAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (f *fakeCategorySweepStore) OrphanDocumentIfUnreferenced(context.Context, int64) (bool, error) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
func (f *fakeCategorySweepStore) DeleteFileBlobsForDocument(context.Context, int64) ([]domain.FileBlob, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (f *fakeCategorySweepStore) DeleteFileBlobsForPhoto(context.Context, int64) ([]domain.FileBlob, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (f *fakeCategorySweepStore) SumFileBlobBytes(context.Context) (int64, error) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
func (f *fakeCategorySweepStore) ListOldestMediaForEviction(context.Context, int) ([]domain.EvictionCandidate, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (f *fakeCategorySweepStore) ListMediaReferences(context.Context, domain.MediaKind, int64) ([]domain.MediaReference, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestZeroGlobalRetentionAgeSkipsCategoriesWithoutOverride guards the bug
|
||||||
|
// reported live: setting the shared Retention age to 0 (meaning "only clean
|
||||||
|
// up categories I explicitly override") used to be indistinguishable from
|
||||||
|
// "everything is older than right now" at the per-category cutoff math,
|
||||||
|
// which would have hard-purged every uncategorized/photo/avatar item
|
||||||
|
// immediately. The sweep must instead skip any category whose *effective*
|
||||||
|
// age (its own override, or the 0 global) is <=0 entirely -- no query issued
|
||||||
|
// for it at all -- while still sweeping a category that has a real,
|
||||||
|
// positive override.
|
||||||
|
func TestZeroGlobalRetentionAgeSkipsCategoriesWithoutOverride(t *testing.T) {
|
||||||
|
fake := &fakeCategorySweepStore{}
|
||||||
|
s := &Service{
|
||||||
|
media: fake,
|
||||||
|
log: zap.NewNop(),
|
||||||
|
storageRetentionGlobalMaxAge: 0,
|
||||||
|
storageRetentionCategoryAges: map[domain.MediaCategory]time.Duration{
|
||||||
|
domain.MediaCategoryVideo: 24 * time.Hour,
|
||||||
|
},
|
||||||
|
storageRetentionAvatarMaxAge: 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := s.DeleteBlobBytesForMediaOlderThan(context.Background(), time.Now(), 50); err != nil {
|
||||||
|
t.Fatalf("DeleteBlobBytesForMediaOlderThan: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(fake.queriedDocCategories) != 1 || fake.queriedDocCategories[0] != domain.MediaCategoryVideo {
|
||||||
|
t.Fatalf("queried document categories = %v, want only [Video]", fake.queriedDocCategories)
|
||||||
|
}
|
||||||
|
if fake.queriedPhotos {
|
||||||
|
t.Fatal("queried regular photos with no photo override and global age 0, want skipped")
|
||||||
|
}
|
||||||
|
if fake.queriedAvatars {
|
||||||
|
t.Fatal("queried avatar photos with no avatar override and global age 0, want skipped")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestZeroGlobalRetentionAgeSkipsOrphanSweepCategoriesWithoutOverride is the
|
||||||
|
// orphan-mode counterpart of the hard-mode test above.
|
||||||
|
func TestZeroGlobalRetentionAgeSkipsOrphanSweepCategoriesWithoutOverride(t *testing.T) {
|
||||||
|
fake := &fakeCategorySweepStore{}
|
||||||
|
s := &Service{
|
||||||
|
media: fake,
|
||||||
|
log: zap.NewNop(),
|
||||||
|
storageRetentionGlobalMaxAge: 0,
|
||||||
|
storageRetentionCategoryAges: map[domain.MediaCategory]time.Duration{
|
||||||
|
domain.MediaCategoryVoice: 48 * time.Hour,
|
||||||
|
},
|
||||||
|
storageRetentionAvatarMaxAge: 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := s.DeleteOrphanedOlderThan(context.Background(), time.Now(), 50); err != nil {
|
||||||
|
t.Fatalf("DeleteOrphanedOlderThan: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(fake.queriedDocCategories) != 1 || fake.queriedDocCategories[0] != domain.MediaCategoryVoice {
|
||||||
|
t.Fatalf("queried document categories = %v, want only [Voice]", fake.queriedDocCategories)
|
||||||
|
}
|
||||||
|
if fake.queriedPhotos {
|
||||||
|
t.Fatal("queried regular photos with no photo override and global age 0, want skipped")
|
||||||
|
}
|
||||||
|
if fake.queriedAvatars {
|
||||||
|
t.Fatal("queried avatar photos with no avatar override and global age 0, want skipped")
|
||||||
|
}
|
||||||
|
}
|
||||||
204
internal/app/files/retention_purge.go
Normal file
204
internal/app/files/retention_purge.go
Normal file
|
|
@ -0,0 +1,204 @@
|
||||||
|
package files
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"go.uber.org/zap"
|
||||||
|
|
||||||
|
"telesrv/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RetentionPurgeMessageEditor is the private-message capability the storage
|
||||||
|
// retention sweep needs to turn a hard-retention/eviction blob purge into a
|
||||||
|
// visible messageActionCustomAction notice. Satisfied by
|
||||||
|
// internal/app/messages.Service (GetMessages resolves the box's peer --
|
||||||
|
// EditMessageRequest requires it, and message_box ref_key only encodes
|
||||||
|
// owner_user_id+box_id -- then EditMessage performs the actual in-place
|
||||||
|
// edit with RetentionPurge set).
|
||||||
|
type RetentionPurgeMessageEditor interface {
|
||||||
|
GetMessages(ctx context.Context, userID int64, ids []int) (domain.MessageList, error)
|
||||||
|
EditMessage(ctx context.Context, userID int64, req domain.EditMessageRequest) (domain.EditMessageResult, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RetentionPurgeChannelEditor is the channel-message counterpart of
|
||||||
|
// RetentionPurgeMessageEditor. Satisfied by internal/app/channels.Service.
|
||||||
|
// Unlike private messages, EditChannelMessageRequest needs no peer lookup --
|
||||||
|
// channelMessageRefKey already encodes the channel id directly. Uses
|
||||||
|
// EditChannelMessageInternal (not the ordinary EditMessage) since this is a
|
||||||
|
// server-internal edit with no acting user/channel member behind it.
|
||||||
|
type RetentionPurgeChannelEditor interface {
|
||||||
|
EditChannelMessageInternal(ctx context.Context, req domain.EditChannelMessageRequest) (domain.EditChannelMessageResult, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetRetentionPurgeNotifier wires the private-message/channel-message edit
|
||||||
|
// capability the storage retention sweep needs to turn a hard-retention/
|
||||||
|
// eviction blob purge into a visible messageActionCustomAction notice on
|
||||||
|
// every message still embedding the purged document/photo. Both app-layer
|
||||||
|
// services are constructed after this Service in cmd/telesrv/main.go (and
|
||||||
|
// the background retention sweep goroutine is started before either exists),
|
||||||
|
// so this is a post-construction setter rather than a NewService option: a
|
||||||
|
// sweep tick that races ahead of this call simply finds both fields nil and
|
||||||
|
// skips the notice for that tick, same as any other per-reference failure
|
||||||
|
// below (best-effort -- the underlying blob purge has already committed and
|
||||||
|
// must never be undone or retried because of a notice failure).
|
||||||
|
func (s *Service) SetRetentionPurgeNotifier(messages RetentionPurgeMessageEditor, channels RetentionPurgeChannelEditor) {
|
||||||
|
s.retentionNotifyMu.Lock()
|
||||||
|
defer s.retentionNotifyMu.Unlock()
|
||||||
|
s.retentionMessages = messages
|
||||||
|
s.retentionChannels = channels
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) retentionNotifier() (RetentionPurgeMessageEditor, RetentionPurgeChannelEditor) {
|
||||||
|
s.retentionNotifyMu.RLock()
|
||||||
|
defer s.retentionNotifyMu.RUnlock()
|
||||||
|
return s.retentionMessages, s.retentionChannels
|
||||||
|
}
|
||||||
|
|
||||||
|
// notifyRetentionPurge turns a just-completed hard-retention/eviction blob
|
||||||
|
// purge of a document/photo into a visible service-message notice on every
|
||||||
|
// message that still embeds it. profile_photo/sticker_set/gift references
|
||||||
|
// have no message to edit and are skipped. Best-effort throughout: a lookup
|
||||||
|
// or edit failure is logged and never propagated -- the underlying blob purge
|
||||||
|
// has already committed and must not be undone or retried because of this.
|
||||||
|
func (s *Service) notifyRetentionPurge(ctx context.Context, kind domain.MediaKind, mediaID int64) {
|
||||||
|
messages, channels := s.retentionNotifier()
|
||||||
|
if messages == nil && channels == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
store, ok := s.media.(mediaRetentionStore)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
refs, err := store.ListMediaReferences(ctx, kind, mediaID)
|
||||||
|
if err != nil {
|
||||||
|
s.log.Warn("list media references for retention purge notice failed",
|
||||||
|
zap.String("media_kind", string(kind)), zap.Int64("media_id", mediaID), zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, ref := range refs {
|
||||||
|
switch ref.RefKind {
|
||||||
|
case domain.MediaRefKindMessageBox:
|
||||||
|
s.notifyRetentionPurgeMessageBox(ctx, messages, ref.RefKey)
|
||||||
|
case domain.MediaRefKindChannelMessage:
|
||||||
|
s.notifyRetentionPurgeChannelMessage(ctx, channels, ref.RefKey)
|
||||||
|
case domain.MediaRefKindProfilePhoto, domain.MediaRefKindStickerSet, domain.MediaRefKindGift:
|
||||||
|
// No message to edit for these ref kinds.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// notifyRetentionPurgeMessageBox parses a message_box ref_key (format
|
||||||
|
// "user:<owner_user_id>:box:<box_id>", see postgres.messageBoxRefKey -- kept
|
||||||
|
// in sync by convention, not a shared symbol, since the ref_key encoding is
|
||||||
|
// an internal storage detail of that package) and edits the box in place.
|
||||||
|
func (s *Service) notifyRetentionPurgeMessageBox(ctx context.Context, messages RetentionPurgeMessageEditor, refKey string) {
|
||||||
|
if messages == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ownerUserID, boxID, ok := parseMessageBoxRefKey(refKey)
|
||||||
|
if !ok {
|
||||||
|
s.log.Warn("unparseable message_box retention purge ref_key", zap.String("ref_key", refKey))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// EditMessageRequest requires the peer explicitly (an ordinary edit's RPC
|
||||||
|
// caller always supplies it) -- resolve it from the box row itself since
|
||||||
|
// this is a server-internal edit with no client request behind it.
|
||||||
|
list, err := messages.GetMessages(ctx, ownerUserID, []int{boxID})
|
||||||
|
if err != nil {
|
||||||
|
s.log.Warn("resolve message_box peer for retention purge notice failed",
|
||||||
|
zap.Int64("owner_user_id", ownerUserID), zap.Int("box_id", boxID), zap.Error(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(list.Messages) == 0 {
|
||||||
|
// Box no longer visible to its owner (deleted) -- nothing to notice.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
peer := list.Messages[0].Peer
|
||||||
|
_, err = messages.EditMessage(ctx, ownerUserID, domain.EditMessageRequest{
|
||||||
|
OwnerUserID: ownerUserID,
|
||||||
|
Peer: peer,
|
||||||
|
ID: boxID,
|
||||||
|
SetRichMessage: true,
|
||||||
|
RichMessage: nil,
|
||||||
|
Media: retentionPurgeNoticeMedia(),
|
||||||
|
RetentionPurge: true,
|
||||||
|
})
|
||||||
|
if err != nil && !errors.Is(err, domain.ErrMessageNotModified) && !errors.Is(err, domain.ErrMessageIDInvalid) {
|
||||||
|
s.log.Warn("retention purge notice edit failed (message_box)",
|
||||||
|
zap.Int64("owner_user_id", ownerUserID), zap.Int("box_id", boxID), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// notifyRetentionPurgeChannelMessage parses a channel_message ref_key
|
||||||
|
// (format "channel:<channel_id>:msg:<message_id>", see
|
||||||
|
// postgres.channelMessageRefKey) and edits the message in place. Unlike
|
||||||
|
// private messages, no peer lookup is needed: EditChannelMessageRequest only
|
||||||
|
// needs the channel id, already encoded in the ref_key.
|
||||||
|
func (s *Service) notifyRetentionPurgeChannelMessage(ctx context.Context, channels RetentionPurgeChannelEditor, refKey string) {
|
||||||
|
if channels == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
channelID, messageID, ok := parseChannelMessageRefKey(refKey)
|
||||||
|
if !ok {
|
||||||
|
s.log.Warn("unparseable channel_message retention purge ref_key", zap.String("ref_key", refKey))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, err := channels.EditChannelMessageInternal(ctx, domain.EditChannelMessageRequest{
|
||||||
|
ChannelID: channelID,
|
||||||
|
ID: messageID,
|
||||||
|
RetentionPurge: true,
|
||||||
|
RetentionPurgeAction: &domain.ChannelMessageAction{Type: domain.ChannelActionCustomText, Text: domain.RetentionPurgeNoticeText},
|
||||||
|
})
|
||||||
|
if err != nil && !errors.Is(err, domain.ErrMessageNotModified) && !errors.Is(err, domain.ErrMessageIDInvalid) {
|
||||||
|
s.log.Warn("retention purge notice edit failed (channel_message)",
|
||||||
|
zap.Int64("channel_id", channelID), zap.Int("message_id", messageID), zap.Error(err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// retentionPurgeNoticeMedia builds the messageActionCustomAction service
|
||||||
|
// media payload private-message edits replace a purged message's media with.
|
||||||
|
func retentionPurgeNoticeMedia() *domain.MessageMedia {
|
||||||
|
return &domain.MessageMedia{
|
||||||
|
Kind: domain.MessageMediaKindService,
|
||||||
|
ServiceAction: &domain.MessageServiceAction{
|
||||||
|
Kind: domain.MessageServiceActionCustomText,
|
||||||
|
Text: domain.RetentionPurgeNoticeText,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseMessageBoxRefKey parses "user:<owner_user_id>:box:<box_id>".
|
||||||
|
// fmt.Sscanf reports success and silently ignores unconsumed trailing input
|
||||||
|
// (e.g. "user:1:box:2garbage" would still scan fine), so a bare cnt/err check
|
||||||
|
// isn't enough -- round-trip the parsed ids back through the same format
|
||||||
|
// (messageBoxRefKey's own layout) and require an exact match to reject any
|
||||||
|
// malformed/truncated/trailing-garbage key. Defensive only: ref_key is
|
||||||
|
// server-written and never externally supplied, but a bug or future
|
||||||
|
// encoding change should fail closed here, not silently misdirect an edit at
|
||||||
|
// the wrong box.
|
||||||
|
func parseMessageBoxRefKey(refKey string) (ownerUserID int64, boxID int, ok bool) {
|
||||||
|
cnt, err := fmt.Sscanf(refKey, "user:%d:box:%d", &ownerUserID, &boxID)
|
||||||
|
if err != nil || cnt != 2 || ownerUserID == 0 || boxID == 0 {
|
||||||
|
return 0, 0, false
|
||||||
|
}
|
||||||
|
if fmt.Sprintf("user:%d:box:%d", ownerUserID, boxID) != refKey {
|
||||||
|
return 0, 0, false
|
||||||
|
}
|
||||||
|
return ownerUserID, boxID, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseChannelMessageRefKey parses "channel:<channel_id>:msg:<message_id>".
|
||||||
|
// See parseMessageBoxRefKey's doc comment -- same round-trip defense against
|
||||||
|
// trailing-garbage input Sscanf alone would silently accept.
|
||||||
|
func parseChannelMessageRefKey(refKey string) (channelID int64, messageID int, ok bool) {
|
||||||
|
cnt, err := fmt.Sscanf(refKey, "channel:%d:msg:%d", &channelID, &messageID)
|
||||||
|
if err != nil || cnt != 2 || channelID == 0 || messageID == 0 {
|
||||||
|
return 0, 0, false
|
||||||
|
}
|
||||||
|
if fmt.Sprintf("channel:%d:msg:%d", channelID, messageID) != refKey {
|
||||||
|
return 0, 0, false
|
||||||
|
}
|
||||||
|
return channelID, messageID, true
|
||||||
|
}
|
||||||
29
internal/app/files/retention_purge_test.go
Normal file
29
internal/app/files/retention_purge_test.go
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
package files
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestParseMessageBoxRefKeyRoundTrip(t *testing.T) {
|
||||||
|
owner, box, ok := parseMessageBoxRefKey("user:42:box:7")
|
||||||
|
if !ok || owner != 42 || box != 7 {
|
||||||
|
t.Fatalf("parse = (%d, %d, %v), want (42, 7, true)", owner, box, ok)
|
||||||
|
}
|
||||||
|
malformed := []string{"", "garbage", "user:1:box:", "user:1:box:2extra", "user:0:box:1", "user:1:box:0", "channel:1:msg:2"}
|
||||||
|
for _, key := range malformed {
|
||||||
|
if _, _, ok := parseMessageBoxRefKey(key); ok {
|
||||||
|
t.Fatalf("parseMessageBoxRefKey(%q) ok=true, want false", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseChannelMessageRefKeyRoundTrip(t *testing.T) {
|
||||||
|
channel, msg, ok := parseChannelMessageRefKey("channel:99:msg:3")
|
||||||
|
if !ok || channel != 99 || msg != 3 {
|
||||||
|
t.Fatalf("parse = (%d, %d, %v), want (99, 3, true)", channel, msg, ok)
|
||||||
|
}
|
||||||
|
malformed := []string{"", "garbage", "channel:1:msg:", "channel:1:msg:2extra", "channel:0:msg:1", "channel:1:msg:0", "user:1:box:2"}
|
||||||
|
for _, key := range malformed {
|
||||||
|
if _, _, ok := parseChannelMessageRefKey(key); ok {
|
||||||
|
t.Fatalf("parseChannelMessageRefKey(%q) ok=true, want false", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -100,6 +100,27 @@ type Service struct {
|
||||||
// restart -- deleting only the DB row would make a "deleted" GIF come
|
// restart -- deleting only the DB row would make a "deleted" GIF come
|
||||||
// back on its own.
|
// back on its own.
|
||||||
gifSeedDir string
|
gifSeedDir string
|
||||||
|
|
||||||
|
// storageRetentionGlobalMaxAge/storageRetentionCategoryAges/
|
||||||
|
// storageRetentionAvatarMaxAge back categoryRetentionAge/avatarRetentionAge
|
||||||
|
// (retention.go) -- the per-category storage retention age overrides plus
|
||||||
|
// the shared global fallback age. Set via WithStorageRetentionAges.
|
||||||
|
storageRetentionGlobalMaxAge time.Duration
|
||||||
|
storageRetentionCategoryAges map[domain.MediaCategory]time.Duration
|
||||||
|
storageRetentionAvatarMaxAge time.Duration
|
||||||
|
// storageMaxTotalBytes is TELESRV_STORAGE_MAX_TOTAL_BYTES, reused by
|
||||||
|
// EvictOldestMediaOverBudget as the active-eviction trigger threshold
|
||||||
|
// (<=0 disables eviction regardless of TELESRV_STORAGE_EVICTION_ENABLE).
|
||||||
|
storageMaxTotalBytes int64
|
||||||
|
|
||||||
|
// retentionNotifyMu guards retentionMessages/retentionChannels: they are
|
||||||
|
// set post-construction (see SetRetentionPurgeNotifier) from
|
||||||
|
// cmd/telesrv/main.go once messageapp/channelapp services exist, which
|
||||||
|
// happens after this Service and the background retention sweep that
|
||||||
|
// reads them are already running.
|
||||||
|
retentionNotifyMu sync.RWMutex
|
||||||
|
retentionMessages RetentionPurgeMessageEditor
|
||||||
|
retentionChannels RetentionPurgeChannelEditor
|
||||||
}
|
}
|
||||||
|
|
||||||
// Option 配置 files 服务的可选能力。
|
// Option 配置 files 服务的可选能力。
|
||||||
|
|
@ -200,6 +221,27 @@ func WithGifCatalog(c store.GifCatalogStore) Option {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithStorageRetentionAges configures the per-category storage retention age
|
||||||
|
// overrides (Config.StorageRetentionMaxAgeByCategory/-MaxAgeAvatar) plus the
|
||||||
|
// shared global fallback age (Config.StorageRetentionMaxAge), consumed by
|
||||||
|
// categoryRetentionAge/avatarRetentionAge in retention.go.
|
||||||
|
func WithStorageRetentionAges(global time.Duration, byCategory map[domain.MediaCategory]time.Duration, avatarMaxAge time.Duration) Option {
|
||||||
|
return func(s *Service) {
|
||||||
|
s.storageRetentionGlobalMaxAge = global
|
||||||
|
s.storageRetentionCategoryAges = byCategory
|
||||||
|
s.storageRetentionAvatarMaxAge = avatarMaxAge
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithStorageMaxTotalBytes records TELESRV_STORAGE_MAX_TOTAL_BYTES for
|
||||||
|
// EvictOldestMediaOverBudget's active-eviction trigger threshold, reusing the
|
||||||
|
// same cap SpaceGuard already enforces against new uploads.
|
||||||
|
func WithStorageMaxTotalBytes(maxBytes int64) Option {
|
||||||
|
return func(s *Service) {
|
||||||
|
s.storageMaxTotalBytes = maxBytes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// WithGifSeedDir records the gif seed directory (cfg.GifSeedDir) so
|
// WithGifSeedDir records the gif seed directory (cfg.GifSeedDir) so
|
||||||
// AdminDeleteUncategorizedGifs can remove a seed-imported entry's source
|
// AdminDeleteUncategorizedGifs can remove a seed-imported entry's source
|
||||||
// file alongside its DB row -- see the field's doc comment for why that
|
// file alongside its DB row -- see the field's doc comment for why that
|
||||||
|
|
|
||||||
|
|
@ -124,6 +124,8 @@ type RetentionWorker struct {
|
||||||
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
|
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
|
||||||
orphanedMedia OrphanedMediaRetentionStore
|
orphanedMedia OrphanedMediaRetentionStore
|
||||||
hardMedia HardMediaRetentionStore
|
hardMedia HardMediaRetentionStore
|
||||||
|
eviction StorageEvictionStore
|
||||||
|
evictionEnabled bool
|
||||||
logger *zap.Logger
|
logger *zap.Logger
|
||||||
retention time.Duration
|
retention time.Duration
|
||||||
botAPIRetention time.Duration
|
botAPIRetention time.Duration
|
||||||
|
|
@ -311,9 +313,20 @@ func (w *RetentionWorker) mediaRetentionInterval() time.Duration {
|
||||||
if w.hardMedia != nil && w.hardMediaMaxAge > 0 && (maxAge == 0 || w.hardMediaMaxAge < maxAge) {
|
if w.hardMedia != nil && w.hardMediaMaxAge > 0 && (maxAge == 0 || w.hardMediaMaxAge < maxAge) {
|
||||||
maxAge = w.hardMediaMaxAge
|
maxAge = w.hardMediaMaxAge
|
||||||
}
|
}
|
||||||
|
evictionActive := w.eviction != nil && w.evictionEnabled
|
||||||
if maxAge <= 0 {
|
if maxAge <= 0 {
|
||||||
|
if !evictionActive {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
// Eviction is reactive to total bytes, not a fixed age, and is
|
||||||
|
// independent of TELESRV_STORAGE_RETENTION_MODE -- it can be the only
|
||||||
|
// thing enabled. Fall back to the shared housekeeping interval so it
|
||||||
|
// still gets its own ticker instead of never running.
|
||||||
|
maxAge = w.interval
|
||||||
|
if maxAge <= 0 {
|
||||||
|
maxAge = time.Hour
|
||||||
|
}
|
||||||
|
}
|
||||||
interval := w.interval
|
interval := w.interval
|
||||||
if interval <= 0 || maxAge < interval {
|
if interval <= 0 || maxAge < interval {
|
||||||
interval = maxAge
|
interval = maxAge
|
||||||
|
|
@ -324,12 +337,36 @@ func (w *RetentionWorker) mediaRetentionInterval() time.Duration {
|
||||||
return interval
|
return interval
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StorageEvictionStore actively reclaims space once total physical blob bytes
|
||||||
|
// exceed TELESRV_STORAGE_MAX_TOTAL_BYTES: the oldest documents/photos
|
||||||
|
// (interleaved by created_at across both tables, regardless of category or
|
||||||
|
// age) are purged -- reusing the exact same blob-purge + retention-purge
|
||||||
|
// notice primitive as HardMediaRetentionStore -- until back under budget.
|
||||||
|
// Independent of TELESRV_STORAGE_RETENTION_MODE.
|
||||||
|
type StorageEvictionStore interface {
|
||||||
|
EvictOldestMediaOverBudget(ctx context.Context, limit int) (int, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithStorageEviction enables the active eviction sweep
|
||||||
|
// (TELESRV_STORAGE_EVICTION_ENABLE). It shares the same ticker as the
|
||||||
|
// orphan/hard media sweeps (see mediaRetentionInterval) and is independent of
|
||||||
|
// TELESRV_STORAGE_RETENTION_MODE -- it can run even when that is "off".
|
||||||
|
func (w *RetentionWorker) WithStorageEviction(store StorageEvictionStore, enabled bool) *RetentionWorker {
|
||||||
|
w.eviction = store
|
||||||
|
w.evictionEnabled = enabled
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
func (w *RetentionWorker) runMediaRetentionOnce(ctx context.Context) {
|
func (w *RetentionWorker) runMediaRetentionOnce(ctx context.Context) {
|
||||||
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
|
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
|
||||||
// Only ever touches documents/photos already marked orphaned (no live
|
// Only ever touches documents/photos already marked orphaned (no live
|
||||||
// message/profile-photo/sticker-set reference remains) -- media still
|
// message/profile-photo/sticker-set reference remains) -- media still
|
||||||
// visible in a conversation is never a candidate, regardless of age.
|
// visible in a conversation is never a candidate, regardless of age.
|
||||||
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
|
// The store itself derives each category's effective cutoff from
|
||||||
|
// "now" (per-category retention age overrides, global age as
|
||||||
|
// fallback) -- this worker only owns when the sweep runs, not the
|
||||||
|
// per-category cutoff math.
|
||||||
|
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now(), w.batch)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
|
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
|
||||||
} else if mediaDeleted > 0 {
|
} else if mediaDeleted > 0 {
|
||||||
|
|
@ -337,19 +374,28 @@ func (w *RetentionWorker) runMediaRetentionOnce(ctx context.Context) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if w.hardMedia != nil && w.hardMediaMaxAge > 0 {
|
if w.hardMedia != nil && w.hardMediaMaxAge > 0 {
|
||||||
// "Hard" mode: purges blob bytes for documents/photos older than
|
// "Hard" mode: purges blob bytes for documents/photos older than the
|
||||||
// hardMediaMaxAge regardless of whether a live reference remains --
|
// effective per-category age (falling back to hardMediaMaxAge)
|
||||||
// the store is required to keep the document/photo metadata row
|
// regardless of whether a live reference remains -- the store is
|
||||||
// intact, only removing file_blobs rows/bytes, so a message still
|
// required to keep the document/photo metadata row intact, only
|
||||||
// renders its media placeholder (LOCATION_INVALID on download)
|
// removing file_blobs rows/bytes, so a message still renders its
|
||||||
// instead of breaking outright.
|
// media placeholder (LOCATION_INVALID on download) instead of
|
||||||
hardDeleted, err := w.hardMedia.DeleteBlobBytesForMediaOlderThan(ctx, time.Now().Add(-w.hardMediaMaxAge), w.batch)
|
// breaking outright.
|
||||||
|
hardDeleted, err := w.hardMedia.DeleteBlobBytesForMediaOlderThan(ctx, time.Now(), w.batch)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
w.logger.Warn("hard media storage retention sweep failed", zap.Error(err))
|
w.logger.Warn("hard media storage retention sweep failed", zap.Error(err))
|
||||||
} else if hardDeleted > 0 {
|
} else if hardDeleted > 0 {
|
||||||
w.logger.Info("hard media storage retention sweep complete", zap.Int("deleted", hardDeleted))
|
w.logger.Info("hard media storage retention sweep complete", zap.Int("deleted", hardDeleted))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if w.eviction != nil && w.evictionEnabled {
|
||||||
|
evicted, err := w.eviction.EvictOldestMediaOverBudget(ctx, w.batch)
|
||||||
|
if err != nil {
|
||||||
|
w.logger.Warn("active storage eviction sweep failed", zap.Error(err))
|
||||||
|
} else if evicted > 0 {
|
||||||
|
w.logger.Info("active storage eviction sweep complete", zap.Int("evicted", evicted))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *RetentionWorker) runOutboxPoisonOnce(ctx context.Context) {
|
func (w *RetentionWorker) runOutboxPoisonOnce(ctx context.Context) {
|
||||||
|
|
|
||||||
|
|
@ -357,6 +357,32 @@ type Config struct {
|
||||||
// RetentionInterval/RetentionBatch cadence alongside every other
|
// RetentionInterval/RetentionBatch cadence alongside every other
|
||||||
// retention check (see maintenance.RetentionWorker).
|
// retention check (see maintenance.RetentionWorker).
|
||||||
StorageRetentionMaxAge time.Duration
|
StorageRetentionMaxAge time.Duration
|
||||||
|
// StorageRetentionMaxAgeByCategory overrides StorageRetentionMaxAge for
|
||||||
|
// one document media category (Photo/Video/RoundVideo/Gif/Music/Voice/
|
||||||
|
// File); a category absent from the map inherits the shared global age.
|
||||||
|
// Populated from TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY> env vars,
|
||||||
|
// one per category -- unset or non-positive means "inherit global" for
|
||||||
|
// that category (a document that hasn't been classified, e.g. a sticker,
|
||||||
|
// always uses the global age -- there is no override for that bucket).
|
||||||
|
// Avatar is not a domain.MediaCategory (a photo becomes/stops being an
|
||||||
|
// avatar via profile_photos, not a document attribute), so it gets its
|
||||||
|
// own StorageRetentionMaxAgeAvatar field instead of a map entry.
|
||||||
|
StorageRetentionMaxAgeByCategory map[domain.MediaCategory]time.Duration
|
||||||
|
// StorageRetentionMaxAgeAvatar overrides StorageRetentionMaxAge for
|
||||||
|
// photos currently active as someone's avatar (profile_photos.active).
|
||||||
|
// <=0 means "inherit global", same convention as
|
||||||
|
// StorageRetentionMaxAgeByCategory.
|
||||||
|
StorageRetentionMaxAgeAvatar time.Duration
|
||||||
|
// StorageEvictionEnable turns on active eviction: once the total physical
|
||||||
|
// blob bytes exceed StorageMaxTotalBytes, the oldest documents/photos
|
||||||
|
// (interleaved by created_at across both tables, regardless of category
|
||||||
|
// or age) are purged the same way "hard" retention mode purges blob
|
||||||
|
// bytes -- repeated until back under budget. Independent of
|
||||||
|
// StorageRetentionMode: eviction can run even when retention mode is
|
||||||
|
// "off". Default false (opt-in), since this changes what
|
||||||
|
// StorageMaxTotalBytes has meant so far (block new uploads only ->
|
||||||
|
// block-and-reclaim from existing ones too).
|
||||||
|
StorageEvictionEnable bool
|
||||||
// StickerSeedDir 是 reaction / sticker 资源种子目录（导入到 documents/sticker_sets + blob）。
|
// StickerSeedDir 是 reaction / sticker 资源种子目录（导入到 documents/sticker_sets + blob）。
|
||||||
StickerSeedDir string
|
StickerSeedDir string
|
||||||
// StickerSeedMaxSets 限制导入的常规贴纸集数量（避免启动时导入过多包），<=0 表示不限。
|
// StickerSeedMaxSets 限制导入的常规贴纸集数量（避免启动时导入过多包），<=0 表示不限。
|
||||||
|
|
@ -980,6 +1006,9 @@ func Load() (Config, error) {
|
||||||
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
|
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
|
||||||
StorageRetentionMode: strings.ToLower(strings.TrimSpace(envOr("TELESRV_STORAGE_RETENTION_MODE", StorageRetentionModeOff))),
|
StorageRetentionMode: strings.ToLower(strings.TrimSpace(envOr("TELESRV_STORAGE_RETENTION_MODE", StorageRetentionModeOff))),
|
||||||
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
|
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
|
||||||
|
StorageRetentionMaxAgeByCategory: storageRetentionMaxAgeByCategoryFromEnv(envDurationOr),
|
||||||
|
StorageRetentionMaxAgeAvatar: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR", 0),
|
||||||
|
StorageEvictionEnable: envBoolOr("TELESRV_STORAGE_EVICTION_ENABLE", false),
|
||||||
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
|
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
|
||||||
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
|
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
|
||||||
PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"),
|
PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"),
|
||||||
|
|
@ -1518,13 +1547,21 @@ func validateStorageConfig(cfg Config) error {
|
||||||
if cfg.StorageRetentionMaxAge < 0 {
|
if cfg.StorageRetentionMaxAge < 0 {
|
||||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
|
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
|
||||||
}
|
}
|
||||||
|
if cfg.StorageRetentionMaxAgeAvatar < 0 {
|
||||||
|
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR must be non-negative")
|
||||||
|
}
|
||||||
switch cfg.StorageRetentionMode {
|
switch cfg.StorageRetentionMode {
|
||||||
case StorageRetentionModeOff:
|
case StorageRetentionModeOff:
|
||||||
// no sweep; StorageRetentionMaxAge is ignored.
|
// no sweep; StorageRetentionMaxAge is ignored.
|
||||||
case StorageRetentionModeOrphan, StorageRetentionModeHard:
|
case StorageRetentionModeOrphan, StorageRetentionModeHard:
|
||||||
if cfg.StorageRetentionMaxAge <= 0 {
|
// StorageRetentionMaxAge (the shared default) is no longer required to
|
||||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_MODE is %q", cfg.StorageRetentionMode)
|
// be positive here: it can legitimately be 0 -- "no sweep by default"
|
||||||
}
|
// -- while individual TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY>/
|
||||||
|
// _AVATAR overrides opt specific categories in. The sweep itself
|
||||||
|
// (internal/app/files/retention.go) skips any category whose
|
||||||
|
// *effective* age (its own override, or this shared default) is <=0,
|
||||||
|
// so a 0 default with no overrides simply means the mode runs and
|
||||||
|
// purges nothing -- equivalent to "off" in practice, not dangerous.
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MODE must be \"off\", \"orphan\", or \"hard\", got %q", cfg.StorageRetentionMode)
|
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MODE must be \"off\", \"orphan\", or \"hard\", got %q", cfg.StorageRetentionMode)
|
||||||
}
|
}
|
||||||
|
|
@ -2075,6 +2112,33 @@ func (e envSource) envFloatOr(key string, def float64) float64 {
|
||||||
return def
|
return def
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// storageRetentionMaxAgeByCategoryFromEnv builds Config.StorageRetentionMaxAgeByCategory:
|
||||||
|
// one TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY> env var per document
|
||||||
|
// media category, using envDurationOr(key, 0) so "unset" and "explicitly 0"
|
||||||
|
// both cleanly mean "inherit the shared global age" -- only a positive value
|
||||||
|
// gets an entry in the map.
|
||||||
|
func storageRetentionMaxAgeByCategoryFromEnv(envDurationOr func(key string, def time.Duration) time.Duration) map[domain.MediaCategory]time.Duration {
|
||||||
|
entries := []struct {
|
||||||
|
category domain.MediaCategory
|
||||||
|
envKey string
|
||||||
|
}{
|
||||||
|
{domain.MediaCategoryPhoto, "TELESRV_STORAGE_RETENTION_MAX_AGE_PHOTO"},
|
||||||
|
{domain.MediaCategoryVideo, "TELESRV_STORAGE_RETENTION_MAX_AGE_VIDEO"},
|
||||||
|
{domain.MediaCategoryRoundVideo, "TELESRV_STORAGE_RETENTION_MAX_AGE_ROUND_VIDEO"},
|
||||||
|
{domain.MediaCategoryGif, "TELESRV_STORAGE_RETENTION_MAX_AGE_GIF"},
|
||||||
|
{domain.MediaCategoryMusic, "TELESRV_STORAGE_RETENTION_MAX_AGE_MUSIC"},
|
||||||
|
{domain.MediaCategoryVoice, "TELESRV_STORAGE_RETENTION_MAX_AGE_VOICE"},
|
||||||
|
{domain.MediaCategoryFile, "TELESRV_STORAGE_RETENTION_MAX_AGE_FILE"},
|
||||||
|
}
|
||||||
|
out := make(map[domain.MediaCategory]time.Duration, len(entries))
|
||||||
|
for _, e := range entries {
|
||||||
|
if age := envDurationOr(e.envKey, 0); age > 0 {
|
||||||
|
out[e.category] = age
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// envDurationOr 读取 time.ParseDuration 格式（如 "200ms"、"30s"）的时长配置；解析失败回退默认值。
|
// envDurationOr 读取 time.ParseDuration 格式（如 "200ms"、"30s"）的时长配置；解析失败回退默认值。
|
||||||
func (e envSource) envDurationOr(key string, def time.Duration) time.Duration {
|
func (e envSource) envDurationOr(key string, def time.Duration) time.Duration {
|
||||||
if v := e.envOr(key, ""); v != "" {
|
if v := e.envOr(key, ""); v != "" {
|
||||||
|
|
|
||||||
|
|
@ -550,14 +550,31 @@ func TestLoadRejectsInvalidStorageRetentionMode(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoadRejectsStorageRetentionModeWithoutMaxAge(t *testing.T) {
|
// TestLoadAcceptsStorageRetentionModeWithZeroMaxAgeAndCategoryOverride guards
|
||||||
|
// a real reported bug: TELESRV_STORAGE_RETENTION_MAX_AGE=0 (the shared
|
||||||
|
// default) used to be rejected outright whenever the mode was orphan/hard,
|
||||||
|
// which made it impossible to run the sweep for only specific
|
||||||
|
// TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY> overrides while leaving
|
||||||
|
// everything else alone. A zero shared default is now legitimate: the sweep
|
||||||
|
// itself (internal/app/files/retention.go) skips any category whose
|
||||||
|
// *effective* age is <=0, so 0 here just means "no default sweep" rather
|
||||||
|
// than "purge everything immediately".
|
||||||
|
func TestLoadAcceptsStorageRetentionModeWithZeroMaxAgeAndCategoryOverride(t *testing.T) {
|
||||||
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
|
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
|
||||||
t.Run(mode, func(t *testing.T) {
|
t.Run(mode, func(t *testing.T) {
|
||||||
disableDefaultConfigFile(t)
|
disableDefaultConfigFile(t)
|
||||||
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
|
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
|
||||||
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "0s")
|
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "0s")
|
||||||
if _, err := Load(); err == nil {
|
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE_VIDEO", "24h")
|
||||||
t.Fatalf("Load accepted TELESRV_STORAGE_RETENTION_MODE=%s with zero max age", mode)
|
cfg, err := Load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load rejected TELESRV_STORAGE_RETENTION_MODE=%s with zero shared max age + a category override: %v", mode, err)
|
||||||
|
}
|
||||||
|
if cfg.StorageRetentionMaxAge != 0 {
|
||||||
|
t.Fatalf("StorageRetentionMaxAge = %v, want 0", cfg.StorageRetentionMaxAge)
|
||||||
|
}
|
||||||
|
if got := cfg.StorageRetentionMaxAgeByCategory[domain.MediaCategoryVideo]; got != 24*time.Hour {
|
||||||
|
t.Fatalf("StorageRetentionMaxAgeByCategory[Video] = %v, want 24h", got)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -634,6 +634,10 @@ const (
|
||||||
ChannelActionSuggestedPostApproval ChannelMessageActionType = "suggested_post_approval"
|
ChannelActionSuggestedPostApproval ChannelMessageActionType = "suggested_post_approval"
|
||||||
ChannelActionSuggestedPostSuccess ChannelMessageActionType = "suggested_post_success"
|
ChannelActionSuggestedPostSuccess ChannelMessageActionType = "suggested_post_success"
|
||||||
ChannelActionSuggestedPostRefund ChannelMessageActionType = "suggested_post_refund"
|
ChannelActionSuggestedPostRefund ChannelMessageActionType = "suggested_post_refund"
|
||||||
|
// ChannelActionCustomText 映射 messageActionCustomAction（stock TL 类型）：
|
||||||
|
// 当前唯一用途是把 storage retention 硬回收/主动淘汰清理掉的频道消息就地
|
||||||
|
// 转成这条通知（见 EditChannelMessageRequest.RetentionPurge）。
|
||||||
|
ChannelActionCustomText ChannelMessageActionType = "custom_text"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ChannelMessageAction describes a service action without depending on tg.*.
|
// ChannelMessageAction describes a service action without depending on tg.*.
|
||||||
|
|
@ -678,6 +682,8 @@ type ChannelMessageAction struct {
|
||||||
SuggestedPostScheduleDate int
|
SuggestedPostScheduleDate int
|
||||||
SuggestedPostPrice *SuggestedPostPrice
|
SuggestedPostPrice *SuggestedPostPrice
|
||||||
SuggestedPostPayerInitiated bool
|
SuggestedPostPayerInitiated bool
|
||||||
|
// Text 仅 custom_text 服务消息使用（messageActionCustomAction.message）。
|
||||||
|
Text string
|
||||||
}
|
}
|
||||||
|
|
||||||
// ChannelMessage is a single stored message in a channel/supergroup.
|
// ChannelMessage is a single stored message in a channel/supergroup.
|
||||||
|
|
@ -1967,6 +1973,15 @@ type EditChannelMessageRequest struct {
|
||||||
// 仅当目标当前 media 仍是 ID==ExpectedWebPageID 的 pending 链接预览才替换。
|
// 仅当目标当前 media 仍是 ID==ExpectedWebPageID 的 pending 链接预览才替换。
|
||||||
WebPageResolve bool
|
WebPageResolve bool
|
||||||
ExpectedWebPageID int64
|
ExpectedWebPageID int64
|
||||||
|
// RetentionPurge 置位时为存储 retention 硬回收/主动淘汰的服务端内部编辑：
|
||||||
|
// 绕过普通的发送者/管理员编辑权限校验，仅由 internal/app/files 的
|
||||||
|
// retention purge 通知路径设置。见 EditMessageRequest.RetentionPurge。
|
||||||
|
RetentionPurge bool
|
||||||
|
// RetentionPurgeAction 与 RetentionPurge 搭配使用：频道消息的服务动作
|
||||||
|
// 存于独立的 Action 字段（不像私聊消息把服务动作折进 Media.Kind=service），
|
||||||
|
// 所以 retention 通知在频道侧必须整条消息转成一条新的服务消息（清空
|
||||||
|
// body/media，只设置这个 Action），而不是像私聊那样替换 Media。
|
||||||
|
RetentionPurgeAction *ChannelMessageAction
|
||||||
}
|
}
|
||||||
|
|
||||||
// EditChannelMessageResult describes one channel edit update.
|
// EditChannelMessageResult describes one channel edit update.
|
||||||
|
|
|
||||||
|
|
@ -591,6 +591,11 @@ const (
|
||||||
// 状态切换与关闭请求。会话级保护不能写入普通消息的 NoForwards 字段。
|
// 状态切换与关闭请求。会话级保护不能写入普通消息的 NoForwards 字段。
|
||||||
MessageServiceActionNoForwardsToggle MessageServiceActionKind = "no_forwards_toggle"
|
MessageServiceActionNoForwardsToggle MessageServiceActionKind = "no_forwards_toggle"
|
||||||
MessageServiceActionNoForwardsRequest MessageServiceActionKind = "no_forwards_request"
|
MessageServiceActionNoForwardsRequest MessageServiceActionKind = "no_forwards_request"
|
||||||
|
// MessageServiceActionCustomText 映射 messageActionCustomAction（stock TL
|
||||||
|
// 类型，两端客户端已原生支持，无需任何 client 补丁）：服务端把纯文本状态
|
||||||
|
// 通知（当前唯一用途：storage retention 硬回收把已清理媒体的消息就地转成
|
||||||
|
// 这条通知）渲染成居中灰色系统气泡，而不是一次看起来像普通编辑的文本替换。
|
||||||
|
MessageServiceActionCustomText MessageServiceActionKind = "custom_text"
|
||||||
)
|
)
|
||||||
|
|
||||||
// MessagePhoneCallAction 是 messageActionPhoneCall 的协议中立载荷。
|
// MessagePhoneCallAction 是 messageActionPhoneCall 的协议中立载荷。
|
||||||
|
|
@ -673,6 +678,8 @@ type MessageServiceAction struct {
|
||||||
RequestedPeer *MessageRequestedPeerAction `json:"requested_peer,omitempty"`
|
RequestedPeer *MessageRequestedPeerAction `json:"requested_peer,omitempty"`
|
||||||
ChatThemeEmoticon string `json:"chat_theme_emoticon,omitempty"`
|
ChatThemeEmoticon string `json:"chat_theme_emoticon,omitempty"`
|
||||||
NoForwards *MessageNoForwardsAction `json:"no_forwards,omitempty"`
|
NoForwards *MessageNoForwardsAction `json:"no_forwards,omitempty"`
|
||||||
|
// Text 承载 MessageServiceActionCustomText 的固定通知文案（messageActionCustomAction.message）。
|
||||||
|
Text string `json:"text,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// MessageMedia 是一条消息媒体载荷的业务表示（落库为消息行上的 JSONB 快照）。
|
// MessageMedia 是一条消息媒体载荷的业务表示（落库为消息行上的 JSONB 快照）。
|
||||||
|
|
|
||||||
|
|
@ -159,6 +159,26 @@ func classifyDocumentCategory(doc *Document) (MediaCategory, bool) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DocumentMediaCategory returns the single document-attribute category
|
||||||
|
// (Video/Gif/Music/Voice/RoundVideo/File) for a document media payload, used
|
||||||
|
// to populate the documents.category column alongside the shared media
|
||||||
|
// index. Unlike ClassifyMediaCategories (which can also add unrelated
|
||||||
|
// categories such as URL from message entities), this only ever answers "what
|
||||||
|
// kind of document is this", returning MediaCategoryNone for a non-document
|
||||||
|
// payload or for sticker/custom-emoji documents (they classify to nothing --
|
||||||
|
// see classifyDocumentCategory). Zero new classification logic: reuses
|
||||||
|
// classifyDocumentCategory exactly.
|
||||||
|
func DocumentMediaCategory(media *MessageMedia) MediaCategory {
|
||||||
|
if media == nil || media.Kind != MessageMediaKindDocument {
|
||||||
|
return MediaCategoryNone
|
||||||
|
}
|
||||||
|
c, ok := classifyDocumentCategory(media.Document)
|
||||||
|
if !ok {
|
||||||
|
return MediaCategoryNone
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
func hasURLEntity(entities []MessageEntity) bool {
|
func hasURLEntity(entities []MessageEntity) bool {
|
||||||
for _, e := range entities {
|
for _, e := range entities {
|
||||||
if e.Type == MessageEntityURL || e.Type == MessageEntityTextURL || e.Type == MessageEntityEmail {
|
if e.Type == MessageEntityURL || e.Type == MessageEntityTextURL || e.Type == MessageEntityEmail {
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
package domain
|
package domain
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
// MediaKind identifies which table a media_references row points into.
|
// MediaKind identifies which table a media_references row points into.
|
||||||
type MediaKind string
|
type MediaKind string
|
||||||
|
|
||||||
|
|
@ -45,6 +47,16 @@ type OrphanCandidate struct {
|
||||||
OrphanedAt int64 // unix seconds
|
OrphanedAt int64 // unix seconds
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EvictionCandidate is one document/photo still owning file_blobs bytes,
|
||||||
|
// used by the active-eviction sweep (TELESRV_STORAGE_EVICTION_ENABLE) to pick
|
||||||
|
// the oldest media overall across both tables once total physical storage
|
||||||
|
// exceeds TELESRV_STORAGE_MAX_TOTAL_BYTES.
|
||||||
|
type EvictionCandidate struct {
|
||||||
|
Kind MediaKind
|
||||||
|
MediaID int64
|
||||||
|
CreatedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
// MediaRefTarget identifies one document/photo embedded in a message's media
|
// MediaRefTarget identifies one document/photo embedded in a message's media
|
||||||
// snapshot.
|
// snapshot.
|
||||||
type MediaRefTarget struct {
|
type MediaRefTarget struct {
|
||||||
|
|
|
||||||
|
|
@ -555,6 +555,12 @@ type OutboxReadDateRequest struct {
|
||||||
ID int
|
ID int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RetentionPurgeNoticeText is the fixed notice text used for the
|
||||||
|
// messageActionCustomAction (MessageServiceActionCustomText) service message
|
||||||
|
// a storage-retention blob purge (hard mode or active eviction) turns a
|
||||||
|
// message into once its underlying document/photo bytes are gone.
|
||||||
|
const RetentionPurgeNoticeText = "This file was deleted by the server's storage retention policy."
|
||||||
|
|
||||||
// EditMessageRequest 是账号视角下编辑一条已发送私聊消息的命令。
|
// EditMessageRequest 是账号视角下编辑一条已发送私聊消息的命令。
|
||||||
// Media 非 nil 时整体替换消息媒体快照（当前唯一调用方是 live location 续报/停止，
|
// Media 非 nil 时整体替换消息媒体快照（当前唯一调用方是 live location 续报/停止，
|
||||||
// rpc 层负责限定媒体种类）；nil 表示纯文本编辑。
|
// rpc 层负责限定媒体种类）；nil 表示纯文本编辑。
|
||||||
|
|
@ -589,6 +595,11 @@ type EditMessageRequest struct {
|
||||||
// 否则返回 ErrMessageNotModified（消息已删/已改/已解析）。
|
// 否则返回 ErrMessageNotModified（消息已删/已改/已解析）。
|
||||||
WebPageResolve bool
|
WebPageResolve bool
|
||||||
ExpectedWebPageID int64
|
ExpectedWebPageID int64
|
||||||
|
// RetentionPurge 置位时为存储 retention 硬回收/主动淘汰的服务端内部编辑：
|
||||||
|
// 绕过 authorEdit/viaBotEdit/WebPageResolve 之外的普通作者校验（回收发生
|
||||||
|
// 在任意账号的历史消息上，不是消息真正作者的操作），仅由
|
||||||
|
// internal/app/files 的 retention purge 通知路径设置。
|
||||||
|
RetentionPurge bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// EditedMessageForUser 描述一次编辑对某个 owner 视角造成的影响。
|
// EditedMessageForUser 描述一次编辑对某个 owner 视角造成的影响。
|
||||||
|
|
|
||||||
|
|
@ -224,6 +224,8 @@ func tgChannelMessageAction(action domain.ChannelMessageAction) tg.MessageAction
|
||||||
return &tg.MessageActionChannelCreate{Title: action.Title}
|
return &tg.MessageActionChannelCreate{Title: action.Title}
|
||||||
case domain.ChannelActionHistoryClear:
|
case domain.ChannelActionHistoryClear:
|
||||||
return &tg.MessageActionHistoryClear{}
|
return &tg.MessageActionHistoryClear{}
|
||||||
|
case domain.ChannelActionCustomText:
|
||||||
|
return &tg.MessageActionCustomAction{Message: action.Text}
|
||||||
case domain.ChannelActionChatAddUser, domain.ChannelActionChatJoined:
|
case domain.ChannelActionChatAddUser, domain.ChannelActionChatJoined:
|
||||||
return &tg.MessageActionChatAddUser{Users: append([]int64(nil), action.UserIDs...)}
|
return &tg.MessageActionChatAddUser{Users: append([]int64(nil), action.UserIDs...)}
|
||||||
case domain.ChannelActionChatJoinedByLink:
|
case domain.ChannelActionChatJoinedByLink:
|
||||||
|
|
|
||||||
|
|
@ -139,6 +139,8 @@ func tgMessageServiceAction(msg domain.Message) tg.MessageActionClass {
|
||||||
switch m.ServiceAction.Kind {
|
switch m.ServiceAction.Kind {
|
||||||
case domain.MessageServiceActionHistoryClear:
|
case domain.MessageServiceActionHistoryClear:
|
||||||
return &tg.MessageActionHistoryClear{}
|
return &tg.MessageActionHistoryClear{}
|
||||||
|
case domain.MessageServiceActionCustomText:
|
||||||
|
return &tg.MessageActionCustomAction{Message: m.ServiceAction.Text}
|
||||||
case domain.MessageServiceActionSuggestProfilePhoto:
|
case domain.MessageServiceActionSuggestProfilePhoto:
|
||||||
if m.ServiceAction.Photo == nil || m.ServiceAction.Photo.ID == 0 {
|
if m.ServiceAction.Photo == nil || m.ServiceAction.Photo.ID == 0 {
|
||||||
return &tg.MessageActionEmpty{}
|
return &tg.MessageActionEmpty{}
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,12 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func (s *ChannelStore) EditChannelMessage(ctx context.Context, req domain.EditChannelMessageRequest) (domain.EditChannelMessageResult, error) {
|
func (s *ChannelStore) EditChannelMessage(ctx context.Context, req domain.EditChannelMessageRequest) (domain.EditChannelMessageResult, error) {
|
||||||
if req.UserID == 0 || req.ChannelID == 0 || req.ID <= 0 {
|
retentionPurge := req.RetentionPurge && req.RetentionPurgeAction != nil
|
||||||
|
// RetentionPurge is a server-internal edit with no acting user (the
|
||||||
|
// storage retention sweep, not an RPC caller) -- UserID==0 is normally
|
||||||
|
// invalid, and req.UserID would otherwise also need to name an existing
|
||||||
|
// channel member for getChannelForMember below.
|
||||||
|
if (req.UserID == 0 && !retentionPurge) || req.ChannelID == 0 || req.ID <= 0 {
|
||||||
return domain.EditChannelMessageResult{}, domain.ErrChannelInvalid
|
return domain.EditChannelMessageResult{}, domain.ErrChannelInvalid
|
||||||
}
|
}
|
||||||
beginner, ok := s.db.(txBeginner)
|
beginner, ok := s.db.(txBeginner)
|
||||||
|
|
@ -44,7 +49,17 @@ func (s *ChannelStore) EditChannelMessage(ctx context.Context, req domain.EditCh
|
||||||
_ = tx.Rollback(ctx)
|
_ = tx.Rollback(ctx)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
channel, member, err := s.getChannelForMember(ctx, tx, req.UserID, req.ChannelID)
|
var (
|
||||||
|
channel domain.Channel
|
||||||
|
member domain.ChannelMember
|
||||||
|
)
|
||||||
|
if retentionPurge {
|
||||||
|
// No membership requirement: this is a server-internal edit, not an
|
||||||
|
// action taken by any particular channel member.
|
||||||
|
channel, err = s.channelByID(ctx, tx, req.ChannelID)
|
||||||
|
} else {
|
||||||
|
channel, member, err = s.getChannelForMember(ctx, tx, req.UserID, req.ChannelID)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.EditChannelMessageResult{}, err
|
return domain.EditChannelMessageResult{}, err
|
||||||
}
|
}
|
||||||
|
|
@ -52,6 +67,17 @@ func (s *ChannelStore) EditChannelMessage(ctx context.Context, req domain.EditCh
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.EditChannelMessageResult{}, err
|
return domain.EditChannelMessageResult{}, err
|
||||||
}
|
}
|
||||||
|
if retentionPurge {
|
||||||
|
if msg.Deleted {
|
||||||
|
return domain.EditChannelMessageResult{}, domain.ErrMessageIDInvalid
|
||||||
|
}
|
||||||
|
if msg.Action != nil {
|
||||||
|
// 幂等守卫：已经是服务消息(含已被本路径转换过的 retention 通知)
|
||||||
|
// 不再重复编辑 -- 见 EditMessageRequest.RetentionPurge 同款守卫。
|
||||||
|
return domain.EditChannelMessageResult{}, domain.ErrMessageNotModified
|
||||||
|
}
|
||||||
|
return s.applyRetentionPurgeChannelMessage(ctx, tx, channel, msg, req)
|
||||||
|
}
|
||||||
if msg.Deleted || msg.Action != nil {
|
if msg.Deleted || msg.Action != nil {
|
||||||
return domain.EditChannelMessageResult{}, domain.ErrMessageIDInvalid
|
return domain.EditChannelMessageResult{}, domain.ErrMessageIDInvalid
|
||||||
}
|
}
|
||||||
|
|
@ -283,6 +309,74 @@ WHERE channel_id = $1 AND id = $2`, req.ChannelID, req.ID, mediaJSON); err != ni
|
||||||
return domain.EditChannelMessageResult{Channel: channel, Message: msg, Event: event, ServiceMessage: serviceMsg, ServiceEvent: serviceEvent, Recipients: recipients}, nil
|
return domain.EditChannelMessageResult{Channel: channel, Message: msg, Event: event, ServiceMessage: serviceMsg, ServiceEvent: serviceEvent, Recipients: recipients}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// applyRetentionPurgeChannelMessage turns msg into a real service message
|
||||||
|
// (messageActionCustomAction via req.RetentionPurgeAction) in place. Unlike a
|
||||||
|
// normal channel edit -- which only ever replaces body/entities/media --
|
||||||
|
// channel service actions live in a structurally separate Action column
|
||||||
|
// (see tgChannelMessage: m.Action != nil renders tg.MessageService instead of
|
||||||
|
// tg.Message), so this clears body/media and sets Action instead of touching
|
||||||
|
// them. Reuses the same pts/durable-event/admin-log machinery as a normal
|
||||||
|
// edit so getChannelDifference and the fanout dispatcher see it identically.
|
||||||
|
func (s *ChannelStore) applyRetentionPurgeChannelMessage(ctx context.Context, tx pgx.Tx, channel domain.Channel, msg domain.ChannelMessage, req domain.EditChannelMessageRequest) (domain.EditChannelMessageResult, error) {
|
||||||
|
actionJSON, err := marshalJSON(req.RetentionPurgeAction, "{}")
|
||||||
|
if err != nil {
|
||||||
|
return domain.EditChannelMessageResult{}, fmt.Errorf("encode retention purge channel action: %w", err)
|
||||||
|
}
|
||||||
|
pts, err := s.reserveChannelPts(ctx, tx, req.ChannelID)
|
||||||
|
if err != nil {
|
||||||
|
return domain.EditChannelMessageResult{}, fmt.Errorf("allocate retention purge channel pts: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `
|
||||||
|
UPDATE channel_messages
|
||||||
|
SET body = '', entities = '[]'::jsonb, media = '{}'::jsonb, action = $3, edit_date = $4, pts = $5, updated_at = now()
|
||||||
|
WHERE channel_id = $1 AND id = $2`, req.ChannelID, req.ID, actionJSON, req.EditDate, pts); err != nil {
|
||||||
|
return domain.EditChannelMessageResult{}, fmt.Errorf("update retention purge channel message: %w", err)
|
||||||
|
}
|
||||||
|
prevMsg := msg
|
||||||
|
msg.Body = ""
|
||||||
|
msg.Entities = nil
|
||||||
|
msg.Media = nil
|
||||||
|
msg.Action = req.RetentionPurgeAction
|
||||||
|
msg.EditDate = req.EditDate
|
||||||
|
msg.Pts = pts
|
||||||
|
// 媒体索引/media_references 靠这次替换后的（空）媒体重建：原文档/照片
|
||||||
|
// 不再被这条消息引用，storage retention 的孤儿判定据此推进。
|
||||||
|
if err := replaceChannelMediaIndexTx(ctx, tx, req.ChannelID, req.ID, msg.Date, msg.Media, nil); err != nil {
|
||||||
|
return domain.EditChannelMessageResult{}, err
|
||||||
|
}
|
||||||
|
event := domain.ChannelUpdateEvent{
|
||||||
|
ChannelID: req.ChannelID,
|
||||||
|
Type: domain.ChannelUpdateEditMessage,
|
||||||
|
Pts: pts,
|
||||||
|
PtsCount: 1,
|
||||||
|
Date: req.EditDate,
|
||||||
|
Message: msg,
|
||||||
|
SenderUserID: req.UserID,
|
||||||
|
}
|
||||||
|
if err := insertChannelEventTx(ctx, tx, event); err != nil {
|
||||||
|
return domain.EditChannelMessageResult{}, err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `UPDATE channels SET pts = $2, updated_at = now() WHERE id = $1`, req.ChannelID, pts); err != nil {
|
||||||
|
return domain.EditChannelMessageResult{}, fmt.Errorf("update retention purge channel pts: %w", err)
|
||||||
|
}
|
||||||
|
channel.Pts = pts
|
||||||
|
if err := s.insertChannelAdminLogTx(ctx, tx, domain.ChannelAdminLogEvent{
|
||||||
|
ChannelID: req.ChannelID,
|
||||||
|
UserID: req.UserID,
|
||||||
|
Date: req.EditDate,
|
||||||
|
Type: domain.ChannelAdminLogEditMessage,
|
||||||
|
PrevMessage: &prevMsg,
|
||||||
|
NewMessage: &msg,
|
||||||
|
}); err != nil {
|
||||||
|
return domain.EditChannelMessageResult{}, err
|
||||||
|
}
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return domain.EditChannelMessageResult{}, fmt.Errorf("commit retention purge channel message: %w", err)
|
||||||
|
}
|
||||||
|
recipients, _ := s.ListActiveChannelMemberIDs(ctx, req.UserID, req.ChannelID, 0)
|
||||||
|
return domain.EditChannelMessageResult{Channel: channel, Message: msg, Event: event, Recipients: recipients}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func isChannelTodoParticipantEdit(req domain.EditChannelMessageRequest, msg domain.ChannelMessage) bool {
|
func isChannelTodoParticipantEdit(req domain.EditChannelMessageRequest, msg domain.ChannelMessage) bool {
|
||||||
if !req.AllowTodoParticipantMutation || req.SetReplyMarkup || req.Media == nil || req.Media.Kind != domain.MessageMediaKindTodo || req.Media.Todo == nil {
|
if !req.AllowTodoParticipantMutation || req.SetReplyMarkup || req.Media == nil || req.Media.Kind != domain.MessageMediaKindTodo || req.Media.Todo == nil {
|
||||||
return false
|
return false
|
||||||
|
|
|
||||||
|
|
@ -64,7 +64,7 @@ func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
cutoff := time.Now().Add(-24 * time.Hour)
|
cutoff := time.Now().Add(-24 * time.Hour)
|
||||||
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
|
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, cutoff, 1000)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
|
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -95,7 +95,7 @@ func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
|
||||||
|
|
||||||
// Idempotent / self-terminating: a second sweep pass no longer selects
|
// Idempotent / self-terminating: a second sweep pass no longer selects
|
||||||
// this document, since it no longer owns any file_blobs row.
|
// this document, since it no longer owns any file_blobs row.
|
||||||
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
|
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, cutoff, 1000)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
|
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -24,9 +24,30 @@ ON CONFLICT (channel_id, id, category) DO NOTHING`, channelID, id, int16(c), dat
|
||||||
return fmt.Errorf("insert channel media index: %w", err)
|
return fmt.Errorf("insert channel media index: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := setDocumentCategoryTx(ctx, tx, media); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindChannelMessage, channelMessageRefKey(channelID, id))
|
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindChannelMessage, channelMessageRefKey(channelID, id))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// setDocumentCategoryTx persists media.Document's retention-sweep category
|
||||||
|
// (see domain.DocumentMediaCategory) onto its documents row, alongside the
|
||||||
|
// message_box_media/channel_message_media index write these callers already
|
||||||
|
// do -- zero new classification logic, just also stamping the already
|
||||||
|
// computed value directly onto documents.category so the per-category
|
||||||
|
// storage retention sweep (internal/app/files/retention.go) can filter on it
|
||||||
|
// without a join. No-op for non-document media.
|
||||||
|
func setDocumentCategoryTx(ctx context.Context, tx pgx.Tx, media *domain.MessageMedia) error {
|
||||||
|
if media == nil || media.Kind != domain.MessageMediaKindDocument || media.Document == nil || media.Document.ID == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
category := domain.DocumentMediaCategory(media)
|
||||||
|
if _, err := tx.Exec(ctx, `UPDATE documents SET category = $2 WHERE id = $1`, media.Document.ID, int16(category)); err != nil {
|
||||||
|
return fmt.Errorf("set document category: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// deleteChannelMediaIndexTx 清掉一条频道消息的全部索引行(编辑改媒体前先清后插)。
|
// deleteChannelMediaIndexTx 清掉一条频道消息的全部索引行(编辑改媒体前先清后插)。
|
||||||
// 只在 replaceChannelMediaIndexTx 内被调用;真正的消息删除不清 *_media 分类索引
|
// 只在 replaceChannelMediaIndexTx 内被调用;真正的消息删除不清 *_media 分类索引
|
||||||
// (读时靠 JOIN deleted 过滤,见文件头注释),但仍需在此清掉 media_references,
|
// (读时靠 JOIN deleted 过滤,见文件头注释),但仍需在此清掉 media_references,
|
||||||
|
|
@ -57,6 +78,9 @@ ON CONFLICT (owner_user_id, box_id, category) DO NOTHING`, ownerUserID, boxID, p
|
||||||
return fmt.Errorf("insert message box media index: %w", err)
|
return fmt.Errorf("insert message box media index: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := setDocumentCategoryTx(ctx, tx, media); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindMessageBox, messageBoxRefKey(ownerUserID, boxID))
|
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindMessageBox, messageBoxRefKey(ownerUserID, boxID))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -105,6 +105,30 @@ RETURNING media_kind, media_id`, string(refKind), refKey)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListMediaReferences returns every live reference row for (kind, mediaID) --
|
||||||
|
// used by the storage retention sweep to turn a hard-retention/eviction blob
|
||||||
|
// purge into a visible notice on every message that still embeds the purged
|
||||||
|
// media. See internal/app/files.notifyRetentionPurge.
|
||||||
|
func (s *MediaStore) ListMediaReferences(ctx context.Context, kind domain.MediaKind, mediaID int64) ([]domain.MediaReference, error) {
|
||||||
|
rows, err := s.q.ListMediaReferences(ctx, sqlcgen.ListMediaReferencesParams{
|
||||||
|
MediaKind: string(kind),
|
||||||
|
MediaID: mediaID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("list media references: %w", err)
|
||||||
|
}
|
||||||
|
out := make([]domain.MediaReference, 0, len(rows))
|
||||||
|
for _, r := range rows {
|
||||||
|
out = append(out, domain.MediaReference{
|
||||||
|
Kind: domain.MediaKind(r.MediaKind),
|
||||||
|
MediaID: r.MediaID,
|
||||||
|
RefKind: domain.MediaRefKind(r.RefKind),
|
||||||
|
RefKey: r.RefKey,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
// ---- storage retention sweep ----
|
// ---- storage retention sweep ----
|
||||||
|
|
||||||
// OrphanDocumentIfUnreferenced marks a document orphaned right now if
|
// OrphanDocumentIfUnreferenced marks a document orphaned right now if
|
||||||
|
|
@ -128,20 +152,22 @@ WHERE id = $1
|
||||||
return tag.RowsAffected() > 0, nil
|
return tag.RowsAffected() > 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListOrphanedDocumentIDsOlderThan returns document ids whose orphaned_at is
|
// ListOrphanedDocumentIDsOlderThan returns document ids in the given category
|
||||||
// set and older than cutoff, oldest first, up to limit.
|
// whose orphaned_at is set and older than cutoff, oldest first, up to limit.
|
||||||
func (s *MediaStore) ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
func (s *MediaStore) ListOrphanedDocumentIDsOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error) {
|
||||||
if limit <= 0 {
|
if limit <= 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return s.q.ListOrphanedDocumentIDsOlderThan(ctx, sqlcgen.ListOrphanedDocumentIDsOlderThanParams{
|
return s.q.ListOrphanedDocumentIDsOlderThan(ctx, sqlcgen.ListOrphanedDocumentIDsOlderThanParams{
|
||||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||||
|
Category: int16(category),
|
||||||
BatchLimit: int32(limit),
|
BatchLimit: int32(limit),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListOrphanedPhotoIDsOlderThan returns photo ids whose orphaned_at is set
|
// ListOrphanedPhotoIDsOlderThan returns photo ids (excluding a live avatar --
|
||||||
// and older than cutoff, oldest first, up to limit.
|
// see ListAvatarOrphanedPhotoIDsOlderThan) whose orphaned_at is set and older
|
||||||
|
// than cutoff, oldest first, up to limit.
|
||||||
func (s *MediaStore) ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
func (s *MediaStore) ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||||
if limit <= 0 {
|
if limit <= 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
|
@ -152,6 +178,19 @@ func (s *MediaStore) ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff t
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListAvatarOrphanedPhotoIDsOlderThan is the "Avatar" category counterpart of
|
||||||
|
// ListOrphanedPhotoIDsOlderThan -- see the query's doc comment for why this
|
||||||
|
// is expected to stay empty in practice under "orphan" mode.
|
||||||
|
func (s *MediaStore) ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||||
|
if limit <= 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return s.q.ListAvatarOrphanedPhotoIDsOlderThan(ctx, sqlcgen.ListAvatarOrphanedPhotoIDsOlderThanParams{
|
||||||
|
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||||
|
BatchLimit: int32(limit),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// CountFileBlobRefs reports how many file_blobs rows still point at
|
// CountFileBlobRefs reports how many file_blobs rows still point at
|
||||||
// (backend, objectKey) -- the caller must not physically delete the object
|
// (backend, objectKey) -- the caller must not physically delete the object
|
||||||
// from that backend while this is > 0 (content-addressed storage: the same
|
// from that backend while this is > 0 (content-addressed storage: the same
|
||||||
|
|
@ -197,24 +236,26 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
|
||||||
return blobs, nil
|
return blobs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListDocumentIDsForHardRetentionOlderThan returns document ids older than
|
// ListDocumentIDsForHardRetentionOlderThan returns document ids in the given
|
||||||
// cutoff (by upload/created_at) that still own at least one file_blobs row,
|
// category older than cutoff (by upload/created_at) that still own at least
|
||||||
// oldest first, up to limit -- the "hard" retention sweep's candidate list.
|
// one file_blobs row, oldest first, up to limit -- the "hard" retention
|
||||||
// Unlike ListOrphanedDocumentIDsOlderThan, this ignores media_references
|
// sweep's candidate list. Unlike ListOrphanedDocumentIDsOlderThan, this
|
||||||
// entirely: a document still referenced by a live message is exactly as
|
// ignores media_references entirely: a document still referenced by a live
|
||||||
// eligible as an orphaned one.
|
// message is exactly as eligible as an orphaned one.
|
||||||
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error) {
|
||||||
if limit <= 0 {
|
if limit <= 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
|
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
|
||||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||||
|
Category: int16(category),
|
||||||
BatchLimit: int32(limit),
|
BatchLimit: int32(limit),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
|
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
|
||||||
// ListDocumentIDsForHardRetentionOlderThan.
|
// ListDocumentIDsForHardRetentionOlderThan (excluding a live avatar -- see
|
||||||
|
// ListAvatarPhotoIDsForHardRetentionOlderThan).
|
||||||
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||||
if limit <= 0 {
|
if limit <= 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
|
@ -225,6 +266,18 @@ func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListAvatarPhotoIDsForHardRetentionOlderThan is the "Avatar" category
|
||||||
|
// counterpart of ListPhotoIDsForHardRetentionOlderThan.
|
||||||
|
func (s *MediaStore) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||||
|
if limit <= 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return s.q.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListAvatarPhotoIDsForHardRetentionOlderThanParams{
|
||||||
|
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||||
|
BatchLimit: int32(limit),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
|
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
|
||||||
// (main body + thumbnail variants), returning what was deleted so the caller
|
// (main body + thumbnail variants), returning what was deleted so the caller
|
||||||
// can physically remove each object from its backend once confirming (via
|
// can physically remove each object from its backend once confirming (via
|
||||||
|
|
@ -292,6 +345,32 @@ func (s *MediaStore) DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]d
|
||||||
return blobs, nil
|
return blobs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListOldestMediaForEviction returns up to limit documents and limit photos
|
||||||
|
// still owning file_blobs bytes, oldest-uploaded first, for the active
|
||||||
|
// eviction sweep to interleave by actual created_at (not drain one table
|
||||||
|
// before the other) -- see domain.EvictionCandidate.
|
||||||
|
func (s *MediaStore) ListOldestMediaForEviction(ctx context.Context, limit int) ([]domain.EvictionCandidate, error) {
|
||||||
|
if limit <= 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
docs, err := s.q.ListOldestDocumentsForEviction(ctx, int32(limit))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("list oldest documents for eviction: %w", err)
|
||||||
|
}
|
||||||
|
photos, err := s.q.ListOldestPhotosForEviction(ctx, int32(limit))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("list oldest photos for eviction: %w", err)
|
||||||
|
}
|
||||||
|
out := make([]domain.EvictionCandidate, 0, len(docs)+len(photos))
|
||||||
|
for _, d := range docs {
|
||||||
|
out = append(out, domain.EvictionCandidate{Kind: domain.MediaKindDocument, MediaID: d.ID, CreatedAt: d.CreatedAt.Time})
|
||||||
|
}
|
||||||
|
for _, p := range photos {
|
||||||
|
out = append(out, domain.EvictionCandidate{Kind: domain.MediaKindPhoto, MediaID: p.ID, CreatedAt: p.CreatedAt.Time})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
|
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
|
||||||
// (one per rendition size), returning what was deleted so the caller can
|
// (one per rendition size), returning what was deleted so the caller can
|
||||||
// physically remove each object from its backend once confirming (via
|
// physically remove each object from its backend once confirming (via
|
||||||
|
|
|
||||||
|
|
@ -83,9 +83,18 @@ func (s *MessageStore) EditMessage(ctx context.Context, req domain.EditMessageRe
|
||||||
}
|
}
|
||||||
authorEdit := target.Outgoing && target.MessageSenderID == req.OwnerUserID && target.FromUserID == req.OwnerUserID
|
authorEdit := target.Outgoing && target.MessageSenderID == req.OwnerUserID && target.FromUserID == req.OwnerUserID
|
||||||
viaBotEdit := req.ViaBotEditBotID != 0 && target.ViaBotID == req.ViaBotEditBotID
|
viaBotEdit := req.ViaBotEditBotID != 0 && target.ViaBotID == req.ViaBotEditBotID
|
||||||
if !authorEdit && !viaBotEdit && !req.WebPageResolve && !validTodoParticipantEdit(req, target, oldEntities) {
|
if !authorEdit && !viaBotEdit && !req.WebPageResolve && !req.RetentionPurge && !validTodoParticipantEdit(req, target, oldEntities) {
|
||||||
return res, domain.ErrMessageAuthorRequired
|
return res, domain.ErrMessageAuthorRequired
|
||||||
}
|
}
|
||||||
|
if req.RetentionPurge {
|
||||||
|
// 幂等守卫：已经是这条 retention 通知的消息不再重复编辑（同一被回收
|
||||||
|
// 媒体可能被多个 box 的 media_references 行各引用一次，比如自己与对端
|
||||||
|
// 各自的 box——同一条共享 private_message 只需真正编辑一次）。
|
||||||
|
if targetMedia, err := decodeMessageMedia(target.MediaJson); err == nil &&
|
||||||
|
targetMedia != nil && targetMedia.Kind == domain.MessageMediaKindService {
|
||||||
|
return res, domain.ErrMessageNotModified
|
||||||
|
}
|
||||||
|
}
|
||||||
richChanged := req.SetRichMessage && !richMessagesEqual(targetRich, req.RichMessage)
|
richChanged := req.SetRichMessage && !richMessagesEqual(targetRich, req.RichMessage)
|
||||||
if req.Media == nil && !req.SetReplyMarkup && !richChanged && target.Body == req.Message && target.HideEdited == req.HideEdited && sameMessageEntities(oldEntities, req.Entities) {
|
if req.Media == nil && !req.SetReplyMarkup && !richChanged && target.Body == req.Message && target.HideEdited == req.HideEdited && sameMessageEntities(oldEntities, req.Entities) {
|
||||||
return res, domain.ErrMessageNotModified
|
return res, domain.ErrMessageNotModified
|
||||||
|
|
|
||||||
|
|
@ -202,6 +202,16 @@ WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
|
||||||
UPDATE photos SET orphaned_at = NULL
|
UPDATE photos SET orphaned_at = NULL
|
||||||
WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
|
WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
|
||||||
|
|
||||||
|
-- name: ListMediaReferences :many
|
||||||
|
-- Every live reference to a document/photo -- used by the storage retention
|
||||||
|
-- sweep to turn a hard-retention/eviction blob purge into a visible notice on
|
||||||
|
-- every message that still embeds the purged media (see
|
||||||
|
-- files.notifyRetentionPurge). profile_photo/sticker_set/gift refs have no
|
||||||
|
-- message to edit and are filtered by the caller, not this query.
|
||||||
|
SELECT media_kind, media_id, ref_kind, ref_key
|
||||||
|
FROM media_references
|
||||||
|
WHERE media_kind = sqlc.arg(media_kind)::text AND media_id = sqlc.arg(media_id)::bigint;
|
||||||
|
|
||||||
-- name: RemoveMediaReference :exec
|
-- name: RemoveMediaReference :exec
|
||||||
DELETE FROM media_references
|
DELETE FROM media_references
|
||||||
WHERE media_kind = sqlc.arg(media_kind)::text
|
WHERE media_kind = sqlc.arg(media_kind)::text
|
||||||
|
|
@ -226,15 +236,36 @@ WHERE id = sqlc.arg(media_id)::bigint
|
||||||
);
|
);
|
||||||
|
|
||||||
-- name: ListOrphanedDocumentIDsOlderThan :many
|
-- name: ListOrphanedDocumentIDsOlderThan :many
|
||||||
|
-- category selects one documents.category bucket per sweep tick (per-category
|
||||||
|
-- retention age, see internal/app/files/retention.go) -- 0 (MediaCategoryNone)
|
||||||
|
-- covers unclassified documents (e.g. stickers), which always use the shared
|
||||||
|
-- global age since there is no per-category override for that bucket.
|
||||||
SELECT id FROM documents
|
SELECT id FROM documents
|
||||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
|
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
|
||||||
|
AND category = sqlc.arg(category)::smallint
|
||||||
ORDER BY orphaned_at ASC
|
ORDER BY orphaned_at ASC
|
||||||
LIMIT sqlc.arg(batch_limit)::int;
|
LIMIT sqlc.arg(batch_limit)::int;
|
||||||
|
|
||||||
-- name: ListOrphanedPhotoIDsOlderThan :many
|
-- name: ListOrphanedPhotoIDsOlderThan :many
|
||||||
SELECT id FROM photos
|
-- Excludes photos currently active as someone's avatar -- see
|
||||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
|
-- ListAvatarOrphanedPhotoIDsOlderThan for that split-off bucket.
|
||||||
ORDER BY orphaned_at ASC
|
SELECT p.id FROM photos p
|
||||||
|
WHERE p.orphaned_at IS NOT NULL AND p.orphaned_at < sqlc.arg(cutoff)::timestamptz
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||||
|
ORDER BY p.orphaned_at ASC
|
||||||
|
LIMIT sqlc.arg(batch_limit)::int;
|
||||||
|
|
||||||
|
-- name: ListAvatarOrphanedPhotoIDsOlderThan :many
|
||||||
|
-- Same as ListOrphanedPhotoIDsOlderThan but only photos currently active as
|
||||||
|
-- someone's avatar (profile_photos.active) -- lets the Avatar category carry
|
||||||
|
-- its own retention age. In practice a live avatar is never orphaned (an
|
||||||
|
-- active profile_photos row is itself a media_references entry), so this
|
||||||
|
-- bucket is expected to stay empty under "orphan" mode; kept for symmetry
|
||||||
|
-- with the "hard" mode avatar split, which is the one that actually matters.
|
||||||
|
SELECT p.id FROM photos p
|
||||||
|
WHERE p.orphaned_at IS NOT NULL AND p.orphaned_at < sqlc.arg(cutoff)::timestamptz
|
||||||
|
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||||
|
ORDER BY p.orphaned_at ASC
|
||||||
LIMIT sqlc.arg(batch_limit)::int;
|
LIMIT sqlc.arg(batch_limit)::int;
|
||||||
|
|
||||||
-- name: ListDocumentIDsForHardRetentionOlderThan :many
|
-- name: ListDocumentIDsForHardRetentionOlderThan :many
|
||||||
|
|
@ -244,9 +275,12 @@ LIMIT sqlc.arg(batch_limit)::int;
|
||||||
-- keeps this sweep from re-selecting the same document forever: once its
|
-- keeps this sweep from re-selecting the same document forever: once its
|
||||||
-- blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
|
-- blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
|
||||||
-- rows but deliberately leaves this documents row in place), it naturally
|
-- rows but deliberately leaves this documents row in place), it naturally
|
||||||
-- drops out of this query on the next pass.
|
-- drops out of this query on the next pass. category scopes to one
|
||||||
|
-- documents.category bucket per sweep tick -- see
|
||||||
|
-- ListOrphanedDocumentIDsOlderThan for the 0/None fallback note.
|
||||||
SELECT d.id FROM documents d
|
SELECT d.id FROM documents d
|
||||||
WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
|
WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
|
||||||
|
AND d.category = sqlc.arg(category)::smallint
|
||||||
AND EXISTS (
|
AND EXISTS (
|
||||||
SELECT 1 FROM file_blobs fb
|
SELECT 1 FROM file_blobs fb
|
||||||
WHERE fb.location_key = 'doc:' || d.id::text
|
WHERE fb.location_key = 'doc:' || d.id::text
|
||||||
|
|
@ -257,9 +291,11 @@ LIMIT sqlc.arg(batch_limit)::int;
|
||||||
|
|
||||||
-- name: ListPhotoIDsForHardRetentionOlderThan :many
|
-- name: ListPhotoIDsForHardRetentionOlderThan :many
|
||||||
-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
|
-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
|
||||||
-- candidate selection, for photos.
|
-- candidate selection, for photos. Excludes photos currently active as
|
||||||
|
-- someone's avatar -- see ListAvatarPhotoIDsForHardRetentionOlderThan.
|
||||||
SELECT p.id FROM photos p
|
SELECT p.id FROM photos p
|
||||||
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||||
AND EXISTS (
|
AND EXISTS (
|
||||||
SELECT 1 FROM file_blobs fb
|
SELECT 1 FROM file_blobs fb
|
||||||
WHERE fb.location_key = 'photo:' || p.id::text
|
WHERE fb.location_key = 'photo:' || p.id::text
|
||||||
|
|
@ -268,6 +304,52 @@ WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
||||||
ORDER BY p.created_at ASC
|
ORDER BY p.created_at ASC
|
||||||
LIMIT sqlc.arg(batch_limit)::int;
|
LIMIT sqlc.arg(batch_limit)::int;
|
||||||
|
|
||||||
|
-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many
|
||||||
|
-- Same as ListPhotoIDsForHardRetentionOlderThan but only photos currently
|
||||||
|
-- active as someone's avatar (profile_photos.active) -- lets the Avatar
|
||||||
|
-- category carry its own retention age, independent of ordinary shared-media
|
||||||
|
-- photos.
|
||||||
|
SELECT p.id FROM photos p
|
||||||
|
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
||||||
|
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1 FROM file_blobs fb
|
||||||
|
WHERE fb.location_key = 'photo:' || p.id::text
|
||||||
|
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||||
|
)
|
||||||
|
ORDER BY p.created_at ASC
|
||||||
|
LIMIT sqlc.arg(batch_limit)::int;
|
||||||
|
|
||||||
|
-- name: ListOldestDocumentsForEviction :many
|
||||||
|
-- Active eviction (TELESRV_STORAGE_EVICTION_ENABLE): candidates are every
|
||||||
|
-- document that still owns at least one file_blobs row, oldest-uploaded
|
||||||
|
-- first, REGARDLESS of category or age -- unlike the retention sweeps above,
|
||||||
|
-- eviction only cares about reclaiming bytes once the total physical budget
|
||||||
|
-- (TELESRV_STORAGE_MAX_TOTAL_BYTES) is exceeded. created_at is returned so
|
||||||
|
-- the caller can interleave these with ListOldestPhotosForEviction by actual
|
||||||
|
-- age instead of draining one table before touching the other.
|
||||||
|
SELECT d.id, d.created_at FROM documents d
|
||||||
|
WHERE EXISTS (
|
||||||
|
SELECT 1 FROM file_blobs fb
|
||||||
|
WHERE fb.location_key = 'doc:' || d.id::text
|
||||||
|
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
||||||
|
)
|
||||||
|
ORDER BY d.created_at ASC
|
||||||
|
LIMIT sqlc.arg(batch_limit)::int;
|
||||||
|
|
||||||
|
-- name: ListOldestPhotosForEviction :many
|
||||||
|
-- See ListOldestDocumentsForEviction -- same oldest-first eviction candidate
|
||||||
|
-- selection, for photos (avatars included: eviction is bytes-only and does
|
||||||
|
-- not honor the Avatar category's separate retention age).
|
||||||
|
SELECT p.id, p.created_at FROM photos p
|
||||||
|
WHERE EXISTS (
|
||||||
|
SELECT 1 FROM file_blobs fb
|
||||||
|
WHERE fb.location_key = 'photo:' || p.id::text
|
||||||
|
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||||
|
)
|
||||||
|
ORDER BY p.created_at ASC
|
||||||
|
LIMIT sqlc.arg(batch_limit)::int;
|
||||||
|
|
||||||
-- name: CountFileBlobRefs :one
|
-- name: CountFileBlobRefs :one
|
||||||
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;
|
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;
|
||||||
|
|
||||||
|
|
|
||||||
360
internal/store/postgres/retention_v2_integration_test.go
Normal file
360
internal/store/postgres/retention_v2_integration_test.go
Normal file
|
|
@ -0,0 +1,360 @@
|
||||||
|
package postgres
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"telesrv/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestRetentionPurgeBypassProducesServiceMessage_PrivateMessage exercises
|
||||||
|
// Part 1 of the storage retention v2 plan end-to-end against a real
|
||||||
|
// Postgres: EditMessage with RetentionPurge=true bypasses the ordinary
|
||||||
|
// author check (the storage retention sweep is not the message's author) and
|
||||||
|
// replaces the message media with a messageActionCustomAction service
|
||||||
|
// payload, and a second retention-purge edit on the now-already-service
|
||||||
|
// message is a no-op (ErrMessageNotModified) -- guarding the idempotency
|
||||||
|
// guard that keeps a shared media_references duplicate (owner+peer both
|
||||||
|
// referencing the same purged document) from double-editing the same box.
|
||||||
|
func TestRetentionPurgeBypassProducesServiceMessage_PrivateMessage(t *testing.T) {
|
||||||
|
pool := testPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
suffix := randomSuffix(t)
|
||||||
|
|
||||||
|
users := NewUserStore(pool)
|
||||||
|
sender, err := users.Create(ctx, domain.User{AccessHash: 81, Phone: "+1997" + suffix + "01", FirstName: "RetentionSender"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sender: %v", err)
|
||||||
|
}
|
||||||
|
recipient, err := users.Create(ctx, domain.User{AccessHash: 82, Phone: "+1997" + suffix + "02", FirstName: "RetentionRecipient"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create recipient: %v", err)
|
||||||
|
}
|
||||||
|
ids := []int64{sender.ID, recipient.ID}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_, _ = pool.Exec(ctx, "DELETE FROM message_boxes WHERE owner_user_id = ANY($1::bigint[])", ids)
|
||||||
|
_, _ = pool.Exec(ctx, "DELETE FROM private_messages WHERE sender_user_id = ANY($1::bigint[])", ids)
|
||||||
|
_, _ = pool.Exec(ctx, "DELETE FROM dialogs WHERE user_id = ANY($1::bigint[])", ids)
|
||||||
|
_, _ = pool.Exec(ctx, "DELETE FROM users WHERE id = ANY($1::bigint[])", ids)
|
||||||
|
})
|
||||||
|
|
||||||
|
messages := NewMessageStore(pool, WithMessageAllocators(&perUserCounterAllocator{}))
|
||||||
|
sent, err := messages.SendPrivateText(ctx, domain.SendPrivateTextRequest{
|
||||||
|
SenderUserID: sender.ID,
|
||||||
|
RecipientUserID: recipient.ID,
|
||||||
|
RandomID: time.Now().UnixNano(),
|
||||||
|
Message: "a file worth keeping",
|
||||||
|
Media: &domain.MessageMedia{
|
||||||
|
Kind: domain.MessageMediaKindDocument,
|
||||||
|
Document: &domain.Document{ID: time.Now().UnixNano(), AccessHash: 1, MimeType: "application/octet-stream"},
|
||||||
|
},
|
||||||
|
Date: int(time.Now().Unix()),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("send private media: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
noticeMedia := &domain.MessageMedia{
|
||||||
|
Kind: domain.MessageMediaKindService,
|
||||||
|
ServiceAction: &domain.MessageServiceAction{
|
||||||
|
Kind: domain.MessageServiceActionCustomText,
|
||||||
|
Text: domain.RetentionPurgeNoticeText,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
req := domain.EditMessageRequest{
|
||||||
|
OwnerUserID: sender.ID,
|
||||||
|
Peer: domain.Peer{Type: domain.PeerTypeUser, ID: recipient.ID},
|
||||||
|
ID: sent.SenderMessage.ID,
|
||||||
|
SetRichMessage: true,
|
||||||
|
Media: noticeMedia,
|
||||||
|
RetentionPurge: true,
|
||||||
|
}
|
||||||
|
res, err := messages.EditMessage(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("EditMessage(RetentionPurge): %v", err)
|
||||||
|
}
|
||||||
|
self := res.Self()
|
||||||
|
if self.Message.Media == nil || self.Message.Media.Kind != domain.MessageMediaKindService {
|
||||||
|
t.Fatalf("edited message media = %+v, want service kind", self.Message.Media)
|
||||||
|
}
|
||||||
|
if self.Message.Media.ServiceAction == nil || self.Message.Media.ServiceAction.Kind != domain.MessageServiceActionCustomText {
|
||||||
|
t.Fatalf("edited message service action = %+v, want custom_text", self.Message.Media.ServiceAction)
|
||||||
|
}
|
||||||
|
if self.Message.Media.ServiceAction.Text != domain.RetentionPurgeNoticeText {
|
||||||
|
t.Fatalf("edited message notice text = %q, want %q", self.Message.Media.ServiceAction.Text, domain.RetentionPurgeNoticeText)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second retention-purge edit is a no-op: the message is already the
|
||||||
|
// notice, so this must not keep bumping pts / re-writing it forever.
|
||||||
|
if _, err := messages.EditMessage(ctx, req); err != domain.ErrMessageNotModified {
|
||||||
|
t.Fatalf("second RetentionPurge edit err = %v, want ErrMessageNotModified", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A non-author, non-bypass edit on someone else's message must still be
|
||||||
|
// rejected -- RetentionPurge only bypasses the author check when it is
|
||||||
|
// actually set, not for ordinary edits in general.
|
||||||
|
ordinary := req
|
||||||
|
ordinary.RetentionPurge = false
|
||||||
|
ordinary.OwnerUserID = recipient.ID
|
||||||
|
ordinary.Peer = domain.Peer{Type: domain.PeerTypeUser, ID: sender.ID}
|
||||||
|
if _, err := messages.EditMessage(ctx, ordinary); err != domain.ErrMessageAuthorRequired {
|
||||||
|
t.Fatalf("non-bypass edit by non-author err = %v, want ErrMessageAuthorRequired", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRetentionPurgeBypassProducesServiceMessage_ChannelMessage is the
|
||||||
|
// channel counterpart of the private-message test above: channel messages
|
||||||
|
// store their service action in a structurally separate Action column (see
|
||||||
|
// tgChannelMessage), so applyRetentionPurgeChannelMessage must clear body/
|
||||||
|
// media and set Action instead of replacing Media like the private path.
|
||||||
|
func TestRetentionPurgeBypassProducesServiceMessage_ChannelMessage(t *testing.T) {
|
||||||
|
pool := testPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
suffix := randomSuffix(t)
|
||||||
|
|
||||||
|
users := NewUserStore(pool)
|
||||||
|
owner, err := users.Create(ctx, domain.User{AccessHash: 83, Phone: "+1997" + suffix + "03", FirstName: "RetentionChOwner"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create owner: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _, _ = pool.Exec(ctx, "DELETE FROM users WHERE id = $1", owner.ID) })
|
||||||
|
|
||||||
|
channels := NewChannelStore(pool)
|
||||||
|
created, err := channels.CreateChannel(ctx, domain.CreateChannelRequest{CreatorUserID: owner.ID, Title: "Retention " + suffix, Megagroup: true, Date: 1700003000})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create channel: %v", err)
|
||||||
|
}
|
||||||
|
channelID := created.Channel.ID
|
||||||
|
t.Cleanup(func() { _, _ = pool.Exec(ctx, "DELETE FROM channels WHERE id = $1", channelID) })
|
||||||
|
|
||||||
|
sent, err := channels.SendChannelMessage(ctx, domain.SendChannelMessageRequest{
|
||||||
|
UserID: owner.ID, ChannelID: channelID, RandomID: 771122, Message: "old file",
|
||||||
|
Media: &domain.MessageMedia{Kind: domain.MessageMediaKindDocument, Document: &domain.Document{ID: time.Now().UnixNano(), AccessHash: 1}},
|
||||||
|
Date: 1700003001,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("send channel message: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req := domain.EditChannelMessageRequest{
|
||||||
|
ChannelID: channelID,
|
||||||
|
ID: sent.Message.ID,
|
||||||
|
RetentionPurge: true,
|
||||||
|
RetentionPurgeAction: &domain.ChannelMessageAction{Type: domain.ChannelActionCustomText, Text: domain.RetentionPurgeNoticeText},
|
||||||
|
}
|
||||||
|
res, err := channels.EditChannelMessage(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("EditChannelMessage(RetentionPurge): %v", err)
|
||||||
|
}
|
||||||
|
if res.Message.Action == nil || res.Message.Action.Type != domain.ChannelActionCustomText {
|
||||||
|
t.Fatalf("edited channel message action = %+v, want custom_text", res.Message.Action)
|
||||||
|
}
|
||||||
|
if res.Message.Action.Text != domain.RetentionPurgeNoticeText {
|
||||||
|
t.Fatalf("edited channel message notice text = %q, want %q", res.Message.Action.Text, domain.RetentionPurgeNoticeText)
|
||||||
|
}
|
||||||
|
if res.Message.Body != "" || !res.Message.Media.IsZero() {
|
||||||
|
t.Fatalf("edited channel message body/media not cleared: body=%q media=%+v", res.Message.Body, res.Message.Media)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Idempotency guard: a second retention-purge edit on an already-service
|
||||||
|
// message must not succeed again.
|
||||||
|
if _, err := channels.EditChannelMessage(ctx, req); err != domain.ErrMessageNotModified {
|
||||||
|
t.Fatalf("second RetentionPurge channel edit err = %v, want ErrMessageNotModified", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read-back via ListChannelHistory renders as a service message too.
|
||||||
|
hist, err := channels.ListChannelHistory(ctx, owner.ID, domain.ChannelHistoryFilter{ChannelID: channelID, Limit: 10})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("list channel history: %v", err)
|
||||||
|
}
|
||||||
|
var found bool
|
||||||
|
for _, m := range hist.Messages {
|
||||||
|
if m.ID == sent.Message.ID {
|
||||||
|
found = true
|
||||||
|
if m.Action == nil || m.Action.Type != domain.ChannelActionCustomText {
|
||||||
|
t.Fatalf("history action = %+v, want custom_text", m.Action)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("purged message %d not found in channel history", sent.Message.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDocumentCategoryPopulatedOnSend guards Part 2's schema/write-path
|
||||||
|
// change: sending a document message stamps the already-computed
|
||||||
|
// domain.DocumentMediaCategory value directly onto the documents row, at the
|
||||||
|
// same point the shared message_box_media index is written -- zero new
|
||||||
|
// classification logic.
|
||||||
|
func TestDocumentCategoryPopulatedOnSend(t *testing.T) {
|
||||||
|
pool := testPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
suffix := randomSuffix(t)
|
||||||
|
|
||||||
|
users := NewUserStore(pool)
|
||||||
|
sender, err := users.Create(ctx, domain.User{AccessHash: 84, Phone: "+1997" + suffix + "04", FirstName: "CatSender"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create sender: %v", err)
|
||||||
|
}
|
||||||
|
recipient, err := users.Create(ctx, domain.User{AccessHash: 85, Phone: "+1997" + suffix + "05", FirstName: "CatRecipient"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create recipient: %v", err)
|
||||||
|
}
|
||||||
|
ids := []int64{sender.ID, recipient.ID}
|
||||||
|
docID := time.Now().UnixNano()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_, _ = pool.Exec(ctx, "DELETE FROM message_boxes WHERE owner_user_id = ANY($1::bigint[])", ids)
|
||||||
|
_, _ = pool.Exec(ctx, "DELETE FROM private_messages WHERE sender_user_id = ANY($1::bigint[])", ids)
|
||||||
|
_, _ = pool.Exec(ctx, "DELETE FROM dialogs WHERE user_id = ANY($1::bigint[])", ids)
|
||||||
|
_, _ = pool.Exec(ctx, "DELETE FROM users WHERE id = ANY($1::bigint[])", ids)
|
||||||
|
_, _ = pool.Exec(ctx, "DELETE FROM documents WHERE id = $1", docID)
|
||||||
|
_, _ = pool.Exec(ctx, "DELETE FROM media_references WHERE media_kind = 'document' AND media_id = $1", docID)
|
||||||
|
})
|
||||||
|
|
||||||
|
media := NewMediaStore(pool)
|
||||||
|
if err := media.PutDocument(ctx, domain.Document{ID: docID, MimeType: "video/mp4", Size: 2048}); err != nil {
|
||||||
|
t.Fatalf("PutDocument: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
messages := NewMessageStore(pool, WithMessageAllocators(&perUserCounterAllocator{}))
|
||||||
|
_, err = messages.SendPrivateText(ctx, domain.SendPrivateTextRequest{
|
||||||
|
SenderUserID: sender.ID,
|
||||||
|
RecipientUserID: recipient.ID,
|
||||||
|
RandomID: time.Now().UnixNano(),
|
||||||
|
Media: &domain.MessageMedia{
|
||||||
|
Kind: domain.MessageMediaKindDocument,
|
||||||
|
Document: &domain.Document{
|
||||||
|
ID: docID, AccessHash: 1, MimeType: "video/mp4",
|
||||||
|
Attributes: []domain.DocumentAttribute{{Kind: domain.DocAttrVideo, W: 640, H: 480, Duration: 5}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Date: int(time.Now().Unix()),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("send video document: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var category int16
|
||||||
|
if err := pool.QueryRow(ctx, "SELECT category FROM documents WHERE id = $1", docID).Scan(&category); err != nil {
|
||||||
|
t.Fatalf("read document category: %v", err)
|
||||||
|
}
|
||||||
|
if domain.MediaCategory(category) != domain.MediaCategoryVideo {
|
||||||
|
t.Fatalf("document category = %d, want %d (Video)", category, domain.MediaCategoryVideo)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory guards Part 2's
|
||||||
|
// query-layer change: ListDocumentIDsForHardRetentionOlderThan now takes a
|
||||||
|
// category argument and must only return documents in that exact bucket,
|
||||||
|
// even when another old, blob-owning document sits in a different category.
|
||||||
|
func TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory(t *testing.T) {
|
||||||
|
pool := testPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
media := NewMediaStore(pool)
|
||||||
|
|
||||||
|
videoID := time.Now().UnixNano()
|
||||||
|
musicID := videoID + 1
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = ANY($1::bigint[])", []int64{videoID, musicID})
|
||||||
|
_, _ = pool.Exec(context.Background(), "DELETE FROM file_blobs WHERE location_key = ANY($1::text[])",
|
||||||
|
[]string{"doc:" + strconv.FormatInt(videoID, 10), "doc:" + strconv.FormatInt(musicID, 10)})
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, d := range []struct {
|
||||||
|
id int64
|
||||||
|
category domain.MediaCategory
|
||||||
|
}{{videoID, domain.MediaCategoryVideo}, {musicID, domain.MediaCategoryMusic}} {
|
||||||
|
if err := media.PutDocument(ctx, domain.Document{ID: d.id, MimeType: "application/octet-stream", Size: 512}); err != nil {
|
||||||
|
t.Fatalf("PutDocument %d: %v", d.id, err)
|
||||||
|
}
|
||||||
|
if _, err := pool.Exec(ctx, "UPDATE documents SET created_at = now() - interval '100 days', category = $2 WHERE id = $1", d.id, int16(d.category)); err != nil {
|
||||||
|
t.Fatalf("backdate/categorize document %d: %v", d.id, err)
|
||||||
|
}
|
||||||
|
blob := postgresTestBlob("doc:"+strconv.FormatInt(d.id, 10), "cat-filter", 512, "application/octet-stream")
|
||||||
|
if err := media.PutFileBlob(ctx, blob); err != nil {
|
||||||
|
t.Fatalf("PutFileBlob %d: %v", d.id, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cutoff := time.Now().Add(-24 * time.Hour)
|
||||||
|
videoIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryVideo, cutoff, 1000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Video): %v", err)
|
||||||
|
}
|
||||||
|
if !containsInt64(videoIDs, videoID) || containsInt64(videoIDs, musicID) {
|
||||||
|
t.Fatalf("video-category candidates = %v, want to include %d and exclude %d", videoIDs, videoID, musicID)
|
||||||
|
}
|
||||||
|
|
||||||
|
musicIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryMusic, cutoff, 1000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Music): %v", err)
|
||||||
|
}
|
||||||
|
if !containsInt64(musicIDs, musicID) || containsInt64(musicIDs, videoID) {
|
||||||
|
t.Fatalf("music-category candidates = %v, want to include %d and exclude %d", musicIDs, musicID, videoID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEvictionListsOldestAcrossDocumentsAndPhotos guards Part 3's eviction
|
||||||
|
// query layer: ListOldestMediaForEviction must return candidates from both
|
||||||
|
// tables (interleaving by created_at is done by the caller in Go, see
|
||||||
|
// files.Service.EvictOldestMediaOverBudget) so the oldest-overall item can be
|
||||||
|
// picked regardless of which table it lives in.
|
||||||
|
func TestEvictionListsOldestAcrossDocumentsAndPhotos(t *testing.T) {
|
||||||
|
pool := testPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
media := NewMediaStore(pool)
|
||||||
|
|
||||||
|
docID := time.Now().UnixNano()
|
||||||
|
photoID := docID + 1
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = $1", docID)
|
||||||
|
_, _ = pool.Exec(context.Background(), "DELETE FROM photos WHERE id = $1", photoID)
|
||||||
|
_, _ = pool.Exec(context.Background(), "DELETE FROM file_blobs WHERE location_key = ANY($1::text[])",
|
||||||
|
[]string{"doc:" + strconv.FormatInt(docID, 10), "photo:" + strconv.FormatInt(photoID, 10)})
|
||||||
|
})
|
||||||
|
|
||||||
|
if err := media.PutDocument(ctx, domain.Document{ID: docID, MimeType: "application/octet-stream", Size: 256}); err != nil {
|
||||||
|
t.Fatalf("PutDocument: %v", err)
|
||||||
|
}
|
||||||
|
// The document is the older of the two (200 days vs 100 days) -- it must
|
||||||
|
// sort before the photo in the merged eviction candidate list.
|
||||||
|
if _, err := pool.Exec(ctx, "UPDATE documents SET created_at = now() - interval '200 days' WHERE id = $1", docID); err != nil {
|
||||||
|
t.Fatalf("backdate document: %v", err)
|
||||||
|
}
|
||||||
|
if err := media.PutFileBlob(ctx, postgresTestBlob("doc:"+strconv.FormatInt(docID, 10), "evict-doc", 256, "application/octet-stream")); err != nil {
|
||||||
|
t.Fatalf("PutFileBlob doc: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := media.PutPhoto(ctx, domain.Photo{ID: photoID, AccessHash: 1}); err != nil {
|
||||||
|
t.Fatalf("PutPhoto: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := pool.Exec(ctx, "UPDATE photos SET created_at = now() - interval '100 days' WHERE id = $1", photoID); err != nil {
|
||||||
|
t.Fatalf("backdate photo: %v", err)
|
||||||
|
}
|
||||||
|
if err := media.PutFileBlob(ctx, postgresTestBlob("photo:"+strconv.FormatInt(photoID, 10), "evict-photo", 256, "image/jpeg")); err != nil {
|
||||||
|
t.Fatalf("PutFileBlob photo: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates, err := media.ListOldestMediaForEviction(ctx, 1000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListOldestMediaForEviction: %v", err)
|
||||||
|
}
|
||||||
|
var doc, photo *domain.EvictionCandidate
|
||||||
|
for i := range candidates {
|
||||||
|
c := candidates[i]
|
||||||
|
switch {
|
||||||
|
case c.Kind == domain.MediaKindDocument && c.MediaID == docID:
|
||||||
|
doc = &candidates[i]
|
||||||
|
case c.Kind == domain.MediaKindPhoto && c.MediaID == photoID:
|
||||||
|
photo = &candidates[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if doc == nil || photo == nil {
|
||||||
|
t.Fatalf("eviction candidates missing doc/photo: %+v", candidates)
|
||||||
|
}
|
||||||
|
if !doc.CreatedAt.Before(photo.CreatedAt) {
|
||||||
|
t.Fatalf("document created_at %v not before photo created_at %v (document should be older)", doc.CreatedAt, photo.CreatedAt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -825,20 +825,103 @@ func (q *Queries) ListAvailableReactions(ctx context.Context) ([]AvailableReacti
|
||||||
return items, nil
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const listAvatarOrphanedPhotoIDsOlderThan = `-- name: ListAvatarOrphanedPhotoIDsOlderThan :many
|
||||||
|
SELECT p.id FROM photos p
|
||||||
|
WHERE p.orphaned_at IS NOT NULL AND p.orphaned_at < $1::timestamptz
|
||||||
|
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||||
|
ORDER BY p.orphaned_at ASC
|
||||||
|
LIMIT $2::int
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListAvatarOrphanedPhotoIDsOlderThanParams struct {
|
||||||
|
Cutoff pgtype.Timestamptz
|
||||||
|
BatchLimit int32
|
||||||
|
}
|
||||||
|
|
||||||
|
// Same as ListOrphanedPhotoIDsOlderThan but only photos currently active as
|
||||||
|
// someone's avatar (profile_photos.active) -- lets the Avatar category carry
|
||||||
|
// its own retention age. In practice a live avatar is never orphaned (an
|
||||||
|
// active profile_photos row is itself a media_references entry), so this
|
||||||
|
// bucket is expected to stay empty under "orphan" mode; kept for symmetry
|
||||||
|
// with the "hard" mode avatar split, which is the one that actually matters.
|
||||||
|
func (q *Queries) ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListAvatarOrphanedPhotoIDsOlderThanParams) ([]int64, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listAvatarOrphanedPhotoIDsOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []int64
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
if err := rows.Scan(&id); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, id)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const listAvatarPhotoIDsForHardRetentionOlderThan = `-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many
|
||||||
|
SELECT p.id FROM photos p
|
||||||
|
WHERE p.created_at < $1::timestamptz
|
||||||
|
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1 FROM file_blobs fb
|
||||||
|
WHERE fb.location_key = 'photo:' || p.id::text
|
||||||
|
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||||
|
)
|
||||||
|
ORDER BY p.created_at ASC
|
||||||
|
LIMIT $2::int
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListAvatarPhotoIDsForHardRetentionOlderThanParams struct {
|
||||||
|
Cutoff pgtype.Timestamptz
|
||||||
|
BatchLimit int32
|
||||||
|
}
|
||||||
|
|
||||||
|
// Same as ListPhotoIDsForHardRetentionOlderThan but only photos currently
|
||||||
|
// active as someone's avatar (profile_photos.active) -- lets the Avatar
|
||||||
|
// category carry its own retention age, independent of ordinary shared-media
|
||||||
|
// photos.
|
||||||
|
func (q *Queries) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListAvatarPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listAvatarPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []int64
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
if err := rows.Scan(&id); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, id)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
|
const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
|
||||||
SELECT d.id FROM documents d
|
SELECT d.id FROM documents d
|
||||||
WHERE d.created_at < $1::timestamptz
|
WHERE d.created_at < $1::timestamptz
|
||||||
|
AND d.category = $2::smallint
|
||||||
AND EXISTS (
|
AND EXISTS (
|
||||||
SELECT 1 FROM file_blobs fb
|
SELECT 1 FROM file_blobs fb
|
||||||
WHERE fb.location_key = 'doc:' || d.id::text
|
WHERE fb.location_key = 'doc:' || d.id::text
|
||||||
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
||||||
)
|
)
|
||||||
ORDER BY d.created_at ASC
|
ORDER BY d.created_at ASC
|
||||||
LIMIT $2::int
|
LIMIT $3::int
|
||||||
`
|
`
|
||||||
|
|
||||||
type ListDocumentIDsForHardRetentionOlderThanParams struct {
|
type ListDocumentIDsForHardRetentionOlderThanParams struct {
|
||||||
Cutoff pgtype.Timestamptz
|
Cutoff pgtype.Timestamptz
|
||||||
|
Category int16
|
||||||
BatchLimit int32
|
BatchLimit int32
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -848,9 +931,11 @@ type ListDocumentIDsForHardRetentionOlderThanParams struct {
|
||||||
// keeps this sweep from re-selecting the same document forever: once its
|
// keeps this sweep from re-selecting the same document forever: once its
|
||||||
// blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
|
// blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
|
||||||
// rows but deliberately leaves this documents row in place), it naturally
|
// rows but deliberately leaves this documents row in place), it naturally
|
||||||
// drops out of this query on the next pass.
|
// drops out of this query on the next pass. category scopes to one
|
||||||
|
// documents.category bucket per sweep tick -- see
|
||||||
|
// ListOrphanedDocumentIDsOlderThan for the 0/None fallback note.
|
||||||
func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
||||||
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
|
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.Category, arg.BatchLimit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -917,20 +1002,156 @@ func (q *Queries) ListFileBlobsByLocationPrefix(ctx context.Context, arg ListFil
|
||||||
return items, nil
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const listMediaReferences = `-- name: ListMediaReferences :many
|
||||||
|
SELECT media_kind, media_id, ref_kind, ref_key
|
||||||
|
FROM media_references
|
||||||
|
WHERE media_kind = $1::text AND media_id = $2::bigint
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListMediaReferencesParams struct {
|
||||||
|
MediaKind string
|
||||||
|
MediaID int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type ListMediaReferencesRow struct {
|
||||||
|
MediaKind string
|
||||||
|
MediaID int64
|
||||||
|
RefKind string
|
||||||
|
RefKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every live reference to a document/photo -- used by the storage retention
|
||||||
|
// sweep to turn a hard-retention/eviction blob purge into a visible notice on
|
||||||
|
// every message that still embeds the purged media (see
|
||||||
|
// files.notifyRetentionPurge). profile_photo/sticker_set/gift refs have no
|
||||||
|
// message to edit and are filtered by the caller, not this query.
|
||||||
|
func (q *Queries) ListMediaReferences(ctx context.Context, arg ListMediaReferencesParams) ([]ListMediaReferencesRow, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listMediaReferences, arg.MediaKind, arg.MediaID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []ListMediaReferencesRow
|
||||||
|
for rows.Next() {
|
||||||
|
var i ListMediaReferencesRow
|
||||||
|
if err := rows.Scan(
|
||||||
|
&i.MediaKind,
|
||||||
|
&i.MediaID,
|
||||||
|
&i.RefKind,
|
||||||
|
&i.RefKey,
|
||||||
|
); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, i)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const listOldestDocumentsForEviction = `-- name: ListOldestDocumentsForEviction :many
|
||||||
|
SELECT d.id, d.created_at FROM documents d
|
||||||
|
WHERE EXISTS (
|
||||||
|
SELECT 1 FROM file_blobs fb
|
||||||
|
WHERE fb.location_key = 'doc:' || d.id::text
|
||||||
|
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
||||||
|
)
|
||||||
|
ORDER BY d.created_at ASC
|
||||||
|
LIMIT $1::int
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListOldestDocumentsForEvictionRow struct {
|
||||||
|
ID int64
|
||||||
|
CreatedAt pgtype.Timestamptz
|
||||||
|
}
|
||||||
|
|
||||||
|
// Active eviction (TELESRV_STORAGE_EVICTION_ENABLE): candidates are every
|
||||||
|
// document that still owns at least one file_blobs row, oldest-uploaded
|
||||||
|
// first, REGARDLESS of category or age -- unlike the retention sweeps above,
|
||||||
|
// eviction only cares about reclaiming bytes once the total physical budget
|
||||||
|
// (TELESRV_STORAGE_MAX_TOTAL_BYTES) is exceeded. created_at is returned so
|
||||||
|
// the caller can interleave these with ListOldestPhotosForEviction by actual
|
||||||
|
// age instead of draining one table before touching the other.
|
||||||
|
func (q *Queries) ListOldestDocumentsForEviction(ctx context.Context, batchLimit int32) ([]ListOldestDocumentsForEvictionRow, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listOldestDocumentsForEviction, batchLimit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []ListOldestDocumentsForEvictionRow
|
||||||
|
for rows.Next() {
|
||||||
|
var i ListOldestDocumentsForEvictionRow
|
||||||
|
if err := rows.Scan(&i.ID, &i.CreatedAt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, i)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const listOldestPhotosForEviction = `-- name: ListOldestPhotosForEviction :many
|
||||||
|
SELECT p.id, p.created_at FROM photos p
|
||||||
|
WHERE EXISTS (
|
||||||
|
SELECT 1 FROM file_blobs fb
|
||||||
|
WHERE fb.location_key = 'photo:' || p.id::text
|
||||||
|
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||||
|
)
|
||||||
|
ORDER BY p.created_at ASC
|
||||||
|
LIMIT $1::int
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListOldestPhotosForEvictionRow struct {
|
||||||
|
ID int64
|
||||||
|
CreatedAt pgtype.Timestamptz
|
||||||
|
}
|
||||||
|
|
||||||
|
// See ListOldestDocumentsForEviction -- same oldest-first eviction candidate
|
||||||
|
// selection, for photos (avatars included: eviction is bytes-only and does
|
||||||
|
// not honor the Avatar category's separate retention age).
|
||||||
|
func (q *Queries) ListOldestPhotosForEviction(ctx context.Context, batchLimit int32) ([]ListOldestPhotosForEvictionRow, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listOldestPhotosForEviction, batchLimit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []ListOldestPhotosForEvictionRow
|
||||||
|
for rows.Next() {
|
||||||
|
var i ListOldestPhotosForEvictionRow
|
||||||
|
if err := rows.Scan(&i.ID, &i.CreatedAt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, i)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
const listOrphanedDocumentIDsOlderThan = `-- name: ListOrphanedDocumentIDsOlderThan :many
|
const listOrphanedDocumentIDsOlderThan = `-- name: ListOrphanedDocumentIDsOlderThan :many
|
||||||
SELECT id FROM documents
|
SELECT id FROM documents
|
||||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
|
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
|
||||||
|
AND category = $2::smallint
|
||||||
ORDER BY orphaned_at ASC
|
ORDER BY orphaned_at ASC
|
||||||
LIMIT $2::int
|
LIMIT $3::int
|
||||||
`
|
`
|
||||||
|
|
||||||
type ListOrphanedDocumentIDsOlderThanParams struct {
|
type ListOrphanedDocumentIDsOlderThanParams struct {
|
||||||
Cutoff pgtype.Timestamptz
|
Cutoff pgtype.Timestamptz
|
||||||
|
Category int16
|
||||||
BatchLimit int32
|
BatchLimit int32
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// category selects one documents.category bucket per sweep tick (per-category
|
||||||
|
// retention age, see internal/app/files/retention.go) -- 0 (MediaCategoryNone)
|
||||||
|
// covers unclassified documents (e.g. stickers), which always use the shared
|
||||||
|
// global age since there is no per-category override for that bucket.
|
||||||
func (q *Queries) ListOrphanedDocumentIDsOlderThan(ctx context.Context, arg ListOrphanedDocumentIDsOlderThanParams) ([]int64, error) {
|
func (q *Queries) ListOrphanedDocumentIDsOlderThan(ctx context.Context, arg ListOrphanedDocumentIDsOlderThanParams) ([]int64, error) {
|
||||||
rows, err := q.db.Query(ctx, listOrphanedDocumentIDsOlderThan, arg.Cutoff, arg.BatchLimit)
|
rows, err := q.db.Query(ctx, listOrphanedDocumentIDsOlderThan, arg.Cutoff, arg.Category, arg.BatchLimit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -950,9 +1171,10 @@ func (q *Queries) ListOrphanedDocumentIDsOlderThan(ctx context.Context, arg List
|
||||||
}
|
}
|
||||||
|
|
||||||
const listOrphanedPhotoIDsOlderThan = `-- name: ListOrphanedPhotoIDsOlderThan :many
|
const listOrphanedPhotoIDsOlderThan = `-- name: ListOrphanedPhotoIDsOlderThan :many
|
||||||
SELECT id FROM photos
|
SELECT p.id FROM photos p
|
||||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
|
WHERE p.orphaned_at IS NOT NULL AND p.orphaned_at < $1::timestamptz
|
||||||
ORDER BY orphaned_at ASC
|
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||||
|
ORDER BY p.orphaned_at ASC
|
||||||
LIMIT $2::int
|
LIMIT $2::int
|
||||||
`
|
`
|
||||||
|
|
||||||
|
|
@ -961,6 +1183,8 @@ type ListOrphanedPhotoIDsOlderThanParams struct {
|
||||||
BatchLimit int32
|
BatchLimit int32
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Excludes photos currently active as someone's avatar -- see
|
||||||
|
// ListAvatarOrphanedPhotoIDsOlderThan for that split-off bucket.
|
||||||
func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrphanedPhotoIDsOlderThanParams) ([]int64, error) {
|
func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrphanedPhotoIDsOlderThanParams) ([]int64, error) {
|
||||||
rows, err := q.db.Query(ctx, listOrphanedPhotoIDsOlderThan, arg.Cutoff, arg.BatchLimit)
|
rows, err := q.db.Query(ctx, listOrphanedPhotoIDsOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -984,6 +1208,7 @@ func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrp
|
||||||
const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
|
const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
|
||||||
SELECT p.id FROM photos p
|
SELECT p.id FROM photos p
|
||||||
WHERE p.created_at < $1::timestamptz
|
WHERE p.created_at < $1::timestamptz
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
|
||||||
AND EXISTS (
|
AND EXISTS (
|
||||||
SELECT 1 FROM file_blobs fb
|
SELECT 1 FROM file_blobs fb
|
||||||
WHERE fb.location_key = 'photo:' || p.id::text
|
WHERE fb.location_key = 'photo:' || p.id::text
|
||||||
|
|
@ -999,7 +1224,8 @@ type ListPhotoIDsForHardRetentionOlderThanParams struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
|
// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
|
||||||
// candidate selection, for photos.
|
// candidate selection, for photos. Excludes photos currently active as
|
||||||
|
// someone's avatar -- see ListAvatarPhotoIDsForHardRetentionOlderThan.
|
||||||
func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
||||||
rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
|
rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
||||||
|
|
@ -1252,6 +1252,7 @@ type Document struct {
|
||||||
CreatedAt pgtype.Timestamptz
|
CreatedAt pgtype.Timestamptz
|
||||||
OwnerUserID int64
|
OwnerUserID int64
|
||||||
OrphanedAt pgtype.Timestamptz
|
OrphanedAt pgtype.Timestamptz
|
||||||
|
Category int16
|
||||||
}
|
}
|
||||||
|
|
||||||
type EncryptedFile struct {
|
type EncryptedFile struct {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue