astra/owpengram-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fix for retention logic

Browse source
This commit is contained in:
onysd 2026-09-03 05:00:42 +03:00
parent 70c0ba44f0
commit e6bfe2d444
35 changed files with 2108 additions and 132 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmd/telesrv-admin/web/dist/assets/index-IZM8WmmY.js → cmd/telesrv-admin/web/dist/assets/index-By5gK7SA.js vendored
View file

File diff suppressed because one or more lines are too long

2
cmd/telesrv-admin/web/dist/index.html vendored
View file

@ -23,7 +23,7 @@
})();
</script>
<script type="module" crossorigin src="/assets/index-IZM8WmmY.js"></script>
<script type="module" crossorigin src="/assets/index-By5gK7SA.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-B7hI8ol7.css">
</head>
<body>

168
cmd/telesrv-admin/web/src/pages/StoragePage.tsx
View file

@ -1,5 +1,6 @@
import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, Loader2, RefreshCw, Search } from "lucide-react";
import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, Loader2, RefreshCw, Search, Settings2, X } from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import { createPortal } from "react-dom";
import { api, errorMessage } from "../api";
import { ActionButton } from "../components/ActionButton";
import { Alert, EmptyRow, Metric, PageFrame, QueryPanel, SectionHead } from "../components/ui";
@ -200,13 +201,28 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
// GB inputs for byte budgets, a mode dropdown + day count for retention.
const STORAGE_ENV_KEYS = {
blobBackend: "TELESRV_BLOB_BACKEND",
maxTotal: "TELESRV_STORAGE_MAX_TOTAL_BYTES",
minFree: "TELESRV_STORAGE_MIN_FREE_BYTES",
maxUploadFile: "TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES",
retentionMode: "TELESRV_STORAGE_RETENTION_MODE",
retentionMaxAge: "TELESRV_STORAGE_RETENTION_MAX_AGE"
retentionMaxAge: "TELESRV_STORAGE_RETENTION_MAX_AGE",
evictionEnable: "TELESRV_STORAGE_EVICTION_ENABLE"
} as const;
// Per-category retention age overrides -- each optional, empty/0 means
// "inherit the shared Retention age" above. Order matches the admin UI list.
const CATEGORY_AGE_FIELDS: { key: string; envKey: string; label: string }[] = [
{ key: "photo", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_PHOTO", label: "Photo" },
{ key: "video", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_VIDEO", label: "Video" },
{ key: "round_video", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_ROUND_VIDEO", label: "Round video (video message)" },
{ key: "gif", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_GIF", label: "GIF" },
{ key: "music", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_MUSIC", label: "Music" },
{ key: "voice", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_VOICE", label: "Voice message" },
{ key: "file", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_FILE", label: "File" },
{ key: "avatar", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR", label: "Avatar" }
];
// Mirrors internal/app/files.MaxUploadPartBytes * MaxUploadParts -- the
// protocol's own upload-part-count ceiling that TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES
// can never legally exceed (internal/config validates this server-side too;
@ -272,11 +288,23 @@ function DurationField({
const totalMinutes = Number(minutes || "0");
const [unitLabel, setUnitLabel] = useState(() => bestDurationUnit(totalMinutes).label);
const unit = DURATION_UNITS.find((u) => u.label === unitLabel) ?? DURATION_UNITS[2];
const amount = totalMinutes > 0 ? totalMinutes / unit.minutes : NaN;
// 0 is a real, distinct value here (e.g. the shared Retention age
// deliberately set to 0 to disable the default sweep) -- unlike
// ByteSizeField's "0/empty = Unlimited" convention, this must not collapse
// to a blank input, or there'd be no way to tell "0" from "not typed yet".
const amount = Number.isFinite(totalMinutes) ? totalMinutes / unit.minutes : NaN;
function handleAmountChange(raw: string) {
if (!raw.trim()) {
onChange("0");
return;
}
const parsed = Number(raw);
if (!raw.trim() || Number.isNaN(parsed) || parsed <= 0) {
if (Number.isNaN(parsed) || parsed < 0) {
onChange("0");
return;
}
if (parsed === 0) {
onChange("0");
return;
}
@ -292,6 +320,7 @@ function DurationField({
min="0"
step="any"
value={Number.isNaN(amount) ? "" : amount}
placeholder={"0"}
disabled={disabled}
onChange={(event) => handleAmountChange(event.target.value)}
/>
@ -357,15 +386,71 @@ function ByteSizeField({
);
}
// CategoryRetentionModal is a focused editor for the per-category age
// overrides -- pulled out of the main Limits & Retention flow (which was
// getting crowded with 8 extra duration fields) into its own dialog, following
// the same modal-backdrop/command-modal pattern as MintCollectibleUsernameModal.
// It edits the same categoryAgeMinutes state the parent already owns; there's
// no separate save here, just Close -- the one shared "Save limits &
// retention settings" button below still covers these values too.
function CategoryRetentionModal({
categoryAgeMinutes,
onChange,
disabled,
onClose
}: {
categoryAgeMinutes: Record<string, string>;
onChange: (key: string, minutes: string) => void;
disabled: boolean;
onClose: () => void;
}) {
return createPortal(
<div className="modal-backdrop" role="presentation">
<section className="modal command-modal" role="dialog" aria-modal="true" aria-label={"Per-category retention overrides"}>
<div className="modal-head">
<div>
<div className="eyebrow">{"Limits & Retention"}</div>
<h2>{"Per-category overrides"}</h2>
</div>
<button className="icon-btn" type="button" onClick={onClose} aria-label={"Close"}><X size={15} /></button>
</div>
<div className="command-body">
<p className="env-field-desc">
{"Leave a category at 0 to inherit the shared Retention age. The mode switch still applies to all of them -- these only change how old that one category's media must be."}
</p>
{CATEGORY_AGE_FIELDS.map((field) => (
<DurationField
key={field.key}
label={field.label}
help={"Inherits the shared Retention age when left at 0."}
minutes={categoryAgeMinutes[field.key] || "0"}
disabled={disabled}
onChange={(minutes) => onChange(field.key, minutes)}
/>
))}
</div>
<div className="modal-actions">
<button className="btn primary" type="button" onClick={onClose}>{"Close"}</button>
</div>
</section>
</div>,
document.body
);
}
function LimitsRetentionSection() {
const [loaded, setLoaded] = useState(false);
const [error, setError] = useState("");
const [initial, setInitial] = useState<Record<string, string>>({});
const [blobBackend, setBlobBackend] = useState("s3");
const [maxTotalBytes, setMaxTotalBytes] = useState("0");
const [minFreeBytes, setMinFreeBytes] = useState("0");
const [maxUploadFileBytes, setMaxUploadFileBytes] = useState("0");
const [retentionMode, setRetentionMode] = useState("off");
const [retentionAgeMinutes, setRetentionAgeMinutes] = useState("43200");
const [categoryAgeMinutes, setCategoryAgeMinutes] = useState<Record<string, string>>({});
const [categoryModalOpen, setCategoryModalOpen] = useState(false);
const [evictionEnable, setEvictionEnable] = useState(false);
async function load() {
setError("");
@ -378,6 +463,7 @@ function LimitsRetentionSection() {
}
}
setInitial(values);
setBlobBackend((values[STORAGE_ENV_KEYS.blobBackend] || "s3").trim().toLowerCase());
setMaxTotalBytes(values[STORAGE_ENV_KEYS.maxTotal] || "0");
setMinFreeBytes(values[STORAGE_ENV_KEYS.minFree] || "0");
setMaxUploadFileBytes(values[STORAGE_ENV_KEYS.maxUploadFile] || "0");
@ -385,6 +471,14 @@ function LimitsRetentionSection() {
setRetentionMode(mode === "orphan" || mode === "hard" ? mode : "off");
const mins = parseDurationMinutes(values[STORAGE_ENV_KEYS.retentionMaxAge] || "720h");
setRetentionAgeMinutes(mins > 0 ? String(Math.max(1, Math.round(mins))) : "43200");
const nextCategoryAges: Record<string, string> = {};
for (const field of CATEGORY_AGE_FIELDS) {
const raw = values[field.envKey] || "";
const categoryMins = parseDurationMinutes(raw);
nextCategoryAges[field.key] = categoryMins > 0 ? String(Math.round(categoryMins)) : "0";
}
setCategoryAgeMinutes(nextCategoryAges);
setEvictionEnable((values[STORAGE_ENV_KEYS.evictionEnable] || "false").trim().toLowerCase() === "true");
setLoaded(true);
} catch (err) {
setError(errorMessage(err));
@ -404,14 +498,26 @@ function LimitsRetentionSection() {
[STORAGE_ENV_KEYS.retentionMode]: retentionMode,
[STORAGE_ENV_KEYS.retentionMaxAge]: retentionMode === "off"
? (initial[STORAGE_ENV_KEYS.retentionMaxAge] || "720h")
: `${Math.max(1, Math.round(Number(retentionAgeMinutes || "0")))}m`
// 0 is a legitimate value here: "no sweep by default", left for
// per-category overrides below to opt specific categories in. Don't
// clamp it up to a minimum of 1 -- that used to make it impossible to
// ever save a 0 global age.
: `${Math.max(0, Math.round(Number(retentionAgeMinutes || "0")))}m`,
[STORAGE_ENV_KEYS.evictionEnable]: evictionEnable ? "true" : "false"
};
// Per-category overrides: a field left at "inherit global" (0) saves as
// empty rather than a redundant explicit duration, matching the "unset
// means inherit" contract on the server side.
for (const field of CATEGORY_AGE_FIELDS) {
const mins = Math.round(Number(categoryAgeMinutes[field.key] || "0"));
next[field.envKey] = mins > 0 ? `${mins}m` : "";
}
const changed: Record<string, string> = {};
for (const [key, value] of Object.entries(next)) {
if ((initial[key] ?? "") !== value) changed[key] = value;
}
return changed;
}, [maxTotalBytes, minFreeBytes, maxUploadFileBytes, retentionMode, retentionAgeMinutes, initial]);
}, [maxTotalBytes, minFreeBytes, maxUploadFileBytes, retentionMode, retentionAgeMinutes, categoryAgeMinutes, evictionEnable, initial]);
const hasChanges = Object.keys(pendingValues).length > 0;
const uploadCeilingExceeded = Number(maxUploadFileBytes || "0") > PROTOCOL_UPLOAD_CEILING_BYTES;
@ -434,12 +540,14 @@ function LimitsRetentionSection() {
bytes={maxTotalBytes}
onChange={setMaxTotalBytes}
/>
<ByteSizeField
label={"Min free space guard"}
help={"localfs backend only: reject new uploads once real free disk space falls below this. Empty/0 disables the check."}
bytes={minFreeBytes}
onChange={setMinFreeBytes}
/>
{blobBackend === "localfs" && (
<ByteSizeField
label={"Min free space guard"}
help={"localfs backend only: reject new uploads once real free disk space falls below this. Empty/0 disables the check."}
bytes={minFreeBytes}
onChange={setMinFreeBytes}
/>
)}
<ByteSizeField
label={"Max single file size"}
help={"Reject a single upload once its total assembled size exceeds this. Empty/0 = unlimited, bounded only by the protocol's own ~4GB per-file ceiling."}
@ -471,9 +579,10 @@ function LimitsRetentionSection() {
<DurationField
label={"Retention age"}
help={
retentionMode === "hard"
(retentionMode === "hard"
? "How old the media itself must be, counted from when it was uploaded, before its bytes are purged."
: "How long a document or photo must have had zero references before its file is deleted."
: "How long a document or photo must have had zero references before its file is deleted.") +
" Set to 0 to disable the sweep by default and only clean up categories you explicitly override below."
}
minutes={retentionAgeMinutes}
disabled={retentionMode === "off"}
@ -481,6 +590,37 @@ function LimitsRetentionSection() {
/>
</div>
<div className="attr-block" style={{ marginTop: "1.5em" }}>
<button className="btn icon-text" type="button" onClick={() => setCategoryModalOpen(true)}>
<Settings2 size={15} /> {"Per-category overrides..."}
</button>
<span className="env-field-desc">
{"Optional -- give Photo/Video/GIF/Music/Voice/File/Avatar their own retention age instead of the shared one above."}
</span>
</div>
{categoryModalOpen && (
<CategoryRetentionModal
categoryAgeMinutes={categoryAgeMinutes}
onChange={(key, minutes) => setCategoryAgeMinutes((prev) => ({ ...prev, [key]: minutes }))}
disabled={retentionMode === "off"}
onClose={() => setCategoryModalOpen(false)}
/>
)}
<div className="attr-block" style={{ marginTop: "1.5em" }}>
<label className="checkline">
<input
type="checkbox"
checked={evictionEnable}
onChange={(event) => setEvictionEnable(event.target.checked)}
/>
{" "}{"Actively reclaim space once over budget"}
</label>
<p className="env-field-desc">
{"Once total physical storage exceeds the Max total storage budget above, actively delete the oldest files (regardless of category or age) until back under budget -- the same way \"hard\" retention mode purges files. Independent of the retention mode above: this can run even when that's Off. Off by default, since this changes the storage budget from block-new-uploads-only to also reclaiming from existing files."}
</p>
</div>
<div className="gift-table-actions env-save-row">
<ActionButton
tone="warn"

51
cmd/telesrv/main.go
View file

@ -550,6 +550,25 @@ func webPagePreviewOption(cfg config.Config) filesapp.Option {
return filesapp.WithWebPagePreview(cfg.WebPagePreviewMaxBytes, cfg.WebPagePreviewRatePerMin)
}
// fastestPositiveDuration returns the smaller of a and b, treating a
// non-positive value as "not configured" rather than as the smallest
// possible duration -- 0 only when both are non-positive (nothing
// configured). Used to derive the storage retention sweep's ticker cadence
// from whichever of the shared default age and its per-category overrides
// asks to run soonest.
func fastestPositiveDuration(a, b time.Duration) time.Duration {
if a <= 0 {
return b
}
if b <= 0 {
return a
}
if a < b {
return a
}
return b
}
func run(logger *zap.Logger) error {
cfg, err := config.Load()
if err != nil {
@ -933,6 +952,8 @@ func run(logger *zap.Logger) error {
filesapp.WithAdditionalBlobBackend(additionalBlobBackend),
filesapp.WithSpaceGuard(spaceGuard),
filesapp.WithMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes),
filesapp.WithStorageRetentionAges(cfg.StorageRetentionMaxAge, cfg.StorageRetentionMaxAgeByCategory, cfg.StorageRetentionMaxAgeAvatar),
filesapp.WithStorageMaxTotalBytes(cfg.StorageMaxTotalBytes),
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
externalMediaOption(cfg),
webPagePreviewOption(cfg),
@ -1081,12 +1102,31 @@ func run(logger *zap.Logger) error {
// TELESRV_STORAGE_RETENTION_MODE is a single 3-way switch: at most one of
// the orphan-only (safe) and hard (age-based, ignores live references)
// media sweeps is ever wired in, matching "off"/"orphan"/"hard".
// The worker's own maxAge parameter only drives how often the sweep
// ticks (internal/app/maintenance.RetentionWorker.mediaRetentionInterval)
// -- the real per-category cutoff math lives entirely in
// files.Service (WithStorageRetentionAges above). Passing the raw shared
// default here would tick as slowly as a 30-day default even when a
// TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY> override asks for a much
// shorter age (or, if the shared default is 0 -- "disabled by default,
// only specific categories opt in" -- would disable the sweep outright,
// since a 0 maxAge here used to gate the whole sweep off). Use the
// fastest positive age across the shared default and every override
// instead, so the ticker -- and the sweep-enabled gate -- reflect
// whatever is actually configured to run soonest.
fastestRetentionAge := fastestPositiveDuration(cfg.StorageRetentionMaxAge, cfg.StorageRetentionMaxAgeAvatar)
for _, age := range cfg.StorageRetentionMaxAgeByCategory {
fastestRetentionAge = fastestPositiveDuration(fastestRetentionAge, age)
}
switch cfg.StorageRetentionMode {
case config.StorageRetentionModeOrphan:
retentionWorker = retentionWorker.WithOrphanedMediaRetention(filesService, cfg.StorageRetentionMaxAge)
retentionWorker = retentionWorker.WithOrphanedMediaRetention(filesService, fastestRetentionAge)
case config.StorageRetentionModeHard:
retentionWorker = retentionWorker.WithHardMediaRetention(filesService, cfg.StorageRetentionMaxAge)
retentionWorker = retentionWorker.WithHardMediaRetention(filesService, fastestRetentionAge)
}
// Active eviction is independent of TELESRV_STORAGE_RETENTION_MODE (can
// run even when that's "off") and reuses the same media sweep ticker.
retentionWorker = retentionWorker.WithStorageEviction(filesService, cfg.StorageEvictionEnable)
go retentionWorker.Run(ctx)
go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"),
cfg.UploadPartTTL,
@ -1383,6 +1423,13 @@ func run(logger *zap.Logger) error {
messageapp.WithSendPermissionChecker(adminService),
messageapp.WithBusinessAutomation(passwordStore, businessAutomationOptions...),
)
// Wires the storage retention sweep's purge-notice capability now that
// both edit-capable app services exist -- filesService (and the
// background retentionWorker goroutine reading it) was constructed
// earlier, before either was available. A sweep tick that races ahead of
// this call simply finds no notifier yet and skips the notice for that
// tick (best-effort, see files.SetRetentionPurgeNotifier).
filesService.SetRetentionPurgeNotifier(messagesService, channelsService)
moderationService := moderationapp.NewService(
moderationReportStore,
moderationapp.WithMessageReaders(messagesService, channelsService),

33
cmd/telesrv/main_test.go
View file

@ -2,10 +2,43 @@ package main
import (
"testing"
"time"
telegramloginapp "telesrv/internal/app/telegramlogin"
)
// TestFastestPositiveDurationIgnoresNonPositiveValues guards a real reported
// bug: the storage retention sweep's ticker cadence used to be driven purely
// by the shared TELESRV_STORAGE_RETENTION_MAX_AGE default (e.g. 30 days),
// even when a much shorter TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY>
// override was configured -- so a category set to "1 minute" would still
// only actually get swept on the shared default's own slow cadence (or, if
// the shared default was 0, would disable the sweep outright). The worker
// must be handed the fastest positive age across the default and every
// override instead.
func TestFastestPositiveDurationIgnoresNonPositiveValues(t *testing.T) {
cases := []struct {
name string
a, b time.Duration
want time.Duration
}{
{"both positive, a smaller", 30 * 24 * time.Hour, time.Minute, time.Minute},
{"both positive, b smaller", time.Minute, 30 * 24 * time.Hour, time.Minute},
{"a zero (disabled), b positive", 0, time.Minute, time.Minute},
{"a positive, b zero (disabled)", time.Minute, 0, time.Minute},
{"a negative, b positive", -time.Hour, time.Minute, time.Minute},
{"both zero (nothing configured)", 0, 0, 0},
{"both negative", -time.Hour, -time.Minute, -time.Minute},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := fastestPositiveDuration(c.a, c.b); got != c.want {
t.Fatalf("fastestPositiveDuration(%v, %v) = %v, want %v", c.a, c.b, got, c.want)
}
})
}
}
func TestTelegramLoginRPCDependencyPreservesDisabledNil(t *testing.T) {
var disabled *telegramloginapp.Service
if dependency := telegramLoginRPCDependency(disabled); dependency != nil {