fixes for storage managament

This commit is contained in:
onysd 2026-09-03 08:33:06 +03:00
parent e6bfe2d444
commit 8ef2b58bf9
29 changed files with 1768 additions and 63 deletions

View file

@ -340,6 +340,15 @@ TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR=
# TOTAL_BYTES otherwise only ever blocks new uploads, never reclaims from # TOTAL_BYTES otherwise only ever blocks new uploads, never reclaims from
# existing ones. # existing ones.
TELESRV_STORAGE_EVICTION_ENABLE=false TELESRV_STORAGE_EVICTION_ENABLE=false
# Secret-chat encrypted files (photos/documents sent in a secret chat) are
# opaque ciphertext the server can't inspect -- none of the retention/
# eviction settings above ever touch them, so they otherwise accumulate
# forever. Secret chats are single-device on both ends (no multi-device
# sync), so once the one recipient device that will ever ask for a file has
# downloaded it in full, the server has no further reason to keep it.
# Default true: set to false if you'd rather keep the ciphertext around
# regardless (it still can't be read without the secret chat's own key).
TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD=true
# ============================================================================== # ==============================================================================

View file

@ -2539,28 +2539,40 @@ const (
) )
// perOwnerMediaSizeSQL is shared between StorageStats and // perOwnerMediaSizeSQL is shared between StorageStats and
// ListAccountStorageUsage: each document row's size is its stored column; // ListAccountStorageUsage: attributes each document/photo's REAL remaining
// each photo row has no single size column (JSONB sizes holds one entry per // bytes (the sum of every file_blobs row it still owns -- main body plus
// rendition), so its "attributed size" is the largest rendition -- the // thumbnail/rendition variants) to its owner, one row per document/photo so
// dominant cost, thumbnails are comparatively tiny. This is an // an outer COUNT(*)/SUM(size) aggregate still gets an accurate file count
// approximation (not the exact sum of every rendition's blob bytes, which // alongside the byte total. This used to read documents.size / the largest
// would require joining file_blobs by location_key prefix) chosen for admin // photo rendition size directly -- a static value on the metadata row that
// visibility, not billing precision. // survives a hard-retention purge unchanged, so it kept counting bytes for
// files whose blobs were long gone. Joining through file_blobs instead means
// a purged item correctly contributes 0: there is nothing left to attribute.
const perOwnerMediaSizeSQL = ` const perOwnerMediaSizeSQL = `
SELECT owner_user_id, size FROM documents SELECT d.owner_user_id, COALESCE((
SELECT SUM(fb.size) FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
), 0) AS size
FROM documents d
UNION ALL UNION ALL
SELECT p.owner_user_id, COALESCE(( SELECT p.owner_user_id, COALESCE((
SELECT MAX((elem->>'size')::bigint) FROM jsonb_array_elements(p.sizes) elem SELECT SUM(fb.size) FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
), 0) AS size ), 0) AS size
FROM photos p FROM photos p
` `
// StorageStatsRow is the admin panel's storage overview: physical bytes // StorageStatsRow is the admin panel's storage overview: physical bytes
// (from file_blobs, backend-dedup-aware -- what's actually consuming disk // (from file_blobs, backend-dedup-aware -- what's actually consuming disk
// or S3) versus logical bytes (sum of the same approximate per-row // or S3, deduplicated exactly once across the whole system) versus logical
// attribution the per-account breakdown uses, which can legitimately be // bytes (the SAME real, still-existing file_blobs bytes as physical, just
// higher than physical when identical content is shared by more than one // summed per-owner via perOwnerMediaSizeSQL without deduplicating content
// document/photo). // shared across accounts/documents -- so logical can legitimately be higher
// than physical when the same blob is attributed to more than one
// document/photo, but a purged file with no file_blobs rows left correctly
// contributes 0 to both, never a stale non-zero "ghost" size).
type StorageStatsRow struct { type StorageStatsRow struct {
PhysicalBytes int64 `json:"PhysicalBytes,string"` PhysicalBytes int64 `json:"PhysicalBytes,string"`
LogicalBytes int64 `json:"LogicalBytes,string"` LogicalBytes int64 `json:"LogicalBytes,string"`

View file

@ -136,6 +136,7 @@ func (s *server) routes() http.Handler {
mux.Handle("POST /api/actions/set-gif-catalog-category", s.requireAuthAPI(http.HandlerFunc(s.handleSetGifCatalogCategoryAPI))) mux.Handle("POST /api/actions/set-gif-catalog-category", s.requireAuthAPI(http.HandlerFunc(s.handleSetGifCatalogCategoryAPI)))
mux.Handle("POST /api/actions/auto-categorize-gif-catalog", s.requireAuthAPI(http.HandlerFunc(s.handleAutoCategorizeGifCatalogAPI))) mux.Handle("POST /api/actions/auto-categorize-gif-catalog", s.requireAuthAPI(http.HandlerFunc(s.handleAutoCategorizeGifCatalogAPI)))
mux.Handle("POST /api/actions/delete-uncategorized-gifs", s.requireAuthAPI(http.HandlerFunc(s.handleDeleteUncategorizedGifsAPI))) mux.Handle("POST /api/actions/delete-uncategorized-gifs", s.requireAuthAPI(http.HandlerFunc(s.handleDeleteUncategorizedGifsAPI)))
mux.Handle("POST /api/actions/storage-manual-purge", s.requireAuthAPI(http.HandlerFunc(s.handleStorageManualPurgeAPI)))
mux.Handle("POST /api/actions/delete-gif-catalog-entry", s.requireAuthAPI(http.HandlerFunc(s.handleDeleteGifCatalogEntryAPI))) mux.Handle("POST /api/actions/delete-gif-catalog-entry", s.requireAuthAPI(http.HandlerFunc(s.handleDeleteGifCatalogEntryAPI)))
mux.Handle("POST /api/actions/mint-collectible-username", s.requireAuthAPI(http.HandlerFunc(s.handleMintCollectibleUsernameAPI))) mux.Handle("POST /api/actions/mint-collectible-username", s.requireAuthAPI(http.HandlerFunc(s.handleMintCollectibleUsernameAPI)))
mux.Handle("POST /api/actions/transfer-collectible-username", s.requireAuthAPI(http.HandlerFunc(s.handleTransferCollectibleUsernameAPI))) mux.Handle("POST /api/actions/transfer-collectible-username", s.requireAuthAPI(http.HandlerFunc(s.handleTransferCollectibleUsernameAPI)))
@ -2040,6 +2041,37 @@ func (s *server) handleDeleteUncategorizedGifsAPI(w http.ResponseWriter, r *http
writeCommandResultAPI(w, result, err) writeCommandResultAPI(w, result, err)
} }
// storageManualPurgeAPIRequest mirrors ManualPurgeStorageRequest's frontend
// payload (see StoragePage.tsx's manual purge modal): categories/include_avatars
// go through as native JSON, created_before is an ISO/RFC3339 date string
// (produced by `new Date(x).toISOString()` on the frontend, same convention
// as freeze_until -- see setAccountFrozenAPIRequest.Until) that encoding/json
// parses straight into *time.Time; the field left undefined by the frontend
// (no date entered) decodes to nil, meaning no age filter at all.
type storageManualPurgeAPIRequest struct {
CommandID string `json:"command_id"`
Reason string `json:"reason"`
Confirm bool `json:"confirm"`
Categories []string `json:"categories"`
IncludeAvatars bool `json:"include_avatars"`
CreatedBefore *time.Time `json:"created_before"`
}
func (s *server) handleStorageManualPurgeAPI(w http.ResponseWriter, r *http.Request) {
var body storageManualPurgeAPIRequest
if !decodeAction(w, r, &body) {
return
}
req := admin.ManualPurgeStorageRequest{
CommandMeta: s.commandMetaFromAPI(r, body.CommandID, body.Reason, body.Confirm, "storage-manual-purge"),
Categories: body.Categories,
IncludeAvatars: body.IncludeAvatars,
CreatedBefore: body.CreatedBefore,
}
result, err := s.callAdminAPI(r.Context(), "/v1/storage/manual-purge", req)
writeCommandResultAPI(w, result, err)
}
type deleteGifCatalogEntryAPIRequest struct { type deleteGifCatalogEntryAPIRequest struct {
CommandID string `json:"command_id"` CommandID string `json:"command_id"`
Reason string `json:"reason"` Reason string `json:"reason"`

View file

@ -23,7 +23,7 @@
})(); })();
</script> </script>
<script type="module" crossorigin src="/assets/index-By5gK7SA.js"></script> <script type="module" crossorigin src="/assets/index-CMLbsGwc.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-B7hI8ol7.css"> <link rel="stylesheet" crossorigin href="/assets/index-B7hI8ol7.css">
</head> </head>
<body> <body>

View file

@ -151,12 +151,12 @@ export function Dashboard({ navigate }: { navigate: Navigate }) {
icon={<HardDrive />} icon={<HardDrive />}
label="Disk free" label="Disk free"
percent={ percent={
host?.Ready && host.DiskTotalBytes > 0 host?.Ready && host.DiskReady && host.DiskTotalBytes > 0
? ((host.DiskTotalBytes - host.DiskFreeBytes) / host.DiskTotalBytes) * 100 ? ((host.DiskTotalBytes - host.DiskFreeBytes) / host.DiskTotalBytes) * 100
: undefined : undefined
} }
valueText={host?.Ready ? formatBytes(String(host.DiskFreeBytes)) : "…"} valueText={host?.Ready && host.DiskReady ? formatBytes(String(host.DiskFreeBytes)) : "…"}
sub={host?.Ready ? `of ${formatBytes(String(host.DiskTotalBytes))}` : undefined} sub={host?.Ready && host.DiskReady ? `of ${formatBytes(String(host.DiskTotalBytes))}` : "no reading yet"}
warnAbove={85} warnAbove={85}
/> />
</Section> </Section>

View file

@ -1,4 +1,4 @@
import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, Loader2, RefreshCw, Search, Settings2, X } from "lucide-react"; import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, ChevronRight, Loader2, RefreshCw, Search, Settings2, X } from "lucide-react";
import { useEffect, useMemo, useState } from "react"; import { useEffect, useMemo, useState } from "react";
import { createPortal } from "react-dom"; import { createPortal } from "react-dom";
import { api, errorMessage } from "../api"; import { api, errorMessage } from "../api";
@ -38,7 +38,7 @@ function SortableHeader({
); );
} }
function StorageOverviewTab() { function StorageOverviewTab({ navigate }: { navigate: Navigate }) {
const [stats, setStats] = useState<StorageStatsResponse | null>(null); const [stats, setStats] = useState<StorageStatsResponse | null>(null);
const [rows, setRows] = useState<AccountStorageRow[]>([]); const [rows, setRows] = useState<AccountStorageRow[]>([]);
const [hasMore, setHasMore] = useState(false); const [hasMore, setHasMore] = useState(false);
@ -148,6 +148,7 @@ function StorageOverviewTab() {
<SortableHeader label={"Account"} sortKey="username" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} /> <SortableHeader label={"Account"} sortKey="username" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
<SortableHeader label={"Storage used"} sortKey="bytes" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} /> <SortableHeader label={"Storage used"} sortKey="bytes" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
<SortableHeader label={"Files"} sortKey="files" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} /> <SortableHeader label={"Files"} sortKey="files" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
<th></th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@ -157,9 +158,10 @@ function StorageOverviewTab() {
<td>{displayUsername(row.Username) || row.FirstName || "-"}</td> <td>{displayUsername(row.Username) || row.FirstName || "-"}</td>
<td className="mono">{formatBytes(row.Bytes)}</td> <td className="mono">{formatBytes(row.Bytes)}</td>
<td className="mono">{formatQuantity(row.FileCount)}</td> <td className="mono">{formatQuantity(row.FileCount)}</td>
<td><button className="row-link" type="button" onClick={() => navigate(`/accounts/${row.UserID}`)}>{"Details"} <ChevronRight size={14} /></button></td>
</tr> </tr>
))} ))}
{rows.length === 0 && <EmptyRow colSpan={4} />} {rows.length === 0 && <EmptyRow colSpan={5} />}
</tbody> </tbody>
</table> </table>
</div> </div>
@ -174,7 +176,7 @@ function StorageOverviewTab() {
); );
} }
export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) { export function StoragePage({ navigate }: { navigate: Navigate }) {
const [tab, setTab] = useState<"overview" | "limits">("overview"); const [tab, setTab] = useState<"overview" | "limits">("overview");
return ( return (
<PageFrame title={"Storage"} eyebrow={"Media / Storage usage"}> <PageFrame title={"Storage"} eyebrow={"Media / Storage usage"}>
@ -186,7 +188,7 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
{"Limits & Retention"} {"Limits & Retention"}
</button> </button>
</div> </div>
{tab === "overview" ? <StorageOverviewTab /> : <LimitsRetentionSection />} {tab === "overview" ? <StorageOverviewTab navigate={navigate} /> : <LimitsRetentionSection />}
</PageFrame> </PageFrame>
); );
} }
@ -438,6 +440,92 @@ function CategoryRetentionModal({
); );
} }
// ManualPurgeStorageModal lets an operator delete media blob bytes right now
// by hand-picked category (and, optionally, avatars) with an optional
// "created before" age cutoff -- the manual counterpart of the automatic
// hard-retention sweep above. Leaving the date empty purges everything
// matching the selected categories regardless of age. Deletion semantics are
// identical to "hard" retention mode: only the file bytes are removed, never
// the document/photo row, so an affected message still renders its
// placeholder (see internal/app/files.Service.ManualPurge's doc comment).
// Follows the same modal-backdrop/command-modal structure as
// CategoryRetentionModal, but hands the actual mutation off to ActionButton
// (reason + dry-run + confirm) since -- unlike the settings above -- this is
// an immediate, irreversible delete rather than a saved .env value.
function ManualPurgeStorageModal({ onClose }: { onClose: () => void }) {
const [selected, setSelected] = useState<Record<string, boolean>>({});
const [dateValue, setDateValue] = useState("");
const documentFields = CATEGORY_AGE_FIELDS.filter((f) => f.key !== "avatar");
const allSelected = CATEGORY_AGE_FIELDS.every((f) => selected[f.key]);
function toggle(key: string) {
setSelected((prev) => ({ ...prev, [key]: !prev[key] }));
}
function toggleAll() {
const next = !allSelected;
const nextSelected: Record<string, boolean> = {};
for (const field of CATEGORY_AGE_FIELDS) nextSelected[field.key] = next;
setSelected(nextSelected);
}
const chosenCategories = documentFields.filter((f) => selected[f.key]).map((f) => f.key);
const includeAvatars = Boolean(selected.avatar);
const canSubmit = chosenCategories.length > 0 || includeAvatars;
return createPortal(
<div className="modal-backdrop" role="presentation">
<section className="modal command-modal" role="dialog" aria-modal="true" aria-label={"Manually purge storage"}>
<div className="modal-head">
<div>
<div className="eyebrow">{"Limits & Retention"}</div>
<h2>{"Manually purge storage"}</h2>
</div>
<button className="icon-btn" type="button" onClick={onClose} aria-label={"Close"}><X size={15} /></button>
</div>
<div className="command-body">
<p className="env-field-desc">
{"Deletes the file bytes of every document/photo matching the categories below, right now -- independent of the retention mode/age configured above. The message/profile-photo itself is never deleted, only its file; a purged item starts showing as unavailable. Leave \"Created before\" empty to purge everything in the selected categories, regardless of age."}
</p>
<div className="attr-block">
<label className="checkline">
<input type="checkbox" checked={allSelected} onChange={toggleAll} />
{" "}{"Select all"}
</label>
{CATEGORY_AGE_FIELDS.map((field) => (
<label key={field.key} className="checkline">
<input type="checkbox" checked={Boolean(selected[field.key])} onChange={() => toggle(field.key)} />
{" "}{field.label}
</label>
))}
</div>
<label className="duration-field">
<span>{"Created before (optional)"}</span>
<input type="date" value={dateValue} onChange={(event) => setDateValue(event.target.value)} />
<span className="env-field-desc">{"Empty = no age limit, purge everything matching the selected categories."}</span>
</label>
</div>
<div className="modal-actions">
<button className="btn" type="button" onClick={onClose}>{"Close"}</button>
<ActionButton
disabled={!canSubmit}
label={"Purge selected storage"}
tone="danger"
path="/api/actions/storage-manual-purge"
payload={() => ({
categories: chosenCategories,
include_avatars: includeAvatars,
created_before: dateValue ? new Date(dateValue).toISOString() : undefined
})}
/>
</div>
</section>
</div>,
document.body
);
}
function LimitsRetentionSection() { function LimitsRetentionSection() {
const [loaded, setLoaded] = useState(false); const [loaded, setLoaded] = useState(false);
const [error, setError] = useState(""); const [error, setError] = useState("");
@ -451,6 +539,7 @@ function LimitsRetentionSection() {
const [categoryAgeMinutes, setCategoryAgeMinutes] = useState<Record<string, string>>({}); const [categoryAgeMinutes, setCategoryAgeMinutes] = useState<Record<string, string>>({});
const [categoryModalOpen, setCategoryModalOpen] = useState(false); const [categoryModalOpen, setCategoryModalOpen] = useState(false);
const [evictionEnable, setEvictionEnable] = useState(false); const [evictionEnable, setEvictionEnable] = useState(false);
const [manualPurgeOpen, setManualPurgeOpen] = useState(false);
async function load() { async function load() {
setError(""); setError("");
@ -621,6 +710,16 @@ function LimitsRetentionSection() {
</p> </p>
</div> </div>
<div className="attr-block" style={{ marginTop: "1.5em" }}>
<button className="btn danger icon-text" type="button" onClick={() => setManualPurgeOpen(true)}>
<Settings2 size={15} /> {"Manually purge storage..."}
</button>
<span className="env-field-desc">
{"Delete media file bytes right now by hand-picked category and an optional age cutoff, independent of the retention settings above."}
</span>
</div>
{manualPurgeOpen && <ManualPurgeStorageModal onClose={() => setManualPurgeOpen(false)} />}
<div className="gift-table-actions env-save-row"> <div className="gift-table-actions env-save-row">
<ActionButton <ActionButton
tone="warn" tone="warn"

View file

@ -753,6 +753,11 @@ export type HostStatsSnapshot = {
MemTotalBytes: number; MemTotalBytes: number;
DiskFreeBytes: number; DiskFreeBytes: number;
DiskTotalBytes: number; DiskTotalBytes: number;
// False when the disk-space sample itself failed (wrong path, not
// created yet, etc) -- DiskFreeBytes/DiskTotalBytes are stale/zero in
// that case, not "the disk is actually full". Independent of Ready,
// which only covers CPU/memory.
DiskReady: boolean;
Ready: boolean; Ready: boolean;
}; };

View file

@ -0,0 +1,166 @@
package admin
import (
"context"
"testing"
"time"
"telesrv/internal/domain"
)
// fakeStorageService is a minimal StorageService double recording what
// ManualPurgeStorage actually asked it to count/purge, so tests can assert on
// category parsing (including "avatar" never becoming a domain.MediaCategory
// argument) without a real files.Service/Postgres.
type fakeStorageService struct {
countCalls int
purgeCalls int
lastCategories []domain.MediaCategory
lastIncludeAvatars bool
lastBefore *time.Time
countDocs, countPhotos int
purgedDocs, purgedPhotos int
bytesReclaimed int64
countErr, purgeErr error
}
func (f *fakeStorageService) CountManualPurgeCandidates(_ context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time) (int, int, error) {
f.countCalls++
f.lastCategories = categories
f.lastIncludeAvatars = includeAvatars
f.lastBefore = before
return f.countDocs, f.countPhotos, f.countErr
}
func (f *fakeStorageService) ManualPurge(_ context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time, _ int) (int, int, int64, error) {
f.purgeCalls++
f.lastCategories = categories
f.lastIncludeAvatars = includeAvatars
f.lastBefore = before
return f.purgedDocs, f.purgedPhotos, f.bytesReclaimed, f.purgeErr
}
func TestManualPurgeStorageDryRunReportsCounts(t *testing.T) {
ctx := context.Background()
repo := newMemoryCommandRepo()
storage := &fakeStorageService{countDocs: 3, countPhotos: 2}
svc := NewService(Dependencies{Commands: repo, Storage: storage, Now: fixedNow})
req := ManualPurgeStorageRequest{
CommandMeta: CommandMeta{CommandID: "purge-dry-1", Actor: "ops", Reason: "cleanup", DryRun: true},
Categories: []string{"video", "Music"},
}
result, err := svc.ManualPurgeStorage(ctx, req)
if err != nil {
t.Fatalf("ManualPurgeStorage(dry-run): %v", err)
}
if !result.DryRun {
t.Fatalf("result.DryRun = false, want true")
}
if result.Details["would_delete_documents"] != 3 || result.Details["would_delete_photos"] != 2 {
t.Fatalf("details = %+v, want would_delete_documents=3, would_delete_photos=2", result.Details)
}
if storage.countCalls != 1 || storage.purgeCalls != 0 {
t.Fatalf("countCalls=%d purgeCalls=%d, want 1/0 for a dry run", storage.countCalls, storage.purgeCalls)
}
want := map[domain.MediaCategory]bool{domain.MediaCategoryVideo: true, domain.MediaCategoryMusic: true}
if len(storage.lastCategories) != 2 || !want[storage.lastCategories[0]] || !want[storage.lastCategories[1]] {
t.Fatalf("lastCategories = %v, want [Video, Music] (case-insensitively parsed)", storage.lastCategories)
}
if storage.lastBefore != nil {
t.Fatalf("lastBefore = %v, want nil (no created_before supplied)", storage.lastBefore)
}
}
func TestManualPurgeStorageConfirmReportsPurgedCounts(t *testing.T) {
ctx := context.Background()
repo := newMemoryCommandRepo()
storage := &fakeStorageService{purgedDocs: 5, purgedPhotos: 1, bytesReclaimed: 4096}
svc := NewService(Dependencies{Commands: repo, Storage: storage, Now: fixedNow})
before := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
req := ManualPurgeStorageRequest{
CommandMeta: CommandMeta{CommandID: "purge-confirm-1", Actor: "ops", Reason: "cleanup"},
Categories: []string{"file"},
CreatedBefore: &before,
}
result, err := svc.ManualPurgeStorage(ctx, req)
if err != nil {
t.Fatalf("ManualPurgeStorage(confirm): %v", err)
}
if result.Details["purged_documents"] != 5 || result.Details["purged_photos"] != 1 || result.Details["bytes_reclaimed"] != int64(4096) {
t.Fatalf("details = %+v, want purged_documents=5, purged_photos=1, bytes_reclaimed=4096", result.Details)
}
if storage.purgeCalls != 1 || storage.countCalls != 0 {
t.Fatalf("purgeCalls=%d countCalls=%d, want 1/0 for a confirm", storage.purgeCalls, storage.countCalls)
}
if storage.lastBefore == nil || !storage.lastBefore.Equal(before) {
t.Fatalf("lastBefore = %v, want %v", storage.lastBefore, before)
}
}
// TestManualPurgeStorageAvatarIsNotAMediaCategory guards the "avatar" key's
// special handling: it must flip IncludeAvatars and must NEVER appear in the
// categories slice handed to the files.Service layer, since Avatar is not a
// domain.MediaCategory (see files.Service.ManualPurge's doc comment).
func TestManualPurgeStorageAvatarIsNotAMediaCategory(t *testing.T) {
ctx := context.Background()
repo := newMemoryCommandRepo()
storage := &fakeStorageService{}
svc := NewService(Dependencies{Commands: repo, Storage: storage, Now: fixedNow})
req := ManualPurgeStorageRequest{
CommandMeta: CommandMeta{CommandID: "purge-avatar-1", Actor: "ops", Reason: "cleanup", DryRun: true},
Categories: []string{"photo"},
IncludeAvatars: true,
}
if _, err := svc.ManualPurgeStorage(ctx, req); err != nil {
t.Fatalf("ManualPurgeStorage: %v", err)
}
if !storage.lastIncludeAvatars {
t.Fatal("lastIncludeAvatars = false, want true")
}
if len(storage.lastCategories) != 1 || storage.lastCategories[0] != domain.MediaCategoryPhoto {
t.Fatalf("lastCategories = %v, want exactly [Photo]", storage.lastCategories)
}
}
// TestManualPurgeStorageRejectsUnknownCategory guards the admin-boundary
// validation: an unknown category key must be rejected with a clear error,
// not silently dropped, and must never reach the underlying service.
func TestManualPurgeStorageRejectsUnknownCategory(t *testing.T) {
ctx := context.Background()
repo := newMemoryCommandRepo()
storage := &fakeStorageService{}
svc := NewService(Dependencies{Commands: repo, Storage: storage, Now: fixedNow})
req := ManualPurgeStorageRequest{
CommandMeta: CommandMeta{CommandID: "purge-bad-1", Actor: "ops", Reason: "cleanup", DryRun: true},
Categories: []string{"sticker"},
}
if _, err := svc.ManualPurgeStorage(ctx, req); err == nil {
t.Fatal("ManualPurgeStorage with an unknown category = nil error, want an error")
}
if storage.countCalls != 0 {
t.Fatalf("countCalls = %d, want 0 (validation must fail before reaching the service)", storage.countCalls)
}
}
// TestManualPurgeStorageRequiresACategoryOrAvatars guards against a request
// that would purge nothing at all (no categories, avatars not included)
// being silently accepted as a no-op success.
func TestManualPurgeStorageRequiresACategoryOrAvatars(t *testing.T) {
ctx := context.Background()
repo := newMemoryCommandRepo()
storage := &fakeStorageService{}
svc := NewService(Dependencies{Commands: repo, Storage: storage, Now: fixedNow})
req := ManualPurgeStorageRequest{
CommandMeta: CommandMeta{CommandID: "purge-empty-1", Actor: "ops", Reason: "cleanup", DryRun: true},
}
if _, err := svc.ManualPurgeStorage(ctx, req); err == nil {
t.Fatal("ManualPurgeStorage with no categories and no avatars = nil error, want an error")
}
}

View file

@ -63,6 +63,10 @@ const (
ActionAutoCategorizeGifCatalog = "gif_catalog.auto_categorize" ActionAutoCategorizeGifCatalog = "gif_catalog.auto_categorize"
ActionDeleteUncategorizedGifs = "gif_catalog.delete_uncategorized" ActionDeleteUncategorizedGifs = "gif_catalog.delete_uncategorized"
ActionDeleteGifCatalogEntry = "gif_catalog.delete" ActionDeleteGifCatalogEntry = "gif_catalog.delete"
// Manual storage purge: admin-chosen categories + optional age cutoff,
// independent of the automatic retention sweep's config-derived
// selection. See StorageService's doc comment.
ActionManualPurgeStorage = "storage.manual_purge"
// Collectible (Fragment-style) username lifecycle. // Collectible (Fragment-style) username lifecycle.
ActionMintCollectibleUsername = "usernames.collectible.mint" ActionMintCollectibleUsername = "usernames.collectible.mint"
ActionTransferCollectibleUsername = "usernames.collectible.transfer" ActionTransferCollectibleUsername = "usernames.collectible.transfer"
@ -337,6 +341,22 @@ type GifCatalogService interface {
AdminDeleteGifCatalogEntry(ctx context.Context, id int64) (bool, error) AdminDeleteGifCatalogEntry(ctx context.Context, id int64) (bool, error)
} }
// StorageService is the admin-console surface over manual storage purge --
// deleting media blob bytes (never the document/photo metadata row) by
// admin-chosen category and an optional age cutoff, independent of the
// automatic retention sweep's config-derived categories/age. See
// internal/app/files.Service.ManualPurge's doc comment for the exact
// deletion semantics (identical to "hard" retention mode).
type StorageService interface {
// CountManualPurgeCandidates is the dry-run preview: an exact count of
// how many documents/photos the selection would purge. before == nil
// means no age filter at all.
CountManualPurgeCandidates(ctx context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time) (docs int, photos int, err error)
// ManualPurge actually deletes the selected media's blob bytes, up to
// limit documents/photos per category/bucket, returning what was purged.
ManualPurge(ctx context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time, limit int) (purgedDocs int, purgedPhotos int, bytesReclaimed int64, err error)
}
// BotService creates bot accounts on behalf of the admin. It mirrors the // BotService creates bot accounts on behalf of the admin. It mirrors the
// owner-scoped /newbot flow: a bot is a users row (is_bot=true) plus a bots row // owner-scoped /newbot flow: a bot is a users row (is_bot=true) plus a bots row
// owned by ownerUserID, and the returned token is shown once to the operator. // owned by ownerUserID, and the returned token is shown once to the operator.
@ -402,6 +422,7 @@ type Dependencies struct {
Photos AvatarResolver Photos AvatarResolver
StickerSets StickerSetsService StickerSets StickerSetsService
GifCatalog GifCatalogService GifCatalog GifCatalogService
Storage StorageService
Bots BotService Bots BotService
Emoji EmojiService Emoji EmojiService
Moderation ModerationService Moderation ModerationService
@ -433,6 +454,7 @@ type Service struct {
photos AvatarResolver photos AvatarResolver
stickerSets StickerSetsService stickerSets StickerSetsService
gifCatalog GifCatalogService gifCatalog GifCatalogService
storage StorageService
bots BotService bots BotService
emoji EmojiService emoji EmojiService
moderation ModerationService moderation ModerationService
@ -492,6 +514,9 @@ func (s *Service) Configure(deps Dependencies) *Service {
if deps.GifCatalog != nil { if deps.GifCatalog != nil {
s.gifCatalog = deps.GifCatalog s.gifCatalog = deps.GifCatalog
} }
if deps.Storage != nil {
s.storage = deps.Storage
}
if deps.Bots != nil { if deps.Bots != nil {
s.bots = deps.Bots s.bots = deps.Bots
} }
@ -749,6 +774,22 @@ type DeleteGifCatalogEntryRequest struct {
ID int64 `json:"id"` ID int64 `json:"id"`
} }
// ManualPurgeStorageRequest is the manual storage purge action's request.
// Categories are the admin UI's category keys (see manualPurgeCategoryKeys)
// -- "photo", "video", "round_video", "gif", "music", "voice", "file".
// "avatar" is not a category key; it is the separate IncludeAvatars flag,
// since Avatar is not a domain.MediaCategory (see
// files.Service.ManualPurge's doc comment). CreatedBefore is an optional
// ISO-8601/RFC3339 cutoff -- nil (the field omitted or JSON null) means no
// age filter at all: every document/photo in the selected categories is
// purged, regardless of age.
type ManualPurgeStorageRequest struct {
CommandMeta
Categories []string `json:"categories"`
IncludeAvatars bool `json:"include_avatars,omitempty"`
CreatedBefore *time.Time `json:"created_before,omitempty"`
}
type SetAccountFrozenRequest struct { type SetAccountFrozenRequest struct {
CommandMeta CommandMeta
UserID int64 `json:"user_id"` UserID int64 `json:"user_id"`
@ -3034,6 +3075,88 @@ func (s *Service) DeleteUncategorizedGifs(ctx context.Context, req DeleteUncateg
}) })
} }
// manualPurgeCategoryKeys maps the admin UI's category keys (see
// CATEGORY_AGE_FIELDS in cmd/telesrv-admin/web/src/pages/StoragePage.tsx) to
// their domain.MediaCategory value. "avatar" is deliberately absent -- it
// maps to ManualPurgeStorageRequest.IncludeAvatars instead, since Avatar is
// not a domain.MediaCategory (see files.Service.ManualPurge's doc comment).
var manualPurgeCategoryKeys = map[string]domain.MediaCategory{
"photo": domain.MediaCategoryPhoto,
"video": domain.MediaCategoryVideo,
"round_video": domain.MediaCategoryRoundVideo,
"gif": domain.MediaCategoryGif,
"music": domain.MediaCategoryMusic,
"voice": domain.MediaCategoryVoice,
"file": domain.MediaCategoryFile,
}
// parseManualPurgeCategories resolves each admin-supplied category key
// (case-insensitive) to its domain.MediaCategory, rejecting an unknown key
// outright rather than silently ignoring it -- an operator who fat-fingers a
// category name deserves an error, not a purge that quietly did less than
// they asked for. Duplicate keys collapse to one entry.
func parseManualPurgeCategories(keys []string) ([]domain.MediaCategory, error) {
categories := make([]domain.MediaCategory, 0, len(keys))
seen := make(map[domain.MediaCategory]bool, len(keys))
for _, raw := range keys {
key := strings.ToLower(strings.TrimSpace(raw))
cat, ok := manualPurgeCategoryKeys[key]
if !ok {
return nil, fmt.Errorf("unknown storage category: %q", raw)
}
if seen[cat] {
continue
}
seen[cat] = true
categories = append(categories, cat)
}
return categories, nil
}
// manualPurgeBatchLimit bounds how many documents/photos a single
// ManualPurgeStorage confirm call purges per category/bucket. An operator
// whose dry-run count exceeds this runs the (idempotent, self-terminating --
// see files.Service.ManualPurge) confirm action again to keep purging.
const manualPurgeBatchLimit = 5000
// ManualPurgeStorage deletes media blob bytes (never the document/photo
// metadata row) by admin-chosen category and an optional "created before"
// cutoff -- the manual counterpart of the automatic hard-retention sweep.
// See StorageService and files.Service.ManualPurge's doc comments for the
// exact semantics.
func (s *Service) ManualPurgeStorage(ctx context.Context, req ManualPurgeStorageRequest) (CommandResult, error) {
if s == nil || s.storage == nil {
return CommandResult{}, fmt.Errorf("storage service is not configured")
}
categories, err := parseManualPurgeCategories(req.Categories)
if err != nil {
return CommandResult{}, err
}
if len(categories) == 0 && !req.IncludeAvatars {
return CommandResult{}, fmt.Errorf("at least one category (or avatars) must be selected")
}
return s.runCommand(ctx, req.CommandMeta, ActionManualPurgeStorage, 0, domain.Peer{}, req, func() (CommandResult, error) {
details := map[string]any{}
if req.DryRun {
// A real count, not just "validated" -- this is a bulk,
// irreversible delete, and an operator confirming it deserves to
// know how much they're about to purge before they do.
docs, photos, err := s.storage.CountManualPurgeCandidates(ctx, categories, req.IncludeAvatars, req.CreatedBefore)
if err != nil {
return CommandResult{}, err
}
details["would_delete_documents"] = docs
details["would_delete_photos"] = photos
return CommandResult{Message: fmt.Sprintf("would purge %d document(s) and %d photo(s)", docs, photos), Details: details}, nil
}
purgedDocs, purgedPhotos, bytesReclaimed, err := s.storage.ManualPurge(ctx, categories, req.IncludeAvatars, req.CreatedBefore, manualPurgeBatchLimit)
details["purged_documents"] = purgedDocs
details["purged_photos"] = purgedPhotos
details["bytes_reclaimed"] = bytesReclaimed
return CommandResult{Message: fmt.Sprintf("purged %d document(s) and %d photo(s)", purgedDocs, purgedPhotos), Details: details}, err
})
}
func (s *Service) DeleteGifCatalogEntry(ctx context.Context, req DeleteGifCatalogEntryRequest) (CommandResult, error) { func (s *Service) DeleteGifCatalogEntry(ctx context.Context, req DeleteGifCatalogEntryRequest) (CommandResult, error) {
if s == nil || s.gifCatalog == nil || req.ID <= 0 { if s == nil || s.gifCatalog == nil || req.ID <= 0 {
return CommandResult{}, domain.ErrGifCatalogEntryInvalid return CommandResult{}, domain.ErrGifCatalogEntryInvalid

View file

@ -83,6 +83,7 @@ type Service interface {
AutoCategorizeGifCatalog(ctx context.Context, req admin.AutoCategorizeGifCatalogRequest) (admin.CommandResult, error) AutoCategorizeGifCatalog(ctx context.Context, req admin.AutoCategorizeGifCatalogRequest) (admin.CommandResult, error)
DeleteUncategorizedGifs(ctx context.Context, req admin.DeleteUncategorizedGifsRequest) (admin.CommandResult, error) DeleteUncategorizedGifs(ctx context.Context, req admin.DeleteUncategorizedGifsRequest) (admin.CommandResult, error)
DeleteGifCatalogEntry(ctx context.Context, req admin.DeleteGifCatalogEntryRequest) (admin.CommandResult, error) DeleteGifCatalogEntry(ctx context.Context, req admin.DeleteGifCatalogEntryRequest) (admin.CommandResult, error)
ManualPurgeStorage(ctx context.Context, req admin.ManualPurgeStorageRequest) (admin.CommandResult, error)
EmojiAnimation(ctx context.Context, documentID int64) ([]byte, bool, error) EmojiAnimation(ctx context.Context, documentID int64) ([]byte, bool, error)
ModerationCases(ctx context.Context, filter domain.ModerationCaseFilter) ([]domain.ModerationCase, error) ModerationCases(ctx context.Context, filter domain.ModerationCaseFilter) ([]domain.ModerationCase, error)
ModerationCase(ctx context.Context, caseID int64) (domain.ModerationCaseDetail, bool, error) ModerationCase(ctx context.Context, caseID int64) (domain.ModerationCaseDetail, bool, error)
@ -218,6 +219,7 @@ func (s *Server) routes() http.Handler {
mux.HandleFunc("POST /v1/gif-catalog/auto-categorize", s.authenticated(s.handleAutoCategorizeGifCatalog)) mux.HandleFunc("POST /v1/gif-catalog/auto-categorize", s.authenticated(s.handleAutoCategorizeGifCatalog))
mux.HandleFunc("POST /v1/gif-catalog/delete-uncategorized", s.authenticated(s.handleDeleteUncategorizedGifs)) mux.HandleFunc("POST /v1/gif-catalog/delete-uncategorized", s.authenticated(s.handleDeleteUncategorizedGifs))
mux.HandleFunc("POST /v1/gif-catalog/delete", s.authenticated(s.handleDeleteGifCatalogEntry)) mux.HandleFunc("POST /v1/gif-catalog/delete", s.authenticated(s.handleDeleteGifCatalogEntry))
mux.HandleFunc("POST /v1/storage/manual-purge", s.authenticated(s.handleManualPurgeStorage))
mux.HandleFunc("GET /v1/stickers/documents/{id}/animation", s.authenticated(s.handleStickerDocumentAnimation)) mux.HandleFunc("GET /v1/stickers/documents/{id}/animation", s.authenticated(s.handleStickerDocumentAnimation))
mux.HandleFunc("GET /v1/gif-catalog/documents/{id}/preview", s.authenticated(s.handleGifCatalogDocumentPreview)) mux.HandleFunc("GET /v1/gif-catalog/documents/{id}/preview", s.authenticated(s.handleGifCatalogDocumentPreview))
mux.HandleFunc("GET /v1/emoji/{id}/animation", s.authenticated(s.handleEmojiAnimation)) mux.HandleFunc("GET /v1/emoji/{id}/animation", s.authenticated(s.handleEmojiAnimation))
@ -783,6 +785,15 @@ func (s *Server) handleDeleteUncategorizedGifs(w http.ResponseWriter, r *http.Re
writeCommandResult(w, result, err) writeCommandResult(w, result, err)
} }
func (s *Server) handleManualPurgeStorage(w http.ResponseWriter, r *http.Request) {
var req admin.ManualPurgeStorageRequest
if !decodeJSON(w, r, &req) {
return
}
result, err := s.svc.ManualPurgeStorage(r.Context(), req)
writeCommandResult(w, result, err)
}
func (s *Server) handleDeleteGifCatalogEntry(w http.ResponseWriter, r *http.Request) { func (s *Server) handleDeleteGifCatalogEntry(w http.ResponseWriter, r *http.Request) {
var req admin.DeleteGifCatalogEntryRequest var req admin.DeleteGifCatalogEntryRequest
if !decodeJSON(w, r, &req) { if !decodeJSON(w, r, &req) {

View file

@ -476,6 +476,10 @@ func (fakeService) DeleteGifCatalogEntry(_ context.Context, req admin.DeleteGifC
return admin.CommandResult{CommandID: req.CommandID, Status: "completed", DryRun: req.DryRun}, nil return admin.CommandResult{CommandID: req.CommandID, Status: "completed", DryRun: req.DryRun}, nil
} }
func (fakeService) ManualPurgeStorage(_ context.Context, req admin.ManualPurgeStorageRequest) (admin.CommandResult, error) {
return admin.CommandResult{CommandID: req.CommandID, Status: "completed", DryRun: req.DryRun}, nil
}
func (fakeService) EmojiAnimation(context.Context, int64) ([]byte, bool, error) { func (fakeService) EmojiAnimation(context.Context, int64) ([]byte, bool, error) {
return []byte(`{"v":"5.7","w":100,"h":100}`), true, nil return []byte(`{"v":"5.7","w":100,"h":100}`), true, nil
} }

View file

@ -69,10 +69,19 @@ type mediaRetentionStore interface {
// photo still referenced by a live message is exactly as eligible as an // photo still referenced by a live message is exactly as eligible as an
// orphaned one once it's old enough. The delete methods physically // orphaned one once it's old enough. The delete methods physically
// remove only the file_blobs row(s)/bytes, never the document/photo // remove only the file_blobs row(s)/bytes, never the document/photo
// metadata row -- see DeleteFileBlobsForDocument's doc comment. // metadata row -- see DeleteFileBlobsForDocument's doc comment. cutoff is
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error) // a *time.Time so it can be nil ("no age filter at all") -- the automatic
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) // sweep below always passes a real cutoff; the manual purge admin action
ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) // (see ManualPurge) is what actually uses nil.
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff *time.Time, limit int) ([]int64, error)
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff *time.Time, limit int) ([]int64, error)
ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff *time.Time, limit int) ([]int64, error)
// Count* are the exact-count counterparts of the List* queries just
// above, used by CountManualPurgeCandidates's dry-run preview: a
// LIMIT-capped list length is not a real total.
CountDocumentsForHardRetention(ctx context.Context, category domain.MediaCategory, cutoff *time.Time) (int, error)
CountPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error)
CountAvatarPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error)
DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error) DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error) DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)
@ -193,7 +202,7 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time
continue continue
} }
cutoff := now.Add(-age) cutoff := now.Add(-age)
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, cutoff, limit) docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, &cutoff, limit)
if err != nil { if err != nil {
return purged, fmt.Errorf("list documents for hard retention (category %d): %w", cat, err) return purged, fmt.Errorf("list documents for hard retention (category %d): %w", cat, err)
} }
@ -213,7 +222,7 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time
} }
if age := s.categoryRetentionAge(domain.MediaCategoryPhoto); age > 0 { if age := s.categoryRetentionAge(domain.MediaCategoryPhoto); age > 0 {
photoCutoff := now.Add(-age) photoCutoff := now.Add(-age)
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, photoCutoff, limit) photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, &photoCutoff, limit)
if err != nil { if err != nil {
return purged, fmt.Errorf("list photos for hard retention: %w", err) return purged, fmt.Errorf("list photos for hard retention: %w", err)
} }
@ -233,7 +242,7 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time
} }
if age := s.avatarRetentionAge(); age > 0 { if age := s.avatarRetentionAge(); age > 0 {
avatarCutoff := now.Add(-age) avatarCutoff := now.Add(-age)
avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, avatarCutoff, limit) avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, &avatarCutoff, limit)
if err != nil { if err != nil {
return purged, fmt.Errorf("list avatar photos for hard retention: %w", err) return purged, fmt.Errorf("list avatar photos for hard retention: %w", err)
} }
@ -254,6 +263,169 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time
return purged, nil return purged, nil
} }
// manualPurgeCategories enumerates the domain.MediaCategory values the
// manual purge admin action (ManualPurge/CountManualPurgeCandidates) accepts.
// Unlike hardRetentionDocumentCategories (used by the automatic sweep),
// this includes MediaCategoryPhoto -- an operator explicitly choosing
// categories to purge expects "Photo" to be one of the choices, even though
// photos have no documents.category column and are resolved through the
// dedicated ListPhotoIDsForHardRetentionOlderThan query instead. Avatar is
// deliberately NOT a domain.MediaCategory member and is instead its own
// includeAvatars bool parameter, matching how the automatic sweep already
// splits avatar photos out via avatarRetentionAge/ListAvatarPhotoIDsForHardRetentionOlderThan.
var manualPurgeCategories = map[domain.MediaCategory]bool{
domain.MediaCategoryPhoto: true,
domain.MediaCategoryVideo: true,
domain.MediaCategoryRoundVideo: true,
domain.MediaCategoryGif: true,
domain.MediaCategoryMusic: true,
domain.MediaCategoryVoice: true,
domain.MediaCategoryFile: true,
}
// validateManualPurgeCategories rejects any category outside
// manualPurgeCategories (e.g. MediaCategoryNone or MediaCategoryURL, which
// are not meaningful purge targets) rather than silently ignoring it -- an
// operator who fat-fingers a category deserves an error, not a purge that
// quietly did less than they asked for.
func validateManualPurgeCategories(categories []domain.MediaCategory) error {
for _, cat := range categories {
if !manualPurgeCategories[cat] {
return fmt.Errorf("unsupported manual purge category: %d", cat)
}
}
return nil
}
// CountManualPurgeCandidates is the dry-run counterpart of ManualPurge: an
// exact count of how many documents/photos the given selection would purge,
// without deleting anything. before may be nil, meaning no age filter at all
// (every document/photo in the selected categories is a candidate,
// regardless of created_at) -- unlike the automatic sweep, which always
// filters by a configured age.
func (s *Service) CountManualPurgeCandidates(ctx context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time) (docs int, photos int, err error) {
store, ok := s.media.(mediaRetentionStore)
if !ok {
return 0, 0, nil
}
if err := validateManualPurgeCategories(categories); err != nil {
return 0, 0, err
}
for _, cat := range categories {
if cat == domain.MediaCategoryPhoto {
n, err := store.CountPhotosForHardRetention(ctx, before)
if err != nil {
return docs, photos, fmt.Errorf("count photos for manual purge: %w", err)
}
photos += n
continue
}
n, err := store.CountDocumentsForHardRetention(ctx, cat, before)
if err != nil {
return docs, photos, fmt.Errorf("count documents for manual purge (category %d): %w", cat, err)
}
docs += n
}
if includeAvatars {
n, err := store.CountAvatarPhotosForHardRetention(ctx, before)
if err != nil {
return docs, photos, fmt.Errorf("count avatar photos for manual purge: %w", err)
}
photos += n
}
return docs, photos, nil
}
// ManualPurge is the admin-triggered counterpart of DeleteBlobBytesForMediaOlderThan
// ("hard" retention mode's blob-purge primitive): instead of config-derived
// categories and a configured retention age, the operator explicitly chooses
// which categories to purge and an optional cutoff date. before == nil means
// no age filter at all -- everything matching the chosen categories is
// purged, regardless of how recently it was created. Deletion semantics are
// identical to the automatic sweep: only file_blobs bytes are removed, never
// the documents/photos metadata row, and notifyRetentionPurge still turns any
// message still displaying the purged media into the retention-purge notice.
// limit bounds how many documents/photos are purged per category/bucket in
// this single call (the admin action loops/paginates by calling again if the
// dry-run count exceeds one call's limit).
func (s *Service) ManualPurge(ctx context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time, limit int) (purgedDocs int, purgedPhotos int, bytesReclaimed int64, err error) {
store, ok := s.media.(mediaRetentionStore)
if !ok || limit <= 0 {
return 0, 0, 0, nil
}
if err := validateManualPurgeCategories(categories); err != nil {
return 0, 0, 0, err
}
for _, cat := range categories {
if cat == domain.MediaCategoryPhoto {
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, before, limit)
if err != nil {
return purgedDocs, purgedPhotos, bytesReclaimed, fmt.Errorf("list photos for manual purge: %w", err)
}
for _, id := range photoIDs {
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
if err != nil {
s.log.Warn("manual purge photo blob delete failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
for _, b := range blobs {
bytesReclaimed += b.Size
}
s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
purgedPhotos++
}
continue
}
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, before, limit)
if err != nil {
return purgedDocs, purgedPhotos, bytesReclaimed, fmt.Errorf("list documents for manual purge (category %d): %w", cat, err)
}
for _, id := range docIDs {
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
if err != nil {
s.log.Warn("manual purge document blob delete failed", zap.Int64("document_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
for _, b := range blobs {
bytesReclaimed += b.Size
}
s.notifyRetentionPurge(ctx, domain.MediaKindDocument, id)
purgedDocs++
}
}
if includeAvatars {
avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, before, limit)
if err != nil {
return purgedDocs, purgedPhotos, bytesReclaimed, fmt.Errorf("list avatar photos for manual purge: %w", err)
}
for _, id := range avatarIDs {
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
if err != nil {
s.log.Warn("manual purge avatar photo blob delete failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
for _, b := range blobs {
bytesReclaimed += b.Size
}
s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
purgedPhotos++
}
}
return purgedDocs, purgedPhotos, bytesReclaimed, nil
}
// EvictOldestMediaOverBudget implements maintenance.StorageEvictionStore // EvictOldestMediaOverBudget implements maintenance.StorageEvictionStore
// (TELESRV_STORAGE_EVICTION_ENABLE): once total physical blob bytes // (TELESRV_STORAGE_EVICTION_ENABLE): once total physical blob bytes
// (SumFileBlobBytes) exceed TELESRV_STORAGE_MAX_TOTAL_BYTES, purges the // (SumFileBlobBytes) exceed TELESRV_STORAGE_MAX_TOTAL_BYTES, purges the

View file

@ -24,21 +24,31 @@ type fakeCategorySweepStore struct {
queriedAvatars bool queriedAvatars bool
} }
func (f *fakeCategorySweepStore) ListDocumentIDsForHardRetentionOlderThan(_ context.Context, category domain.MediaCategory, _ time.Time, _ int) ([]int64, error) { func (f *fakeCategorySweepStore) ListDocumentIDsForHardRetentionOlderThan(_ context.Context, category domain.MediaCategory, _ *time.Time, _ int) ([]int64, error) {
f.queriedDocCategories = append(f.queriedDocCategories, category) f.queriedDocCategories = append(f.queriedDocCategories, category)
return nil, nil return nil, nil
} }
func (f *fakeCategorySweepStore) ListPhotoIDsForHardRetentionOlderThan(context.Context, time.Time, int) ([]int64, error) { func (f *fakeCategorySweepStore) ListPhotoIDsForHardRetentionOlderThan(context.Context, *time.Time, int) ([]int64, error) {
f.queriedPhotos = true f.queriedPhotos = true
return nil, nil return nil, nil
} }
func (f *fakeCategorySweepStore) ListAvatarPhotoIDsForHardRetentionOlderThan(context.Context, time.Time, int) ([]int64, error) { func (f *fakeCategorySweepStore) ListAvatarPhotoIDsForHardRetentionOlderThan(context.Context, *time.Time, int) ([]int64, error) {
f.queriedAvatars = true f.queriedAvatars = true
return nil, nil return nil, nil
} }
func (f *fakeCategorySweepStore) CountDocumentsForHardRetention(context.Context, domain.MediaCategory, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeCategorySweepStore) CountPhotosForHardRetention(context.Context, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeCategorySweepStore) CountAvatarPhotosForHardRetention(context.Context, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeCategorySweepStore) ListOrphanedDocumentIDsOlderThan(_ context.Context, category domain.MediaCategory, _ time.Time, _ int) ([]int64, error) { func (f *fakeCategorySweepStore) ListOrphanedDocumentIDsOlderThan(_ context.Context, category domain.MediaCategory, _ time.Time, _ int) ([]int64, error) {
f.queriedDocCategories = append(f.queriedDocCategories, category) f.queriedDocCategories = append(f.queriedDocCategories, category)
return nil, nil return nil, nil

View file

@ -0,0 +1,314 @@
package files
import (
"context"
"fmt"
"testing"
"time"
"go.uber.org/zap"
"telesrv/internal/domain"
"telesrv/internal/store"
)
// fakeManualPurgeStore is a mediaRetentionStore fake that mimics the SQL
// nullable-cutoff contract in Go: a nil cutoff matches every candidate
// regardless of its createdAt, a non-nil cutoff only matches candidates
// strictly older than it -- same semantics as the real
// ListDocumentIDsForHardRetentionOlderThan/ListPhotoIDsForHardRetentionOlderThan
// queries (see internal/store/postgres/queries/media.sql). Embeds a nil
// store.MediaStore so it satisfies that (large) interface for free, same
// trick as fakeCategorySweepStore in retention_category_zero_test.go.
type fakeManualPurgeStore struct {
store.MediaStore
docsByCategory map[domain.MediaCategory][]int64
docCreatedAt map[int64]time.Time
photos []int64
photoCreatedAt map[int64]time.Time
avatarPhotos []int64
avatarCreated map[int64]time.Time
deletedDocs []int64
deletedPhotos []int64
}
func matchCutoff(createdAt time.Time, cutoff *time.Time) bool {
if cutoff == nil {
return true
}
return createdAt.Before(*cutoff)
}
func (f *fakeManualPurgeStore) ListDocumentIDsForHardRetentionOlderThan(_ context.Context, category domain.MediaCategory, cutoff *time.Time, limit int) ([]int64, error) {
var out []int64
for _, id := range f.docsByCategory[category] {
if !matchCutoff(f.docCreatedAt[id], cutoff) {
continue
}
out = append(out, id)
if len(out) >= limit {
break
}
}
return out, nil
}
func (f *fakeManualPurgeStore) CountDocumentsForHardRetention(ctx context.Context, category domain.MediaCategory, cutoff *time.Time) (int, error) {
ids, err := f.ListDocumentIDsForHardRetentionOlderThan(ctx, category, cutoff, len(f.docsByCategory[category])+1)
return len(ids), err
}
func (f *fakeManualPurgeStore) ListPhotoIDsForHardRetentionOlderThan(_ context.Context, cutoff *time.Time, limit int) ([]int64, error) {
var out []int64
for _, id := range f.photos {
if !matchCutoff(f.photoCreatedAt[id], cutoff) {
continue
}
out = append(out, id)
if len(out) >= limit {
break
}
}
return out, nil
}
func (f *fakeManualPurgeStore) CountPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error) {
ids, err := f.ListPhotoIDsForHardRetentionOlderThan(ctx, cutoff, len(f.photos)+1)
return len(ids), err
}
func (f *fakeManualPurgeStore) ListAvatarPhotoIDsForHardRetentionOlderThan(_ context.Context, cutoff *time.Time, limit int) ([]int64, error) {
var out []int64
for _, id := range f.avatarPhotos {
if !matchCutoff(f.avatarCreated[id], cutoff) {
continue
}
out = append(out, id)
if len(out) >= limit {
break
}
}
return out, nil
}
func (f *fakeManualPurgeStore) CountAvatarPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error) {
ids, err := f.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, cutoff, len(f.avatarPhotos)+1)
return len(ids), err
}
func (f *fakeManualPurgeStore) DeleteFileBlobsForDocument(_ context.Context, id int64) ([]domain.FileBlob, error) {
f.deletedDocs = append(f.deletedDocs, id)
return []domain.FileBlob{{
LocationKey: fmt.Sprintf("doc:%d", id), Backend: domain.MediaBackendLocalFS,
ObjectKey: fmt.Sprintf("obj-doc-%d", id), Size: 100,
}}, nil
}
func (f *fakeManualPurgeStore) DeleteFileBlobsForPhoto(_ context.Context, id int64) ([]domain.FileBlob, error) {
f.deletedPhotos = append(f.deletedPhotos, id)
return []domain.FileBlob{{
LocationKey: fmt.Sprintf("photo:%d", id), Backend: domain.MediaBackendLocalFS,
ObjectKey: fmt.Sprintf("obj-photo-%d", id), Size: 50,
}}, nil
}
// The rest of mediaRetentionStore isn't exercised by ManualPurge/
// CountManualPurgeCandidates -- stub it out, same as
// fakeCategorySweepStore.
func (f *fakeManualPurgeStore) ListOrphanedDocumentIDsOlderThan(context.Context, domain.MediaCategory, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) ListOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) ListAvatarOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
return nil, nil
}
// CountFileBlobRefs deliberately returns 1 (not 0): ManualPurge's deleted
// blobs must still be accounted for (deleteOrphanedBlobs's cache eviction),
// but this fake has no real BlobBackend wired up, so it reports the blob as
// still referenced elsewhere -- skipping the physical backend.Delete call
// that would otherwise need a working s.blobs.
func (f *fakeManualPurgeStore) CountFileBlobRefs(context.Context, string, string) (int, error) {
return 1, nil
}
func (f *fakeManualPurgeStore) DeleteDocumentAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) DeletePhotoAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) OrphanDocumentIfUnreferenced(context.Context, int64) (bool, error) {
return false, nil
}
func (f *fakeManualPurgeStore) SumFileBlobBytes(context.Context) (int64, error) { return 0, nil }
func (f *fakeManualPurgeStore) ListOldestMediaForEviction(context.Context, int) ([]domain.EvictionCandidate, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) ListMediaReferences(context.Context, domain.MediaKind, int64) ([]domain.MediaReference, error) {
return nil, nil
}
func newManualPurgeTestService(fake *fakeManualPurgeStore) *Service {
return &Service{
media: fake,
log: zap.NewNop(),
blobCache: newBlobMetaCache(64),
byteCache: newBlobBytesCache(1 << 20),
}
}
// TestManualPurgeOnlyTouchesSelectedCategory guards ManualPurge's per-category
// scoping: purging just "Video" must never touch a Music document sitting in
// the same fake store, mirroring the real query's category filter (see
// TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory in
// internal/store/postgres).
func TestManualPurgeOnlyTouchesSelectedCategory(t *testing.T) {
fake := &fakeManualPurgeStore{
docsByCategory: map[domain.MediaCategory][]int64{
domain.MediaCategoryVideo: {1},
domain.MediaCategoryMusic: {2},
},
docCreatedAt: map[int64]time.Time{
1: time.Now().Add(-100 * 24 * time.Hour),
2: time.Now().Add(-100 * 24 * time.Hour),
},
}
s := newManualPurgeTestService(fake)
purgedDocs, purgedPhotos, bytes, err := s.ManualPurge(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryVideo}, false, nil, 100)
if err != nil {
t.Fatalf("ManualPurge: %v", err)
}
if purgedDocs != 1 || purgedPhotos != 0 {
t.Fatalf("purgedDocs=%d purgedPhotos=%d, want 1/0", purgedDocs, purgedPhotos)
}
if bytes != 100 {
t.Fatalf("bytesReclaimed = %d, want 100", bytes)
}
if len(fake.deletedDocs) != 1 || fake.deletedDocs[0] != 1 {
t.Fatalf("deletedDocs = %v, want only [1] (the Video document)", fake.deletedDocs)
}
}
// TestManualPurgeNilBeforeIgnoresAge is the unit-level counterpart of the
// Postgres integration test TestManualPurgeNilCutoffIgnoresAge: a document
// created "just now" must still be a purge candidate when before is nil,
// unlike the automatic sweep which always filters by a configured age.
func TestManualPurgeNilBeforeIgnoresAge(t *testing.T) {
fake := &fakeManualPurgeStore{
docsByCategory: map[domain.MediaCategory][]int64{
domain.MediaCategoryFile: {42},
},
docCreatedAt: map[int64]time.Time{
42: time.Now(), // created just now
},
}
s := newManualPurgeTestService(fake)
// Dry-run: nil before must count the fresh document.
docs, photos, err := s.CountManualPurgeCandidates(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryFile}, false, nil)
if err != nil {
t.Fatalf("CountManualPurgeCandidates(nil before): %v", err)
}
if docs != 1 || photos != 0 {
t.Fatalf("CountManualPurgeCandidates(nil before) = (%d, %d), want (1, 0)", docs, photos)
}
// A real cutoff strictly before "now" must exclude the fresh document.
past := time.Now().Add(-time.Hour)
docs, photos, err = s.CountManualPurgeCandidates(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryFile}, false, &past)
if err != nil {
t.Fatalf("CountManualPurgeCandidates(past before): %v", err)
}
if docs != 0 || photos != 0 {
t.Fatalf("CountManualPurgeCandidates(past before) = (%d, %d), want (0, 0)", docs, photos)
}
// Confirm: nil before must actually purge the fresh document.
purgedDocs, _, _, err := s.ManualPurge(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryFile}, false, nil, 100)
if err != nil {
t.Fatalf("ManualPurge(nil before): %v", err)
}
if purgedDocs != 1 {
t.Fatalf("ManualPurge(nil before) purgedDocs = %d, want 1", purgedDocs)
}
}
// TestManualPurgePhotoCategoryUsesPhotoQuery guards that selecting the
// "Photo" category (unlike the other categories, which are documents.category
// buckets) routes through the dedicated photo query rather than being
// silently dropped or misrouted to the document path.
func TestManualPurgePhotoCategoryUsesPhotoQuery(t *testing.T) {
fake := &fakeManualPurgeStore{
photos: []int64{7},
photoCreatedAt: map[int64]time.Time{7: time.Now()},
}
s := newManualPurgeTestService(fake)
purgedDocs, purgedPhotos, _, err := s.ManualPurge(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryPhoto}, false, nil, 100)
if err != nil {
t.Fatalf("ManualPurge: %v", err)
}
if purgedDocs != 0 || purgedPhotos != 1 {
t.Fatalf("purgedDocs=%d purgedPhotos=%d, want 0/1", purgedDocs, purgedPhotos)
}
if len(fake.deletedPhotos) != 1 || fake.deletedPhotos[0] != 7 {
t.Fatalf("deletedPhotos = %v, want only [7]", fake.deletedPhotos)
}
}
// TestManualPurgeIncludeAvatarsIsIndependentOfCategories guards that
// includeAvatars purges avatar photos even when Photo is not among the
// selected categories (and vice versa: selecting Photo alone must not also
// sweep avatars) -- the same split the automatic sweep already has between
// categoryRetentionAge(Photo) and avatarRetentionAge().
func TestManualPurgeIncludeAvatarsIsIndependentOfCategories(t *testing.T) {
fake := &fakeManualPurgeStore{
avatarPhotos: []int64{9},
avatarCreated: map[int64]time.Time{9: time.Now()},
}
s := newManualPurgeTestService(fake)
// No categories selected, but includeAvatars=true must still purge it.
purgedDocs, purgedPhotos, _, err := s.ManualPurge(context.Background(), nil, true, nil, 100)
if err != nil {
t.Fatalf("ManualPurge: %v", err)
}
if purgedDocs != 0 || purgedPhotos != 1 {
t.Fatalf("purgedDocs=%d purgedPhotos=%d, want 0/1", purgedDocs, purgedPhotos)
}
if len(fake.deletedPhotos) != 1 || fake.deletedPhotos[0] != 9 {
t.Fatalf("deletedPhotos = %v, want only [9]", fake.deletedPhotos)
}
}
// TestManualPurgeRejectsUnknownCategory guards validateManualPurgeCategories:
// an operator who fat-fingers a category should get an error, not a purge
// that quietly did less than they asked for.
func TestManualPurgeRejectsUnknownCategory(t *testing.T) {
fake := &fakeManualPurgeStore{}
s := newManualPurgeTestService(fake)
invalid := domain.MediaCategory(99)
if _, _, _, err := s.ManualPurge(context.Background(), []domain.MediaCategory{invalid}, false, nil, 100); err == nil {
t.Fatal("ManualPurge with an unsupported category = nil error, want an error")
}
if _, _, err := s.CountManualPurgeCandidates(context.Background(), []domain.MediaCategory{invalid}, false, nil); err == nil {
t.Fatal("CountManualPurgeCandidates with an unsupported category = nil error, want an error")
}
// MediaCategoryNone is a real category, but not one ManualPurge accepts
// (see manualPurgeCategories's doc comment) -- it must be rejected too,
// not silently treated as "match nothing"/"match everything".
if _, _, _, err := s.ManualPurge(context.Background(), []domain.MediaCategory{domain.MediaCategoryNone}, false, nil, 100); err == nil {
t.Fatal("ManualPurge with MediaCategoryNone = nil error, want an error")
}
}

View file

@ -0,0 +1,52 @@
package files
import (
"context"
"fmt"
"go.uber.org/zap"
"telesrv/internal/domain"
)
// DeleteEncryptedFileBlob deletes a secret-chat encrypted file's blob bytes
// (location_key "enc:<id>") once the recipient has fully downloaded it --
// see internal/rpc/upload.go's onUploadGetFile, which calls this after a
// getFile response whose bytes reach the end of the file. Secret chats are
// single-device on both ends (no multi-device sync, unlike ordinary chats),
// so once the one recipient device that will ever ask for this ciphertext
// has it, the server has no further reason to keep it -- unlike ordinary
// message media, there is no "another device might still need to fetch
// this" case to protect against.
//
// No-op unless TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD is enabled
// (WithSecretChatDeleteFileAfterDownload, default true). Best-effort: the
// caller is a fire-and-forget goroutine off the download response, so
// errors are only logged, never propagated -- a failed cleanup here must
// never turn into a failed or delayed file download.
func (s *Service) DeleteEncryptedFileBlob(ctx context.Context, locationKey string) error {
if !s.secretChatDeleteFileAfterDownload {
return nil
}
store, ok := s.media.(mediaRetentionStore)
if !ok {
return nil
}
blob, found, err := s.media.GetFileBlob(ctx, locationKey)
if err != nil {
return fmt.Errorf("get encrypted file blob: %w", err)
}
if !found {
// Already deleted (e.g. a racing duplicate getFile request for the
// same final chunk) -- nothing left to do.
return nil
}
if err := s.media.DeleteFileBlobRow(ctx, locationKey); err != nil {
return fmt.Errorf("delete encrypted file blob row: %w", err)
}
s.blobCache.delete(locationKey)
s.deleteOrphanedBlobs(ctx, store, []domain.FileBlob{blob})
s.log.Info("secret chat file deleted after download",
zap.String("location_key", locationKey), zap.Int64("size", blob.Size))
return nil
}

View file

@ -0,0 +1,185 @@
package files
import (
"context"
"errors"
"testing"
"time"
"go.uber.org/zap"
"telesrv/internal/domain"
"telesrv/internal/store"
)
// fakeEncryptedBlobStore is a minimal stateful store.MediaStore double for
// DeleteEncryptedFileBlob: a real in-memory map of location_key -> FileBlob
// (so GetFileBlob/DeleteFileBlobRow actually observe each other's effects).
// CountFileBlobRefs always reports "still referenced" so deleteOrphanedBlobs
// never reaches its physical-backend-delete path -- this test's Service has
// no real blob backend configured, and the point here is the file_blobs row
// deletion, which DeleteFileBlobRow already covers directly. Embeds a nil
// store.MediaStore for the rest of that large interface's methods, which
// this test never calls.
type fakeEncryptedBlobStore struct {
store.MediaStore
blobs map[string]domain.FileBlob
deletedRows []string
countRefsErr error
getFileBlobErr error
}
func (f *fakeEncryptedBlobStore) GetFileBlob(_ context.Context, key string) (domain.FileBlob, bool, error) {
if f.getFileBlobErr != nil {
return domain.FileBlob{}, false, f.getFileBlobErr
}
b, ok := f.blobs[key]
return b, ok, nil
}
func (f *fakeEncryptedBlobStore) DeleteFileBlobRow(_ context.Context, key string) error {
f.deletedRows = append(f.deletedRows, key)
delete(f.blobs, key)
return nil
}
func (f *fakeEncryptedBlobStore) CountFileBlobRefs(context.Context, string, string) (int, error) {
if f.countRefsErr != nil {
return 0, f.countRefsErr
}
// Reported as still-referenced (>0) so deleteOrphanedBlobs's physical
// backend-delete path (this test's Service has no real backend
// configured) is never reached -- this test only cares about the
// file_blobs row itself, which DeleteFileBlobRow already removed by the
// time deleteOrphanedBlobs runs.
return 1, nil
}
// The rest of mediaRetentionStore's methods aren't part of store.MediaStore
// (kept out of the hot RPC-facing interface on purpose -- see that
// interface's own doc comment), so embedding store.MediaStore alone doesn't
// satisfy the type assertion DeleteEncryptedFileBlob performs. Stub them
// out; this test never exercises them.
func (f *fakeEncryptedBlobStore) ListOrphanedDocumentIDsOlderThan(context.Context, domain.MediaCategory, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListAvatarOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) DeleteDocumentAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) DeletePhotoAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) OrphanDocumentIfUnreferenced(context.Context, int64) (bool, error) {
return false, nil
}
func (f *fakeEncryptedBlobStore) ListDocumentIDsForHardRetentionOlderThan(context.Context, domain.MediaCategory, *time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListPhotoIDsForHardRetentionOlderThan(context.Context, *time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListAvatarPhotoIDsForHardRetentionOlderThan(context.Context, *time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) CountDocumentsForHardRetention(context.Context, domain.MediaCategory, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeEncryptedBlobStore) CountPhotosForHardRetention(context.Context, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeEncryptedBlobStore) CountAvatarPhotosForHardRetention(context.Context, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeEncryptedBlobStore) DeleteFileBlobsForDocument(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) DeleteFileBlobsForPhoto(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) SumFileBlobBytes(context.Context) (int64, error) {
return 0, nil
}
func (f *fakeEncryptedBlobStore) ListOldestMediaForEviction(context.Context, int) ([]domain.EvictionCandidate, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListMediaReferences(context.Context, domain.MediaKind, int64) ([]domain.MediaReference, error) {
return nil, nil
}
func newTestServiceForSecretChatFiles(media store.MediaStore, enabled bool) *Service {
return &Service{
media: media,
log: zap.NewNop(),
blobCache: newBlobMetaCache(8),
secretChatDeleteFileAfterDownload: enabled,
}
}
func TestDeleteEncryptedFileBlobNoOpWhenDisabled(t *testing.T) {
fake := &fakeEncryptedBlobStore{blobs: map[string]domain.FileBlob{
"enc:1": {LocationKey: "enc:1", Backend: domain.MediaBackendS3, ObjectKey: "obj-1", Size: 42},
}}
s := newTestServiceForSecretChatFiles(fake, false)
if err := s.DeleteEncryptedFileBlob(context.Background(), "enc:1"); err != nil {
t.Fatalf("DeleteEncryptedFileBlob: %v", err)
}
if len(fake.deletedRows) != 0 {
t.Fatalf("deleted rows = %v, want none (feature disabled)", fake.deletedRows)
}
if _, ok := fake.blobs["enc:1"]; !ok {
t.Fatal("blob was removed from the store despite the feature being disabled")
}
}
func TestDeleteEncryptedFileBlobDeletesWhenEnabled(t *testing.T) {
fake := &fakeEncryptedBlobStore{blobs: map[string]domain.FileBlob{
"enc:1": {LocationKey: "enc:1", Backend: domain.MediaBackendS3, ObjectKey: "obj-1", Size: 42},
}}
s := newTestServiceForSecretChatFiles(fake, true)
if err := s.DeleteEncryptedFileBlob(context.Background(), "enc:1"); err != nil {
t.Fatalf("DeleteEncryptedFileBlob: %v", err)
}
if len(fake.deletedRows) != 1 || fake.deletedRows[0] != "enc:1" {
t.Fatalf("deleted rows = %v, want [enc:1]", fake.deletedRows)
}
if _, ok := fake.blobs["enc:1"]; ok {
t.Fatal("blob row still present after DeleteEncryptedFileBlob")
}
}
func TestDeleteEncryptedFileBlobNoOpWhenAlreadyGone(t *testing.T) {
fake := &fakeEncryptedBlobStore{blobs: map[string]domain.FileBlob{}}
s := newTestServiceForSecretChatFiles(fake, true)
// A racing duplicate final-chunk request for the same file: the blob is
// already deleted. Must not error.
if err := s.DeleteEncryptedFileBlob(context.Background(), "enc:404"); err != nil {
t.Fatalf("DeleteEncryptedFileBlob for a missing blob: %v", err)
}
if len(fake.deletedRows) != 0 {
t.Fatalf("deleted rows = %v, want none", fake.deletedRows)
}
}
func TestDeleteEncryptedFileBlobPropagatesLookupError(t *testing.T) {
fake := &fakeEncryptedBlobStore{
blobs: map[string]domain.FileBlob{"enc:1": {LocationKey: "enc:1"}},
getFileBlobErr: errors.New("db unavailable"),
}
s := newTestServiceForSecretChatFiles(fake, true)
if err := s.DeleteEncryptedFileBlob(context.Background(), "enc:1"); err == nil {
t.Fatal("DeleteEncryptedFileBlob = nil error, want the underlying lookup failure surfaced")
}
if len(fake.deletedRows) != 0 {
t.Fatalf("deleted rows = %v, want none (lookup failed before any delete)", fake.deletedRows)
}
}

View file

@ -157,6 +157,13 @@ func (f *fakeMediaStore) GetFileBlob(_ context.Context, key string) (domain.File
return b, ok, nil return b, ok, nil
} }
func (f *fakeMediaStore) DeleteFileBlobRow(_ context.Context, key string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.blobs, key)
return nil
}
func (f *fakeMediaStore) GetFileBlobs(_ context.Context, keys []string) (map[string]domain.FileBlob, error) { func (f *fakeMediaStore) GetFileBlobs(_ context.Context, keys []string) (map[string]domain.FileBlob, error) {
f.mu.Lock() f.mu.Lock()
defer f.mu.Unlock() defer f.mu.Unlock()

View file

@ -112,6 +112,10 @@ type Service struct {
// EvictOldestMediaOverBudget as the active-eviction trigger threshold // EvictOldestMediaOverBudget as the active-eviction trigger threshold
// (<=0 disables eviction regardless of TELESRV_STORAGE_EVICTION_ENABLE). // (<=0 disables eviction regardless of TELESRV_STORAGE_EVICTION_ENABLE).
storageMaxTotalBytes int64 storageMaxTotalBytes int64
// secretChatDeleteFileAfterDownload is
// TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD -- see
// DeleteEncryptedFileBlob's doc comment.
secretChatDeleteFileAfterDownload bool
// retentionNotifyMu guards retentionMessages/retentionChannels: they are // retentionNotifyMu guards retentionMessages/retentionChannels: they are
// set post-construction (see SetRetentionPurgeNotifier) from // set post-construction (see SetRetentionPurgeNotifier) from
@ -242,6 +246,18 @@ func WithStorageMaxTotalBytes(maxBytes int64) Option {
} }
} }
// WithSecretChatDeleteFileAfterDownload enables DeleteEncryptedFileBlob's
// actual deletion (TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD). Default
// true -- see that env var's doc comment in internal/config/config.go for
// why (secret-chat blobs have no other cleanup path at all otherwise).
// Passing false opts back out for a self-hoster who wants the ciphertext
// kept around regardless.
func WithSecretChatDeleteFileAfterDownload(enabled bool) Option {
return func(s *Service) {
s.secretChatDeleteFileAfterDownload = enabled
}
}
// WithGifSeedDir records the gif seed directory (cfg.GifSeedDir) so // WithGifSeedDir records the gif seed directory (cfg.GifSeedDir) so
// AdminDeleteUncategorizedGifs can remove a seed-imported entry's source // AdminDeleteUncategorizedGifs can remove a seed-imported entry's source
// file alongside its DB row -- see the field's doc comment for why that // file alongside its DB row -- see the field's doc comment for why that

View file

@ -383,6 +383,19 @@ type Config struct {
// StorageMaxTotalBytes has meant so far (block new uploads only -> // StorageMaxTotalBytes has meant so far (block new uploads only ->
// block-and-reclaim from existing ones too). // block-and-reclaim from existing ones too).
StorageEvictionEnable bool StorageEvictionEnable bool
// SecretChatDeleteFileAfterDownload deletes a secret-chat encrypted
// file's blob bytes (location_key "enc:<id>") as soon as the recipient
// has fully downloaded it. Secret chats are single-device on both ends
// (no multi-device sync), so once delivered there is no other device
// that will ever ask for it again -- unlike ordinary message media,
// which stays downloadable for re-fetch/sync/forward. Default true:
// none of the automatic retention/eviction machinery touches these
// blobs at all (they have no documents/photos row), so leaving this off
// means they simply accumulate forever with no way to reclaim the
// space -- an explicit TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD=false
// opts back out for a self-hoster who wants the ciphertext kept around
// regardless.
SecretChatDeleteFileAfterDownload bool
// StickerSeedDir 是 reaction / sticker 资源种子目录(导入到 documents/sticker_sets + blob // StickerSeedDir 是 reaction / sticker 资源种子目录(导入到 documents/sticker_sets + blob
StickerSeedDir string StickerSeedDir string
// StickerSeedMaxSets 限制导入的常规贴纸集数量(避免启动时导入过多包),<=0 表示不限。 // StickerSeedMaxSets 限制导入的常规贴纸集数量(避免启动时导入过多包),<=0 表示不限。
@ -1009,6 +1022,7 @@ func Load() (Config, error) {
StorageRetentionMaxAgeByCategory: storageRetentionMaxAgeByCategoryFromEnv(envDurationOr), StorageRetentionMaxAgeByCategory: storageRetentionMaxAgeByCategoryFromEnv(envDurationOr),
StorageRetentionMaxAgeAvatar: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR", 0), StorageRetentionMaxAgeAvatar: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR", 0),
StorageEvictionEnable: envBoolOr("TELESRV_STORAGE_EVICTION_ENABLE", false), StorageEvictionEnable: envBoolOr("TELESRV_STORAGE_EVICTION_ENABLE", false),
SecretChatDeleteFileAfterDownload: envBoolOr("TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD", true),
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"), StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300), StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"), PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"),

View file

@ -7,6 +7,8 @@ package hoststats
import ( import (
"context" "context"
"os"
"path/filepath"
"sync" "sync"
"time" "time"
) )
@ -14,12 +16,18 @@ import (
// Snapshot is the last successfully sampled host-resource reading. Ready is // Snapshot is the last successfully sampled host-resource reading. Ready is
// false until the first sample completes, so callers can distinguish "0% CPU" // false until the first sample completes, so callers can distinguish "0% CPU"
// from "no data yet" instead of rendering a misleading zero on startup. // from "no data yet" instead of rendering a misleading zero on startup.
// DiskReady is a separate flag: disk stats are sampled from a configured
// path (see NewPoller) that can fail independently of CPU/memory sampling
// (wrong working directory, path not created yet, etc) -- without it, a
// failed disk read looked identical to "this server's disk is completely
// full" (0 free bytes) instead of "we don't have a reading right now".
type Snapshot struct { type Snapshot struct {
CPUPercent float64 CPUPercent float64
MemUsedBytes int64 MemUsedBytes int64
MemTotalBytes int64 MemTotalBytes int64
DiskFreeBytes int64 DiskFreeBytes int64
DiskTotalBytes int64 DiskTotalBytes int64
DiskReady bool
Ready bool Ready bool
} }
@ -29,6 +37,10 @@ type Snapshot struct {
// for the local blob-storage free-space guard. // for the local blob-storage free-space guard.
type Poller struct { type Poller struct {
diskPath string diskPath string
// diskFreeBytesFn defaults to the platform diskFreeBytes function;
// overridable in tests to simulate a failing/succeeding disk read
// without touching the real filesystem/OS call.
diskFreeBytesFn func(path string) (free, total int64, err error)
mu sync.RWMutex mu sync.RWMutex
snap Snapshot snap Snapshot
@ -38,12 +50,31 @@ type Poller struct {
// NewPoller creates a poller that reports free/total disk space for the // NewPoller creates a poller that reports free/total disk space for the
// filesystem containing diskPath (pass the server's data/blob directory, or // filesystem containing diskPath (pass the server's data/blob directory, or
// "." if it doesn't matter which volume). // "." if it doesn't matter which volume). diskPath is resolved to an
// absolute path up front (a relative path depends on the process's current
// directory, which callers shouldn't have to reason about here) and, if it
// doesn't exist yet -- e.g. an S3-backend deployment whose local blob
// staging directory is only created on first upload -- walked up to the
// nearest existing ancestor, since GetDiskFreeSpaceEx/statfs need a real
// path and every ancestor is on the same volume anyway.
func NewPoller(diskPath string) *Poller { func NewPoller(diskPath string) *Poller {
if diskPath == "" { if diskPath == "" {
diskPath = "." diskPath = "."
} }
return &Poller{diskPath: diskPath} if abs, err := filepath.Abs(diskPath); err == nil {
diskPath = abs
}
for {
if _, err := os.Stat(diskPath); err == nil {
break
}
parent := filepath.Dir(diskPath)
if parent == diskPath {
break
}
diskPath = parent
}
return &Poller{diskPath: diskPath, diskFreeBytesFn: diskFreeBytes}
} }
// Snapshot returns the last sample. Safe to call concurrently with Run. // Snapshot returns the last sample. Safe to call concurrently with Run.
@ -72,13 +103,27 @@ func (p *Poller) Run(ctx context.Context, interval time.Duration) {
} }
func (p *Poller) sampleOnce() { func (p *Poller) sampleOnce() {
p.mu.RLock()
prevFree, prevTotal := p.snap.DiskFreeBytes, p.snap.DiskTotalBytes
p.mu.RUnlock()
var snap Snapshot var snap Snapshot
snap.CPUPercent = p.cpu.sample() snap.CPUPercent = p.cpu.sample()
if used, total, err := memStats(); err == nil { if used, total, err := memStats(); err == nil {
snap.MemUsedBytes, snap.MemTotalBytes = used, total snap.MemUsedBytes, snap.MemTotalBytes = used, total
} }
if free, total, err := diskFreeBytes(p.diskPath); err == nil { if free, total, err := p.diskFreeBytesFn(p.diskPath); err == nil {
snap.DiskFreeBytes, snap.DiskTotalBytes = free, total snap.DiskFreeBytes, snap.DiskTotalBytes = free, total
snap.DiskReady = true
} else {
// Keep the last known-good byte values stored (harmless, and a
// reasonable fallback for any future caller that wants "last known"
// over nothing) but DiskReady reflects THIS sample, not a stale one
// -- a failure must show as "no current reading", not silently keep
// claiming Ready while quietly reusing old numbers forever if the
// underlying path became permanently unreadable.
snap.DiskFreeBytes, snap.DiskTotalBytes = prevFree, prevTotal
snap.DiskReady = false
} }
snap.Ready = true snap.Ready = true

View file

@ -0,0 +1,106 @@
package hoststats
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
)
// TestSampleOncePreservesLastGoodDiskReadingOnFailure guards the bug
// reported live: a failed disk-space sample used to silently reset
// DiskFreeBytes/DiskTotalBytes to 0 while still marking the overall
// snapshot Ready -- rendering as "0 bytes free" (indistinguishable from an
// actually-full disk) instead of "no reading right now". A failed sample
// must keep the last known-good reading and report DiskReady=false.
func TestSampleOncePreservesLastGoodDiskReadingOnFailure(t *testing.T) {
p := &Poller{diskFreeBytesFn: func(string) (int64, int64, error) {
return 1234, 5678, nil
}}
p.sampleOnce()
first := p.Snapshot()
if !first.DiskReady || first.DiskFreeBytes != 1234 || first.DiskTotalBytes != 5678 {
t.Fatalf("first snapshot = %+v, want a successful disk reading", first)
}
p.diskFreeBytesFn = func(string) (int64, int64, error) {
return 0, 0, errors.New("disk stat failed")
}
p.sampleOnce()
second := p.Snapshot()
if second.DiskReady {
t.Fatal("DiskReady = true after a failed sample, want false")
}
if second.DiskFreeBytes != 1234 || second.DiskTotalBytes != 5678 {
t.Fatalf("disk fields after failed sample = (%d, %d), want the preserved (1234, 5678)", second.DiskFreeBytes, second.DiskTotalBytes)
}
// CPU/memory sampling must still complete and mark Ready, independent
// of the disk failure.
if !second.Ready {
t.Fatal("Ready = false after a disk-only failure, want true (CPU/mem still sampled)")
}
}
// TestSampleOnceNeverReadyWithoutAnyPriorSuccess confirms a disk read that
// has NEVER succeeded (not just failed after a prior success) still reports
// DiskReady=false with zero-value fields, not a fabricated 0-bytes-free
// reading.
func TestSampleOnceNeverReadyWithoutAnyPriorSuccess(t *testing.T) {
p := &Poller{diskFreeBytesFn: func(string) (int64, int64, error) {
return 0, 0, errors.New("never worked")
}}
p.sampleOnce()
snap := p.Snapshot()
if snap.DiskReady {
t.Fatal("DiskReady = true with no successful sample ever, want false")
}
if !snap.Ready {
t.Fatal("Ready = false, want true (CPU/mem sampling is independent of disk)")
}
}
// TestNewPollerWalksUpToNearestExistingAncestor guards the other half of the
// fix: a configured disk path that doesn't exist yet (e.g. an S3-backend
// deployment's local blob staging directory, only created on first upload)
// must not permanently break disk stats -- NewPoller walks up to the
// nearest existing ancestor instead of handing GetDiskFreeSpaceEx/statfs a
// path they will always fail to stat.
func TestNewPollerWalksUpToNearestExistingAncestor(t *testing.T) {
tmp := t.TempDir()
missing := filepath.Join(tmp, "not-created-yet", "nested", "deeper")
p := NewPoller(missing)
if p.diskPath != tmp {
t.Fatalf("resolved disk path = %q, want the nearest existing ancestor %q", p.diskPath, tmp)
}
if _, err := os.Stat(p.diskPath); err != nil {
t.Fatalf("resolved disk path %q does not exist: %v", p.diskPath, err)
}
}
// TestNewPollerResolvesRelativePathToAbsolute confirms a relative diskPath
// (as configured today, e.g. "data/blobs") no longer silently depends on
// whatever the process's current directory happens to be at NewPoller time.
func TestNewPollerResolvesRelativePathToAbsolute(t *testing.T) {
p := NewPoller(".")
if !filepath.IsAbs(p.diskPath) {
t.Fatalf("resolved disk path %q is not absolute", p.diskPath)
}
}
// TestRunSamplesImmediatelyOnStart confirms Run's documented behavior
// (sampleOnce before entering the ticker loop) without relying on any
// ticker firing or a busy-wait: cancel the context right away and check the
// synchronous first sample already landed by the time Run returns.
func TestRunSamplesImmediatelyOnStart(t *testing.T) {
p := &Poller{diskFreeBytesFn: func(string) (int64, int64, error) { return 1, 1, nil }}
ctx, cancel := context.WithCancel(context.Background())
cancel()
p.Run(ctx, time.Hour)
if !p.Snapshot().Ready {
t.Fatal("Run did not produce a ready snapshot from its initial sample")
}
}

View file

@ -0,0 +1,46 @@
package procctl
import (
"os"
"path/filepath"
"testing"
)
// TestReadEnvGroupsSensitiveFlag guards against sensitiveKeyRe false-positive
// matches on the word "SECRET" in a name that means Telegram's secret-chat
// feature, not a credential (e.g. TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD),
// while still flagging real credential-shaped names as sensitive.
func TestReadEnvGroupsSensitiveFlag(t *testing.T) {
root := t.TempDir()
tmpl := "## Storage & Media -- test group\n" +
"# desc\n" +
"TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD=true\n" +
"# desc\n" +
"TELESRV_ADMIN_PASSWORD=\n" +
"# desc\n" +
"TELESRV_BOT_API_KEY=\n"
if err := os.WriteFile(filepath.Join(root, ".env.example"), []byte(tmpl), 0o644); err != nil {
t.Fatalf("write .env.example: %v", err)
}
groups, err := NewManager(root).ReadEnvGroups()
if err != nil {
t.Fatalf("ReadEnvGroups: %v", err)
}
got := map[string]bool{}
for _, g := range groups {
for _, f := range g.Fields {
got[f.Key] = f.Sensitive
}
}
if got["TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD"] {
t.Errorf("TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD marked sensitive, want not (it's a boolean toggle, not a credential)")
}
if !got["TELESRV_ADMIN_PASSWORD"] {
t.Errorf("TELESRV_ADMIN_PASSWORD not marked sensitive, want sensitive")
}
if !got["TELESRV_BOT_API_KEY"] {
t.Errorf("TELESRV_BOT_API_KEY not marked sensitive, want sensitive")
}
}

View file

@ -490,6 +490,11 @@ var (
activeFieldRe = regexp.MustCompile(`^(TELESRV_[A-Z0-9_]+)=(.*)$`) activeFieldRe = regexp.MustCompile(`^(TELESRV_[A-Z0-9_]+)=(.*)$`)
commentedFieldRe = regexp.MustCompile(`^#\s*(TELESRV_[A-Z0-9_]+)=(.*)$`) commentedFieldRe = regexp.MustCompile(`^#\s*(TELESRV_[A-Z0-9_]+)=(.*)$`)
sensitiveKeyRe = regexp.MustCompile(`(PASSWORD|SECRET|_TOKEN|API_KEY)`) sensitiveKeyRe = regexp.MustCompile(`(PASSWORD|SECRET|_TOKEN|API_KEY)`)
// sensitiveKeyExceptRe excludes names that trip sensitiveKeyRe on the
// word "SECRET" while meaning Telegram's secret-chat feature, not a
// credential (e.g. TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD) --
// there's nothing to mask there, it's a plain boolean toggle.
sensitiveKeyExceptRe = regexp.MustCompile(`SECRET_CHAT`)
groupHeaderRe = regexp.MustCompile(`^##\s*(.+?)\s*--\s*(.+)$`) groupHeaderRe = regexp.MustCompile(`^##\s*(.+?)\s*--\s*(.+)$`)
sectionBreakRe = regexp.MustCompile(`^#\s*={10,}\s*$`) sectionBreakRe = regexp.MustCompile(`^#\s*={10,}\s*$`)
) )
@ -554,7 +559,7 @@ func (m *Manager) ReadEnvGroups() ([]EnvGroup, error) {
DefaultValue: defaultValue, DefaultValue: defaultValue,
Description: description, Description: description,
EnabledByDefault: enabledByDefault, EnabledByDefault: enabledByDefault,
Sensitive: sensitiveKeyRe.MatchString(key), Sensitive: sensitiveKeyRe.MatchString(key) && !sensitiveKeyExceptRe.MatchString(key),
Value: value, Value: value,
}) })
} }

View file

@ -64,7 +64,7 @@ func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
}) })
cutoff := time.Now().Add(-24 * time.Hour) cutoff := time.Now().Add(-24 * time.Hour)
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, cutoff, 1000) ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, &cutoff, 1000)
if err != nil { if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err) t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
} }
@ -95,7 +95,7 @@ func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
// Idempotent / self-terminating: a second sweep pass no longer selects // Idempotent / self-terminating: a second sweep pass no longer selects
// this document, since it no longer owns any file_blobs row. // this document, since it no longer owns any file_blobs row.
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, cutoff, 1000) ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, &cutoff, 1000)
if err != nil { if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err) t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
} }

View file

@ -236,48 +236,90 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
return blobs, nil return blobs, nil
} }
// nullableTimestamptz converts an optional cutoff into the nullable
// pgtype.Timestamptz the hard-retention List*/Count* queries expect: nil
// means "no age filter at all" (sqlc.narg(cutoff) IS NULL in the query),
// matching the manual-purge admin action's "no date = everything" contract.
// The automatic sweep (files.Service.DeleteBlobBytesForMediaOlderThan)
// always passes a non-nil cutoff.
func nullableTimestamptz(cutoff *time.Time) pgtype.Timestamptz {
if cutoff == nil {
return pgtype.Timestamptz{}
}
return pgtype.Timestamptz{Time: *cutoff, Valid: true}
}
// ListDocumentIDsForHardRetentionOlderThan returns document ids in the given // ListDocumentIDsForHardRetentionOlderThan returns document ids in the given
// category older than cutoff (by upload/created_at) that still own at least // category older than cutoff (by upload/created_at) that still own at least
// one file_blobs row, oldest first, up to limit -- the "hard" retention // one file_blobs row, oldest first, up to limit -- the "hard" retention
// sweep's candidate list. Unlike ListOrphanedDocumentIDsOlderThan, this // sweep's candidate list. Unlike ListOrphanedDocumentIDsOlderThan, this
// ignores media_references entirely: a document still referenced by a live // ignores media_references entirely: a document still referenced by a live
// message is exactly as eligible as an orphaned one. // message is exactly as eligible as an orphaned one. cutoff may be nil,
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error) { // meaning no age filter at all (every document in the category is a
// candidate) -- used by the manual purge admin action.
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff *time.Time, limit int) ([]int64, error) {
if limit <= 0 { if limit <= 0 {
return nil, nil return nil, nil
} }
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{ return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, Cutoff: nullableTimestamptz(cutoff),
Category: int16(category), Category: int16(category),
BatchLimit: int32(limit), BatchLimit: int32(limit),
}) })
} }
// CountDocumentsForHardRetention is the exact-count counterpart of
// ListDocumentIDsForHardRetentionOlderThan, for the manual purge admin
// action's dry-run preview.
func (s *MediaStore) CountDocumentsForHardRetention(ctx context.Context, category domain.MediaCategory, cutoff *time.Time) (int, error) {
n, err := s.q.CountDocumentsForHardRetention(ctx, sqlcgen.CountDocumentsForHardRetentionParams{
Cutoff: nullableTimestamptz(cutoff),
Category: int16(category),
})
return int(n), err
}
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of // ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
// ListDocumentIDsForHardRetentionOlderThan (excluding a live avatar -- see // ListDocumentIDsForHardRetentionOlderThan (excluding a live avatar -- see
// ListAvatarPhotoIDsForHardRetentionOlderThan). // ListAvatarPhotoIDsForHardRetentionOlderThan). cutoff may be nil -- see
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) { // ListDocumentIDsForHardRetentionOlderThan.
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff *time.Time, limit int) ([]int64, error) {
if limit <= 0 { if limit <= 0 {
return nil, nil return nil, nil
} }
return s.q.ListPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListPhotoIDsForHardRetentionOlderThanParams{ return s.q.ListPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListPhotoIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, Cutoff: nullableTimestamptz(cutoff),
BatchLimit: int32(limit), BatchLimit: int32(limit),
}) })
} }
// CountPhotosForHardRetention is the exact-count counterpart of
// ListPhotoIDsForHardRetentionOlderThan.
func (s *MediaStore) CountPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error) {
n, err := s.q.CountPhotosForHardRetention(ctx, nullableTimestamptz(cutoff))
return int(n), err
}
// ListAvatarPhotoIDsForHardRetentionOlderThan is the "Avatar" category // ListAvatarPhotoIDsForHardRetentionOlderThan is the "Avatar" category
// counterpart of ListPhotoIDsForHardRetentionOlderThan. // counterpart of ListPhotoIDsForHardRetentionOlderThan. cutoff may be nil --
func (s *MediaStore) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) { // see ListDocumentIDsForHardRetentionOlderThan.
func (s *MediaStore) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff *time.Time, limit int) ([]int64, error) {
if limit <= 0 { if limit <= 0 {
return nil, nil return nil, nil
} }
return s.q.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListAvatarPhotoIDsForHardRetentionOlderThanParams{ return s.q.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListAvatarPhotoIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, Cutoff: nullableTimestamptz(cutoff),
BatchLimit: int32(limit), BatchLimit: int32(limit),
}) })
} }
// CountAvatarPhotosForHardRetention is the exact-count counterpart of
// ListAvatarPhotoIDsForHardRetentionOlderThan.
func (s *MediaStore) CountAvatarPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error) {
n, err := s.q.CountAvatarPhotosForHardRetention(ctx, nullableTimestamptz(cutoff))
return int(n), err
}
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns // DeleteFileBlobsForDocument deletes every file_blobs row a document owns
// (main body + thumbnail variants), returning what was deleted so the caller // (main body + thumbnail variants), returning what was deleted so the caller
// can physically remove each object from its backend once confirming (via // can physically remove each object from its backend once confirming (via

View file

@ -277,9 +277,14 @@ LIMIT sqlc.arg(batch_limit)::int;
-- rows but deliberately leaves this documents row in place), it naturally -- rows but deliberately leaves this documents row in place), it naturally
-- drops out of this query on the next pass. category scopes to one -- drops out of this query on the next pass. category scopes to one
-- documents.category bucket per sweep tick -- see -- documents.category bucket per sweep tick -- see
-- ListOrphanedDocumentIDsOlderThan for the 0/None fallback note. -- ListOrphanedDocumentIDsOlderThan for the 0/None fallback note. cutoff is
-- nullable: NULL means no age filter at all (every document in the category
-- is a candidate, regardless of created_at) -- used by the manual purge admin
-- action, which unlike the automatic sweep can target "everything" with no
-- age cutoff. The automatic sweep (files.Service.DeleteBlobBytesForMediaOlderThan)
-- always passes a real, non-null cutoff.
SELECT d.id FROM documents d SELECT d.id FROM documents d
WHERE d.created_at < sqlc.arg(cutoff)::timestamptz WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR d.created_at < sqlc.narg(cutoff)::timestamptz)
AND d.category = sqlc.arg(category)::smallint AND d.category = sqlc.arg(category)::smallint
AND EXISTS ( AND EXISTS (
SELECT 1 FROM file_blobs fb SELECT 1 FROM file_blobs fb
@ -289,12 +294,26 @@ WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
ORDER BY d.created_at ASC ORDER BY d.created_at ASC
LIMIT sqlc.arg(batch_limit)::int; LIMIT sqlc.arg(batch_limit)::int;
-- name: CountDocumentsForHardRetention :one
-- Exact-count counterpart of ListDocumentIDsForHardRetentionOlderThan, used
-- by the manual purge admin action's dry-run preview: a LIMIT-capped list
-- length is not good enough there, the operator needs the real total.
SELECT COUNT(*)::int FROM documents d
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR d.created_at < sqlc.narg(cutoff)::timestamptz)
AND d.category = sqlc.arg(category)::smallint
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
);
-- name: ListPhotoIDsForHardRetentionOlderThan :many -- name: ListPhotoIDsForHardRetentionOlderThan :many
-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention -- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
-- candidate selection, for photos. Excludes photos currently active as -- candidate selection (including the nullable cutoff), for photos. Excludes
-- someone's avatar -- see ListAvatarPhotoIDsForHardRetentionOlderThan. -- photos currently active as someone's avatar -- see
-- ListAvatarPhotoIDsForHardRetentionOlderThan.
SELECT p.id FROM photos p SELECT p.id FROM photos p
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR p.created_at < sqlc.narg(cutoff)::timestamptz)
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active) AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS ( AND EXISTS (
SELECT 1 FROM file_blobs fb SELECT 1 FROM file_blobs fb
@ -304,13 +323,25 @@ WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
ORDER BY p.created_at ASC ORDER BY p.created_at ASC
LIMIT sqlc.arg(batch_limit)::int; LIMIT sqlc.arg(batch_limit)::int;
-- name: CountPhotosForHardRetention :one
-- Exact-count counterpart of ListPhotoIDsForHardRetentionOlderThan -- see
-- CountDocumentsForHardRetention.
SELECT COUNT(*)::int FROM photos p
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR p.created_at < sqlc.narg(cutoff)::timestamptz)
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
);
-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many -- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many
-- Same as ListPhotoIDsForHardRetentionOlderThan but only photos currently -- Same as ListPhotoIDsForHardRetentionOlderThan but only photos currently
-- active as someone's avatar (profile_photos.active) -- lets the Avatar -- active as someone's avatar (profile_photos.active) -- lets the Avatar
-- category carry its own retention age, independent of ordinary shared-media -- category carry its own retention age, independent of ordinary shared-media
-- photos. -- photos.
SELECT p.id FROM photos p SELECT p.id FROM photos p
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR p.created_at < sqlc.narg(cutoff)::timestamptz)
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active) AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS ( AND EXISTS (
SELECT 1 FROM file_blobs fb SELECT 1 FROM file_blobs fb
@ -320,6 +351,17 @@ WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
ORDER BY p.created_at ASC ORDER BY p.created_at ASC
LIMIT sqlc.arg(batch_limit)::int; LIMIT sqlc.arg(batch_limit)::int;
-- name: CountAvatarPhotosForHardRetention :one
-- Exact-count counterpart of ListAvatarPhotoIDsForHardRetentionOlderThan.
SELECT COUNT(*)::int FROM photos p
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR p.created_at < sqlc.narg(cutoff)::timestamptz)
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
);
-- name: ListOldestDocumentsForEviction :many -- name: ListOldestDocumentsForEviction :many
-- Active eviction (TELESRV_STORAGE_EVICTION_ENABLE): candidates are every -- Active eviction (TELESRV_STORAGE_EVICTION_ENABLE): candidates are every
-- document that still owns at least one file_blobs row, oldest-uploaded -- document that still owns at least one file_blobs row, oldest-uploaded

View file

@ -279,7 +279,7 @@ func TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory(t *testing.T) {
} }
cutoff := time.Now().Add(-24 * time.Hour) cutoff := time.Now().Add(-24 * time.Hour)
videoIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryVideo, cutoff, 1000) videoIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryVideo, &cutoff, 1000)
if err != nil { if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Video): %v", err) t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Video): %v", err)
} }
@ -287,7 +287,7 @@ func TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory(t *testing.T) {
t.Fatalf("video-category candidates = %v, want to include %d and exclude %d", videoIDs, videoID, musicID) t.Fatalf("video-category candidates = %v, want to include %d and exclude %d", videoIDs, videoID, musicID)
} }
musicIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryMusic, cutoff, 1000) musicIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryMusic, &cutoff, 1000)
if err != nil { if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Music): %v", err) t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Music): %v", err)
} }
@ -296,6 +296,123 @@ func TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory(t *testing.T) {
} }
} }
// TestManualPurgeNilCutoffIgnoresAge guards the manual purge admin action's
// query-layer contract: a nil cutoff on the hard-retention List/Count
// queries must mean "no age filter at all" -- a document created just now is
// still a candidate -- while a real, non-nil cutoff still filters normally.
// This is the query-layer half of files.Service.ManualPurge/CountManualPurgeCandidates's
// "no date = everything matching the categories, regardless of age" contract
// (the automatic sweep, by contrast, never passes a nil cutoff).
func TestManualPurgeNilCutoffIgnoresAge(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
media := NewMediaStore(pool)
freshID := time.Now().UnixNano()
locationKey := "doc:" + strconv.FormatInt(freshID, 10)
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = $1", freshID)
_, _ = pool.Exec(context.Background(), "DELETE FROM file_blobs WHERE location_key = $1", locationKey)
})
if err := media.PutDocument(ctx, domain.Document{ID: freshID, MimeType: "application/octet-stream", Size: 128}); err != nil {
t.Fatalf("PutDocument: %v", err)
}
// Deliberately NOT backdated -- created_at stays "just now", unlike every
// other hard-retention test in this file, since the whole point here is
// that a nil cutoff must not exempt a fresh document.
if _, err := pool.Exec(ctx, "UPDATE documents SET category = $2 WHERE id = $1", freshID, int16(domain.MediaCategoryFile)); err != nil {
t.Fatalf("categorize document: %v", err)
}
blob := postgresTestBlob(locationKey, "manual-purge-nil-cutoff", 128, "application/octet-stream")
if err := media.PutFileBlob(ctx, blob); err != nil {
t.Fatalf("PutFileBlob: %v", err)
}
// nil cutoff: the fresh document IS a candidate.
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryFile, nil, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(nil cutoff): %v", err)
}
if !containsInt64(ids, freshID) {
t.Fatalf("nil-cutoff candidates = %v, want to include fresh document %d", ids, freshID)
}
count, err := media.CountDocumentsForHardRetention(ctx, domain.MediaCategoryFile, nil)
if err != nil {
t.Fatalf("CountDocumentsForHardRetention(nil cutoff): %v", err)
}
if count < 1 {
t.Fatalf("nil-cutoff count = %d, want >= 1 (must include fresh document)", count)
}
// A real cutoff in the past still excludes the fresh document.
pastCutoff := time.Now().Add(-24 * time.Hour)
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryFile, &pastCutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(past cutoff): %v", err)
}
if containsInt64(ids, freshID) {
t.Fatalf("past-cutoff candidates = %v, want to exclude fresh document %d", ids, freshID)
}
count, err = media.CountDocumentsForHardRetention(ctx, domain.MediaCategoryFile, &pastCutoff)
if err != nil {
t.Fatalf("CountDocumentsForHardRetention(past cutoff): %v", err)
}
// Some other, unrelated old File-category document could in principle
// exist in this shared test database, so this only asserts the fresh
// document itself dropped out, via the id-level checks above; the count
// assertion below just guards against the query going the wrong
// direction entirely (e.g. cutoff being ignored and still counting
// everything).
allCount, err := media.CountDocumentsForHardRetention(ctx, domain.MediaCategoryFile, nil)
if err != nil {
t.Fatalf("CountDocumentsForHardRetention(nil, recheck): %v", err)
}
if count > allCount {
t.Fatalf("past-cutoff count %d > nil-cutoff count %d, cutoff filter is backwards", count, allCount)
}
}
// TestManualPurgePhotoAndAvatarNilCutoffIgnoresAge is the photo/avatar
// counterpart of TestManualPurgeNilCutoffIgnoresAge.
func TestManualPurgePhotoAndAvatarNilCutoffIgnoresAge(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
media := NewMediaStore(pool)
photoID := time.Now().UnixNano()
locationKey := "photo:" + strconv.FormatInt(photoID, 10)
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), "DELETE FROM photos WHERE id = $1", photoID)
_, _ = pool.Exec(context.Background(), "DELETE FROM file_blobs WHERE location_key = $1", locationKey)
})
if err := media.PutPhoto(ctx, domain.Photo{ID: photoID, AccessHash: 1}); err != nil {
t.Fatalf("PutPhoto: %v", err)
}
blob := postgresTestBlob(locationKey, "manual-purge-photo-nil-cutoff", 64, "image/jpeg")
if err := media.PutFileBlob(ctx, blob); err != nil {
t.Fatalf("PutFileBlob: %v", err)
}
ids, err := media.ListPhotoIDsForHardRetentionOlderThan(ctx, nil, 1000)
if err != nil {
t.Fatalf("ListPhotoIDsForHardRetentionOlderThan(nil cutoff): %v", err)
}
if !containsInt64(ids, photoID) {
t.Fatalf("nil-cutoff photo candidates = %v, want to include fresh photo %d", ids, photoID)
}
pastCutoff := time.Now().Add(-24 * time.Hour)
ids, err = media.ListPhotoIDsForHardRetentionOlderThan(ctx, &pastCutoff, 1000)
if err != nil {
t.Fatalf("ListPhotoIDsForHardRetentionOlderThan(past cutoff): %v", err)
}
if containsInt64(ids, photoID) {
t.Fatalf("past-cutoff photo candidates = %v, want to exclude fresh photo %d", ids, photoID)
}
}
// TestEvictionListsOldestAcrossDocumentsAndPhotos guards Part 3's eviction // TestEvictionListsOldestAcrossDocumentsAndPhotos guards Part 3's eviction
// query layer: ListOldestMediaForEviction must return candidates from both // query layer: ListOldestMediaForEviction must return candidates from both
// tables (interleaving by created_at is done by the caller in Go, see // tables (interleaving by created_at is done by the caller in Go, see

View file

@ -79,6 +79,51 @@ func (q *Queries) CountAvailableReactions(ctx context.Context) (int32, error) {
return total, err return total, err
} }
const countAvatarPhotosForHardRetention = `-- name: CountAvatarPhotosForHardRetention :one
SELECT COUNT(*)::int FROM photos p
WHERE ($1::timestamptz IS NULL OR p.created_at < $1::timestamptz)
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
`
// Exact-count counterpart of ListAvatarPhotoIDsForHardRetentionOlderThan.
func (q *Queries) CountAvatarPhotosForHardRetention(ctx context.Context, cutoff pgtype.Timestamptz) (int32, error) {
row := q.db.QueryRow(ctx, countAvatarPhotosForHardRetention, cutoff)
var column_1 int32
err := row.Scan(&column_1)
return column_1, err
}
const countDocumentsForHardRetention = `-- name: CountDocumentsForHardRetention :one
SELECT COUNT(*)::int FROM documents d
WHERE ($1::timestamptz IS NULL OR d.created_at < $1::timestamptz)
AND d.category = $2::smallint
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
)
`
type CountDocumentsForHardRetentionParams struct {
Cutoff pgtype.Timestamptz
Category int16
}
// Exact-count counterpart of ListDocumentIDsForHardRetentionOlderThan, used
// by the manual purge admin action's dry-run preview: a LIMIT-capped list
// length is not good enough there, the operator needs the real total.
func (q *Queries) CountDocumentsForHardRetention(ctx context.Context, arg CountDocumentsForHardRetentionParams) (int32, error) {
row := q.db.QueryRow(ctx, countDocumentsForHardRetention, arg.Cutoff, arg.Category)
var column_1 int32
err := row.Scan(&column_1)
return column_1, err
}
const countFileBlobRefs = `-- name: CountFileBlobRefs :one const countFileBlobRefs = `-- name: CountFileBlobRefs :one
SELECT COUNT(*)::int FROM file_blobs WHERE backend = $1::text AND object_key = $2::text SELECT COUNT(*)::int FROM file_blobs WHERE backend = $1::text AND object_key = $2::text
` `
@ -95,6 +140,26 @@ func (q *Queries) CountFileBlobRefs(ctx context.Context, arg CountFileBlobRefsPa
return column_1, err return column_1, err
} }
const countPhotosForHardRetention = `-- name: CountPhotosForHardRetention :one
SELECT COUNT(*)::int FROM photos p
WHERE ($1::timestamptz IS NULL OR p.created_at < $1::timestamptz)
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
`
// Exact-count counterpart of ListPhotoIDsForHardRetentionOlderThan -- see
// CountDocumentsForHardRetention.
func (q *Queries) CountPhotosForHardRetention(ctx context.Context, cutoff pgtype.Timestamptz) (int32, error) {
row := q.db.QueryRow(ctx, countPhotosForHardRetention, cutoff)
var column_1 int32
err := row.Scan(&column_1)
return column_1, err
}
const countProfilePhotos = `-- name: CountProfilePhotos :one const countProfilePhotos = `-- name: CountProfilePhotos :one
SELECT count(*)::int AS total SELECT count(*)::int AS total
FROM profile_photos FROM profile_photos
@ -866,7 +931,7 @@ func (q *Queries) ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, arg L
const listAvatarPhotoIDsForHardRetentionOlderThan = `-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many const listAvatarPhotoIDsForHardRetentionOlderThan = `-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many
SELECT p.id FROM photos p SELECT p.id FROM photos p
WHERE p.created_at < $1::timestamptz WHERE ($1::timestamptz IS NULL OR p.created_at < $1::timestamptz)
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active) AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS ( AND EXISTS (
SELECT 1 FROM file_blobs fb SELECT 1 FROM file_blobs fb
@ -908,7 +973,7 @@ func (q *Queries) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Contex
const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
SELECT d.id FROM documents d SELECT d.id FROM documents d
WHERE d.created_at < $1::timestamptz WHERE ($1::timestamptz IS NULL OR d.created_at < $1::timestamptz)
AND d.category = $2::smallint AND d.category = $2::smallint
AND EXISTS ( AND EXISTS (
SELECT 1 FROM file_blobs fb SELECT 1 FROM file_blobs fb
@ -933,7 +998,12 @@ type ListDocumentIDsForHardRetentionOlderThanParams struct {
// rows but deliberately leaves this documents row in place), it naturally // rows but deliberately leaves this documents row in place), it naturally
// drops out of this query on the next pass. category scopes to one // drops out of this query on the next pass. category scopes to one
// documents.category bucket per sweep tick -- see // documents.category bucket per sweep tick -- see
// ListOrphanedDocumentIDsOlderThan for the 0/None fallback note. // ListOrphanedDocumentIDsOlderThan for the 0/None fallback note. cutoff is
// nullable: NULL means no age filter at all (every document in the category
// is a candidate, regardless of created_at) -- used by the manual purge admin
// action, which unlike the automatic sweep can target "everything" with no
// age cutoff. The automatic sweep (files.Service.DeleteBlobBytesForMediaOlderThan)
// always passes a real, non-null cutoff.
func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) { func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.Category, arg.BatchLimit) rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.Category, arg.BatchLimit)
if err != nil { if err != nil {
@ -1207,7 +1277,7 @@ func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrp
const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
SELECT p.id FROM photos p SELECT p.id FROM photos p
WHERE p.created_at < $1::timestamptz WHERE ($1::timestamptz IS NULL OR p.created_at < $1::timestamptz)
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active) AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS ( AND EXISTS (
SELECT 1 FROM file_blobs fb SELECT 1 FROM file_blobs fb
@ -1224,8 +1294,9 @@ type ListPhotoIDsForHardRetentionOlderThanParams struct {
} }
// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention // See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
// candidate selection, for photos. Excludes photos currently active as // candidate selection (including the nullable cutoff), for photos. Excludes
// someone's avatar -- see ListAvatarPhotoIDsForHardRetentionOlderThan. // photos currently active as someone's avatar -- see
// ListAvatarPhotoIDsForHardRetentionOlderThan.
func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) { func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit) rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil { if err != nil {