astra/owpengram-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fixes for storage managament

Browse source
This commit is contained in:
onysd 2026-09-03 08:33:06 +03:00
parent e6bfe2d444
commit 8ef2b58bf9
29 changed files with 1768 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

9
.env.example
View file

@ -340,6 +340,15 @@ TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR=
# TOTAL_BYTES otherwise only ever blocks new uploads, never reclaims from
# existing ones.
TELESRV_STORAGE_EVICTION_ENABLE=false
# Secret-chat encrypted files (photos/documents sent in a secret chat) are
# opaque ciphertext the server can't inspect -- none of the retention/
# eviction settings above ever touch them, so they otherwise accumulate
# forever. Secret chats are single-device on both ends (no multi-device
# sync), so once the one recipient device that will ever ask for a file has
# downloaded it in full, the server has no further reason to keep it.
# Default true: set to false if you'd rather keep the ciphertext around
# regardless (it still can't be read without the secret chat's own key).
TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD=true
# ==============================================================================

38
cmd/telesrv-admin/readstore.go
View file

@ -2539,28 +2539,40 @@ const (
)
// perOwnerMediaSizeSQL is shared between StorageStats and
// ListAccountStorageUsage: each document row's size is its stored column;
// each photo row has no single size column (JSONB sizes holds one entry per
// rendition), so its "attributed size" is the largest rendition -- the
// dominant cost, thumbnails are comparatively tiny. This is an
// approximation (not the exact sum of every rendition's blob bytes, which
// would require joining file_blobs by location_key prefix) chosen for admin
// visibility, not billing precision.
// ListAccountStorageUsage: attributes each document/photo's REAL remaining
// bytes (the sum of every file_blobs row it still owns -- main body plus
// thumbnail/rendition variants) to its owner, one row per document/photo so
// an outer COUNT(*)/SUM(size) aggregate still gets an accurate file count
// alongside the byte total. This used to read documents.size / the largest
// photo rendition size directly -- a static value on the metadata row that
// survives a hard-retention purge unchanged, so it kept counting bytes for
// files whose blobs were long gone. Joining through file_blobs instead means
// a purged item correctly contributes 0: there is nothing left to attribute.
const perOwnerMediaSizeSQL = `
SELECT owner_user_id, size FROM documents
SELECT d.owner_user_id, COALESCE((
SELECT SUM(fb.size) FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
), 0) AS size
FROM documents d
UNION ALL
SELECT p.owner_user_id, COALESCE((
SELECT MAX((elem->>'size')::bigint) FROM jsonb_array_elements(p.sizes) elem
SELECT SUM(fb.size) FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
), 0) AS size
FROM photos p
`
// StorageStatsRow is the admin panel's storage overview: physical bytes
// (from file_blobs, backend-dedup-aware -- what's actually consuming disk
// or S3) versus logical bytes (sum of the same approximate per-row
// attribution the per-account breakdown uses, which can legitimately be
// higher than physical when identical content is shared by more than one
// document/photo).
// or S3, deduplicated exactly once across the whole system) versus logical
// bytes (the SAME real, still-existing file_blobs bytes as physical, just
// summed per-owner via perOwnerMediaSizeSQL without deduplicating content
// shared across accounts/documents -- so logical can legitimately be higher
// than physical when the same blob is attributed to more than one
// document/photo, but a purged file with no file_blobs rows left correctly
// contributes 0 to both, never a stale non-zero "ghost" size).
type StorageStatsRow struct {
PhysicalBytes int64 `json:"PhysicalBytes,string"`
LogicalBytes int64 `json:"LogicalBytes,string"`

32
cmd/telesrv-admin/server.go
View file

@ -136,6 +136,7 @@ func (s *server) routes() http.Handler {
mux.Handle("POST /api/actions/set-gif-catalog-category", s.requireAuthAPI(http.HandlerFunc(s.handleSetGifCatalogCategoryAPI)))
mux.Handle("POST /api/actions/auto-categorize-gif-catalog", s.requireAuthAPI(http.HandlerFunc(s.handleAutoCategorizeGifCatalogAPI)))
mux.Handle("POST /api/actions/delete-uncategorized-gifs", s.requireAuthAPI(http.HandlerFunc(s.handleDeleteUncategorizedGifsAPI)))
mux.Handle("POST /api/actions/storage-manual-purge", s.requireAuthAPI(http.HandlerFunc(s.handleStorageManualPurgeAPI)))
mux.Handle("POST /api/actions/delete-gif-catalog-entry", s.requireAuthAPI(http.HandlerFunc(s.handleDeleteGifCatalogEntryAPI)))
mux.Handle("POST /api/actions/mint-collectible-username", s.requireAuthAPI(http.HandlerFunc(s.handleMintCollectibleUsernameAPI)))
mux.Handle("POST /api/actions/transfer-collectible-username", s.requireAuthAPI(http.HandlerFunc(s.handleTransferCollectibleUsernameAPI)))
@ -2040,6 +2041,37 @@ func (s *server) handleDeleteUncategorizedGifsAPI(w http.ResponseWriter, r *http
writeCommandResultAPI(w, result, err)
}
// storageManualPurgeAPIRequest mirrors ManualPurgeStorageRequest's frontend
// payload (see StoragePage.tsx's manual purge modal): categories/include_avatars
// go through as native JSON, created_before is an ISO/RFC3339 date string
// (produced by `new Date(x).toISOString()` on the frontend, same convention
// as freeze_until -- see setAccountFrozenAPIRequest.Until) that encoding/json
// parses straight into *time.Time; the field left undefined by the frontend
// (no date entered) decodes to nil, meaning no age filter at all.
type storageManualPurgeAPIRequest struct {
CommandID string `json:"command_id"`
Reason string `json:"reason"`
Confirm bool `json:"confirm"`
Categories []string `json:"categories"`
IncludeAvatars bool `json:"include_avatars"`
CreatedBefore *time.Time `json:"created_before"`
}
func (s *server) handleStorageManualPurgeAPI(w http.ResponseWriter, r *http.Request) {
var body storageManualPurgeAPIRequest
if !decodeAction(w, r, &body) {
return
}
req := admin.ManualPurgeStorageRequest{
CommandMeta: s.commandMetaFromAPI(r, body.CommandID, body.Reason, body.Confirm, "storage-manual-purge"),
Categories: body.Categories,
IncludeAvatars: body.IncludeAvatars,
CreatedBefore: body.CreatedBefore,
}
result, err := s.callAdminAPI(r.Context(), "/v1/storage/manual-purge", req)
writeCommandResultAPI(w, result, err)
}
type deleteGifCatalogEntryAPIRequest struct {
CommandID string `json:"command_id"`
Reason string `json:"reason"`

4
cmd/telesrv-admin/web/dist/assets/index-By5gK7SA.js → cmd/telesrv-admin/web/dist/assets/index-CMLbsGwc.js vendored
View file

File diff suppressed because one or more lines are too long

2
cmd/telesrv-admin/web/dist/index.html vendored
View file

@ -23,7 +23,7 @@
})();
</script>
<script type="module" crossorigin src="/assets/index-By5gK7SA.js"></script>
<script type="module" crossorigin src="/assets/index-CMLbsGwc.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-B7hI8ol7.css">
</head>
<body>

6
cmd/telesrv-admin/web/src/pages/Dashboard.tsx
View file

@ -151,12 +151,12 @@ export function Dashboard({ navigate }: { navigate: Navigate }) {
icon={<HardDrive />}
label="Disk free"
percent={
host?.Ready && host.DiskTotalBytes > 0
host?.Ready && host.DiskReady && host.DiskTotalBytes > 0
? ((host.DiskTotalBytes - host.DiskFreeBytes) / host.DiskTotalBytes) * 100
: undefined
}
valueText={host?.Ready ? formatBytes(String(host.DiskFreeBytes)) : "…"}
sub={host?.Ready ? `of ${formatBytes(String(host.DiskTotalBytes))}` : undefined}
valueText={host?.Ready && host.DiskReady ? formatBytes(String(host.DiskFreeBytes)) : "…"}
sub={host?.Ready && host.DiskReady ? `of ${formatBytes(String(host.DiskTotalBytes))}` : "no reading yet"}
warnAbove={85}
/>
</Section>

109
cmd/telesrv-admin/web/src/pages/StoragePage.tsx
View file

@ -1,4 +1,4 @@
import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, Loader2, RefreshCw, Search, Settings2, X } from "lucide-react";
import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, ChevronRight, Loader2, RefreshCw, Search, Settings2, X } from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import { createPortal } from "react-dom";
import { api, errorMessage } from "../api";
@ -38,7 +38,7 @@ function SortableHeader({
);
}
function StorageOverviewTab() {
function StorageOverviewTab({ navigate }: { navigate: Navigate }) {
const [stats, setStats] = useState<StorageStatsResponse | null>(null);
const [rows, setRows] = useState<AccountStorageRow[]>([]);
const [hasMore, setHasMore] = useState(false);
@ -148,6 +148,7 @@ function StorageOverviewTab() {
<SortableHeader label={"Account"} sortKey="username" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
<SortableHeader label={"Storage used"} sortKey="bytes" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
<SortableHeader label={"Files"} sortKey="files" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
<th></th>
</tr>
</thead>
<tbody>
@ -157,9 +158,10 @@ function StorageOverviewTab() {
<td>{displayUsername(row.Username) || row.FirstName || "-"}</td>
<td className="mono">{formatBytes(row.Bytes)}</td>
<td className="mono">{formatQuantity(row.FileCount)}</td>
<td><button className="row-link" type="button" onClick={() => navigate(`/accounts/${row.UserID}`)}>{"Details"} <ChevronRight size={14} /></button></td>
</tr>
))}
{rows.length === 0 && <EmptyRow colSpan={4} />}
{rows.length === 0 && <EmptyRow colSpan={5} />}
</tbody>
</table>
</div>
@ -174,7 +176,7 @@ function StorageOverviewTab() {
);
}
export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
export function StoragePage({ navigate }: { navigate: Navigate }) {
const [tab, setTab] = useState<"overview" | "limits">("overview");
return (
<PageFrame title={"Storage"} eyebrow={"Media / Storage usage"}>
@ -186,7 +188,7 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
{"Limits & Retention"}
</button>
</div>
{tab === "overview" ? <StorageOverviewTab /> : <LimitsRetentionSection />}
{tab === "overview" ? <StorageOverviewTab navigate={navigate} /> : <LimitsRetentionSection />}
</PageFrame>
);
}
@ -438,6 +440,92 @@ function CategoryRetentionModal({
);
}
// ManualPurgeStorageModal lets an operator delete media blob bytes right now
// by hand-picked category (and, optionally, avatars) with an optional
// "created before" age cutoff -- the manual counterpart of the automatic
// hard-retention sweep above. Leaving the date empty purges everything
// matching the selected categories regardless of age. Deletion semantics are
// identical to "hard" retention mode: only the file bytes are removed, never
// the document/photo row, so an affected message still renders its
// placeholder (see internal/app/files.Service.ManualPurge's doc comment).
// Follows the same modal-backdrop/command-modal structure as
// CategoryRetentionModal, but hands the actual mutation off to ActionButton
// (reason + dry-run + confirm) since -- unlike the settings above -- this is
// an immediate, irreversible delete rather than a saved .env value.
function ManualPurgeStorageModal({ onClose }: { onClose: () => void }) {
const [selected, setSelected] = useState<Record<string, boolean>>({});
const [dateValue, setDateValue] = useState("");
const documentFields = CATEGORY_AGE_FIELDS.filter((f) => f.key !== "avatar");
const allSelected = CATEGORY_AGE_FIELDS.every((f) => selected[f.key]);
function toggle(key: string) {
setSelected((prev) => ({ ...prev, [key]: !prev[key] }));
}
function toggleAll() {
const next = !allSelected;
const nextSelected: Record<string, boolean> = {};
for (const field of CATEGORY_AGE_FIELDS) nextSelected[field.key] = next;
setSelected(nextSelected);
}
const chosenCategories = documentFields.filter((f) => selected[f.key]).map((f) => f.key);
const includeAvatars = Boolean(selected.avatar);
const canSubmit = chosenCategories.length > 0 || includeAvatars;
return createPortal(
<div className="modal-backdrop" role="presentation">
<section className="modal command-modal" role="dialog" aria-modal="true" aria-label={"Manually purge storage"}>
<div className="modal-head">
<div>
<div className="eyebrow">{"Limits & Retention"}</div>
<h2>{"Manually purge storage"}</h2>
</div>
<button className="icon-btn" type="button" onClick={onClose} aria-label={"Close"}><X size={15} /></button>
</div>
<div className="command-body">
<p className="env-field-desc">
{"Deletes the file bytes of every document/photo matching the categories below, right now -- independent of the retention mode/age configured above. The message/profile-photo itself is never deleted, only its file; a purged item starts showing as unavailable. Leave \"Created before\" empty to purge everything in the selected categories, regardless of age."}
</p>
<div className="attr-block">
<label className="checkline">
<input type="checkbox" checked={allSelected} onChange={toggleAll} />
{" "}{"Select all"}
</label>
{CATEGORY_AGE_FIELDS.map((field) => (
<label key={field.key} className="checkline">
<input type="checkbox" checked={Boolean(selected[field.key])} onChange={() => toggle(field.key)} />
{" "}{field.label}
</label>
))}
</div>
<label className="duration-field">
<span>{"Created before (optional)"}</span>
<input type="date" value={dateValue} onChange={(event) => setDateValue(event.target.value)} />
<span className="env-field-desc">{"Empty = no age limit, purge everything matching the selected categories."}</span>
</label>
</div>
<div className="modal-actions">
<button className="btn" type="button" onClick={onClose}>{"Close"}</button>
<ActionButton
disabled={!canSubmit}
label={"Purge selected storage"}
tone="danger"
path="/api/actions/storage-manual-purge"
payload={() => ({
categories: chosenCategories,
include_avatars: includeAvatars,
created_before: dateValue ? new Date(dateValue).toISOString() : undefined
})}
/>
</div>
</section>
</div>,
document.body
);
}
function LimitsRetentionSection() {
const [loaded, setLoaded] = useState(false);
const [error, setError] = useState("");
@ -451,6 +539,7 @@ function LimitsRetentionSection() {
const [categoryAgeMinutes, setCategoryAgeMinutes] = useState<Record<string, string>>({});
const [categoryModalOpen, setCategoryModalOpen] = useState(false);
const [evictionEnable, setEvictionEnable] = useState(false);
const [manualPurgeOpen, setManualPurgeOpen] = useState(false);
async function load() {
setError("");
@ -621,6 +710,16 @@ function LimitsRetentionSection() {
</p>
</div>
<div className="attr-block" style={{ marginTop: "1.5em" }}>
<button className="btn danger icon-text" type="button" onClick={() => setManualPurgeOpen(true)}>
<Settings2 size={15} /> {"Manually purge storage..."}
</button>
<span className="env-field-desc">
{"Delete media file bytes right now by hand-picked category and an optional age cutoff, independent of the retention settings above."}
</span>
</div>
{manualPurgeOpen && <ManualPurgeStorageModal onClose={() => setManualPurgeOpen(false)} />}
<div className="gift-table-actions env-save-row">
<ActionButton
tone="warn"

5
cmd/telesrv-admin/web/src/types.ts
View file

@ -753,6 +753,11 @@ export type HostStatsSnapshot = {
MemTotalBytes: number;
DiskFreeBytes: number;
DiskTotalBytes: number;
// False when the disk-space sample itself failed (wrong path, not
// created yet, etc) -- DiskFreeBytes/DiskTotalBytes are stale/zero in
// that case, not "the disk is actually full". Independent of Ready,
// which only covers CPU/memory.
DiskReady: boolean;
Ready: boolean;
};

166
internal/admin/manual_purge_storage_test.go Normal file
View file

@ -0,0 +1,166 @@
package admin
import (
"context"
"testing"
"time"
"telesrv/internal/domain"
)
// fakeStorageService is a minimal StorageService double recording what
// ManualPurgeStorage actually asked it to count/purge, so tests can assert on
// category parsing (including "avatar" never becoming a domain.MediaCategory
// argument) without a real files.Service/Postgres.
type fakeStorageService struct {
countCalls int
purgeCalls int
lastCategories []domain.MediaCategory
lastIncludeAvatars bool
lastBefore *time.Time
countDocs, countPhotos int
purgedDocs, purgedPhotos int
bytesReclaimed int64
countErr, purgeErr error
}
func (f *fakeStorageService) CountManualPurgeCandidates(_ context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time) (int, int, error) {
f.countCalls++
f.lastCategories = categories
f.lastIncludeAvatars = includeAvatars
f.lastBefore = before
return f.countDocs, f.countPhotos, f.countErr
}
func (f *fakeStorageService) ManualPurge(_ context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time, _ int) (int, int, int64, error) {
f.purgeCalls++
f.lastCategories = categories
f.lastIncludeAvatars = includeAvatars
f.lastBefore = before
return f.purgedDocs, f.purgedPhotos, f.bytesReclaimed, f.purgeErr
}
func TestManualPurgeStorageDryRunReportsCounts(t *testing.T) {
ctx := context.Background()
repo := newMemoryCommandRepo()
storage := &fakeStorageService{countDocs: 3, countPhotos: 2}
svc := NewService(Dependencies{Commands: repo, Storage: storage, Now: fixedNow})
req := ManualPurgeStorageRequest{
CommandMeta: CommandMeta{CommandID: "purge-dry-1", Actor: "ops", Reason: "cleanup", DryRun: true},
Categories: []string{"video", "Music"},
}
result, err := svc.ManualPurgeStorage(ctx, req)
if err != nil {
t.Fatalf("ManualPurgeStorage(dry-run): %v", err)
}
if !result.DryRun {
t.Fatalf("result.DryRun = false, want true")
}
if result.Details["would_delete_documents"] != 3 || result.Details["would_delete_photos"] != 2 {
t.Fatalf("details = %+v, want would_delete_documents=3, would_delete_photos=2", result.Details)
}
if storage.countCalls != 1 || storage.purgeCalls != 0 {
t.Fatalf("countCalls=%d purgeCalls=%d, want 1/0 for a dry run", storage.countCalls, storage.purgeCalls)
}
want := map[domain.MediaCategory]bool{domain.MediaCategoryVideo: true, domain.MediaCategoryMusic: true}
if len(storage.lastCategories) != 2 || !want[storage.lastCategories[0]] || !want[storage.lastCategories[1]] {
t.Fatalf("lastCategories = %v, want [Video, Music] (case-insensitively parsed)", storage.lastCategories)
}
if storage.lastBefore != nil {
t.Fatalf("lastBefore = %v, want nil (no created_before supplied)", storage.lastBefore)
}
}
func TestManualPurgeStorageConfirmReportsPurgedCounts(t *testing.T) {
ctx := context.Background()
repo := newMemoryCommandRepo()
storage := &fakeStorageService{purgedDocs: 5, purgedPhotos: 1, bytesReclaimed: 4096}
svc := NewService(Dependencies{Commands: repo, Storage: storage, Now: fixedNow})
before := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
req := ManualPurgeStorageRequest{
CommandMeta: CommandMeta{CommandID: "purge-confirm-1", Actor: "ops", Reason: "cleanup"},
Categories: []string{"file"},
CreatedBefore: &before,
}
result, err := svc.ManualPurgeStorage(ctx, req)
if err != nil {
t.Fatalf("ManualPurgeStorage(confirm): %v", err)
}
if result.Details["purged_documents"] != 5 || result.Details["purged_photos"] != 1 || result.Details["bytes_reclaimed"] != int64(4096) {
t.Fatalf("details = %+v, want purged_documents=5, purged_photos=1, bytes_reclaimed=4096", result.Details)
}
if storage.purgeCalls != 1 || storage.countCalls != 0 {
t.Fatalf("purgeCalls=%d countCalls=%d, want 1/0 for a confirm", storage.purgeCalls, storage.countCalls)
}
if storage.lastBefore == nil || !storage.lastBefore.Equal(before) {
t.Fatalf("lastBefore = %v, want %v", storage.lastBefore, before)
}
}
// TestManualPurgeStorageAvatarIsNotAMediaCategory guards the "avatar" key's
// special handling: it must flip IncludeAvatars and must NEVER appear in the
// categories slice handed to the files.Service layer, since Avatar is not a
// domain.MediaCategory (see files.Service.ManualPurge's doc comment).
func TestManualPurgeStorageAvatarIsNotAMediaCategory(t *testing.T) {
ctx := context.Background()
repo := newMemoryCommandRepo()
storage := &fakeStorageService{}
svc := NewService(Dependencies{Commands: repo, Storage: storage, Now: fixedNow})
req := ManualPurgeStorageRequest{
CommandMeta: CommandMeta{CommandID: "purge-avatar-1", Actor: "ops", Reason: "cleanup", DryRun: true},
Categories: []string{"photo"},
IncludeAvatars: true,
}
if _, err := svc.ManualPurgeStorage(ctx, req); err != nil {
t.Fatalf("ManualPurgeStorage: %v", err)
}
if !storage.lastIncludeAvatars {
t.Fatal("lastIncludeAvatars = false, want true")
}
if len(storage.lastCategories) != 1 || storage.lastCategories[0] != domain.MediaCategoryPhoto {
t.Fatalf("lastCategories = %v, want exactly [Photo]", storage.lastCategories)
}
}
// TestManualPurgeStorageRejectsUnknownCategory guards the admin-boundary
// validation: an unknown category key must be rejected with a clear error,
// not silently dropped, and must never reach the underlying service.
func TestManualPurgeStorageRejectsUnknownCategory(t *testing.T) {
ctx := context.Background()
repo := newMemoryCommandRepo()
storage := &fakeStorageService{}
svc := NewService(Dependencies{Commands: repo, Storage: storage, Now: fixedNow})
req := ManualPurgeStorageRequest{
CommandMeta: CommandMeta{CommandID: "purge-bad-1", Actor: "ops", Reason: "cleanup", DryRun: true},
Categories: []string{"sticker"},
}
if _, err := svc.ManualPurgeStorage(ctx, req); err == nil {
t.Fatal("ManualPurgeStorage with an unknown category = nil error, want an error")
}
if storage.countCalls != 0 {
t.Fatalf("countCalls = %d, want 0 (validation must fail before reaching the service)", storage.countCalls)
}
}
// TestManualPurgeStorageRequiresACategoryOrAvatars guards against a request
// that would purge nothing at all (no categories, avatars not included)
// being silently accepted as a no-op success.
func TestManualPurgeStorageRequiresACategoryOrAvatars(t *testing.T) {
ctx := context.Background()
repo := newMemoryCommandRepo()
storage := &fakeStorageService{}
svc := NewService(Dependencies{Commands: repo, Storage: storage, Now: fixedNow})
req := ManualPurgeStorageRequest{
CommandMeta: CommandMeta{CommandID: "purge-empty-1", Actor: "ops", Reason: "cleanup", DryRun: true},
}
if _, err := svc.ManualPurgeStorage(ctx, req); err == nil {
t.Fatal("ManualPurgeStorage with no categories and no avatars = nil error, want an error")
}
}

123
internal/admin/service.go
View file

@ -63,6 +63,10 @@ const (
ActionAutoCategorizeGifCatalog = "gif_catalog.auto_categorize"
ActionDeleteUncategorizedGifs = "gif_catalog.delete_uncategorized"
ActionDeleteGifCatalogEntry = "gif_catalog.delete"
// Manual storage purge: admin-chosen categories + optional age cutoff,
// independent of the automatic retention sweep's config-derived
// selection. See StorageService's doc comment.
ActionManualPurgeStorage = "storage.manual_purge"
// Collectible (Fragment-style) username lifecycle.
ActionMintCollectibleUsername = "usernames.collectible.mint"
ActionTransferCollectibleUsername = "usernames.collectible.transfer"
@ -337,6 +341,22 @@ type GifCatalogService interface {
AdminDeleteGifCatalogEntry(ctx context.Context, id int64) (bool, error)
}
// StorageService is the admin-console surface over manual storage purge --
// deleting media blob bytes (never the document/photo metadata row) by
// admin-chosen category and an optional age cutoff, independent of the
// automatic retention sweep's config-derived categories/age. See
// internal/app/files.Service.ManualPurge's doc comment for the exact
// deletion semantics (identical to "hard" retention mode).
type StorageService interface {
// CountManualPurgeCandidates is the dry-run preview: an exact count of
// how many documents/photos the selection would purge. before == nil
// means no age filter at all.
CountManualPurgeCandidates(ctx context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time) (docs int, photos int, err error)
// ManualPurge actually deletes the selected media's blob bytes, up to
// limit documents/photos per category/bucket, returning what was purged.
ManualPurge(ctx context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time, limit int) (purgedDocs int, purgedPhotos int, bytesReclaimed int64, err error)
}
// BotService creates bot accounts on behalf of the admin. It mirrors the
// owner-scoped /newbot flow: a bot is a users row (is_bot=true) plus a bots row
// owned by ownerUserID, and the returned token is shown once to the operator.
@ -402,6 +422,7 @@ type Dependencies struct {
Photos AvatarResolver
StickerSets StickerSetsService
GifCatalog GifCatalogService
Storage StorageService
Bots BotService
Emoji EmojiService
Moderation ModerationService
@ -433,6 +454,7 @@ type Service struct {
photos AvatarResolver
stickerSets StickerSetsService
gifCatalog GifCatalogService
storage StorageService
bots BotService
emoji EmojiService
moderation ModerationService
@ -492,6 +514,9 @@ func (s *Service) Configure(deps Dependencies) *Service {
if deps.GifCatalog != nil {
s.gifCatalog = deps.GifCatalog
}
if deps.Storage != nil {
s.storage = deps.Storage
}
if deps.Bots != nil {
s.bots = deps.Bots
}
@ -749,6 +774,22 @@ type DeleteGifCatalogEntryRequest struct {
ID int64 `json:"id"`
}
// ManualPurgeStorageRequest is the manual storage purge action's request.
// Categories are the admin UI's category keys (see manualPurgeCategoryKeys)
// -- "photo", "video", "round_video", "gif", "music", "voice", "file".
// "avatar" is not a category key; it is the separate IncludeAvatars flag,
// since Avatar is not a domain.MediaCategory (see
// files.Service.ManualPurge's doc comment). CreatedBefore is an optional
// ISO-8601/RFC3339 cutoff -- nil (the field omitted or JSON null) means no
// age filter at all: every document/photo in the selected categories is
// purged, regardless of age.
type ManualPurgeStorageRequest struct {
CommandMeta
Categories []string `json:"categories"`
IncludeAvatars bool `json:"include_avatars,omitempty"`
CreatedBefore *time.Time `json:"created_before,omitempty"`
}
type SetAccountFrozenRequest struct {
CommandMeta
UserID int64 `json:"user_id"`
@ -3034,6 +3075,88 @@ func (s *Service) DeleteUncategorizedGifs(ctx context.Context, req DeleteUncateg
})
}
// manualPurgeCategoryKeys maps the admin UI's category keys (see
// CATEGORY_AGE_FIELDS in cmd/telesrv-admin/web/src/pages/StoragePage.tsx) to
// their domain.MediaCategory value. "avatar" is deliberately absent -- it
// maps to ManualPurgeStorageRequest.IncludeAvatars instead, since Avatar is
// not a domain.MediaCategory (see files.Service.ManualPurge's doc comment).
var manualPurgeCategoryKeys = map[string]domain.MediaCategory{
"photo": domain.MediaCategoryPhoto,
"video": domain.MediaCategoryVideo,
"round_video": domain.MediaCategoryRoundVideo,
"gif": domain.MediaCategoryGif,
"music": domain.MediaCategoryMusic,
"voice": domain.MediaCategoryVoice,
"file": domain.MediaCategoryFile,
}
// parseManualPurgeCategories resolves each admin-supplied category key
// (case-insensitive) to its domain.MediaCategory, rejecting an unknown key
// outright rather than silently ignoring it -- an operator who fat-fingers a
// category name deserves an error, not a purge that quietly did less than
// they asked for. Duplicate keys collapse to one entry.
func parseManualPurgeCategories(keys []string) ([]domain.MediaCategory, error) {
categories := make([]domain.MediaCategory, 0, len(keys))
seen := make(map[domain.MediaCategory]bool, len(keys))
for _, raw := range keys {
key := strings.ToLower(strings.TrimSpace(raw))
cat, ok := manualPurgeCategoryKeys[key]
if !ok {
return nil, fmt.Errorf("unknown storage category: %q", raw)
}
if seen[cat] {
continue
}
seen[cat] = true
categories = append(categories, cat)
}
return categories, nil
}
// manualPurgeBatchLimit bounds how many documents/photos a single
// ManualPurgeStorage confirm call purges per category/bucket. An operator
// whose dry-run count exceeds this runs the (idempotent, self-terminating --
// see files.Service.ManualPurge) confirm action again to keep purging.
const manualPurgeBatchLimit = 5000
// ManualPurgeStorage deletes media blob bytes (never the document/photo
// metadata row) by admin-chosen category and an optional "created before"
// cutoff -- the manual counterpart of the automatic hard-retention sweep.
// See StorageService and files.Service.ManualPurge's doc comments for the
// exact semantics.
func (s *Service) ManualPurgeStorage(ctx context.Context, req ManualPurgeStorageRequest) (CommandResult, error) {
if s == nil || s.storage == nil {
return CommandResult{}, fmt.Errorf("storage service is not configured")
}
categories, err := parseManualPurgeCategories(req.Categories)
if err != nil {
return CommandResult{}, err
}
if len(categories) == 0 && !req.IncludeAvatars {
return CommandResult{}, fmt.Errorf("at least one category (or avatars) must be selected")
}
return s.runCommand(ctx, req.CommandMeta, ActionManualPurgeStorage, 0, domain.Peer{}, req, func() (CommandResult, error) {
details := map[string]any{}
if req.DryRun {
// A real count, not just "validated" -- this is a bulk,
// irreversible delete, and an operator confirming it deserves to
// know how much they're about to purge before they do.
docs, photos, err := s.storage.CountManualPurgeCandidates(ctx, categories, req.IncludeAvatars, req.CreatedBefore)
if err != nil {
return CommandResult{}, err
}
details["would_delete_documents"] = docs
details["would_delete_photos"] = photos
return CommandResult{Message: fmt.Sprintf("would purge %d document(s) and %d photo(s)", docs, photos), Details: details}, nil
}
purgedDocs, purgedPhotos, bytesReclaimed, err := s.storage.ManualPurge(ctx, categories, req.IncludeAvatars, req.CreatedBefore, manualPurgeBatchLimit)
details["purged_documents"] = purgedDocs
details["purged_photos"] = purgedPhotos
details["bytes_reclaimed"] = bytesReclaimed
return CommandResult{Message: fmt.Sprintf("purged %d document(s) and %d photo(s)", purgedDocs, purgedPhotos), Details: details}, err
})
}
func (s *Service) DeleteGifCatalogEntry(ctx context.Context, req DeleteGifCatalogEntryRequest) (CommandResult, error) {
if s == nil || s.gifCatalog == nil || req.ID <= 0 {
return CommandResult{}, domain.ErrGifCatalogEntryInvalid

11
internal/adminapi/server.go
View file

@ -83,6 +83,7 @@ type Service interface {
AutoCategorizeGifCatalog(ctx context.Context, req admin.AutoCategorizeGifCatalogRequest) (admin.CommandResult, error)
DeleteUncategorizedGifs(ctx context.Context, req admin.DeleteUncategorizedGifsRequest) (admin.CommandResult, error)
DeleteGifCatalogEntry(ctx context.Context, req admin.DeleteGifCatalogEntryRequest) (admin.CommandResult, error)
ManualPurgeStorage(ctx context.Context, req admin.ManualPurgeStorageRequest) (admin.CommandResult, error)
EmojiAnimation(ctx context.Context, documentID int64) ([]byte, bool, error)
ModerationCases(ctx context.Context, filter domain.ModerationCaseFilter) ([]domain.ModerationCase, error)
ModerationCase(ctx context.Context, caseID int64) (domain.ModerationCaseDetail, bool, error)
@ -218,6 +219,7 @@ func (s *Server) routes() http.Handler {
mux.HandleFunc("POST /v1/gif-catalog/auto-categorize", s.authenticated(s.handleAutoCategorizeGifCatalog))
mux.HandleFunc("POST /v1/gif-catalog/delete-uncategorized", s.authenticated(s.handleDeleteUncategorizedGifs))
mux.HandleFunc("POST /v1/gif-catalog/delete", s.authenticated(s.handleDeleteGifCatalogEntry))
mux.HandleFunc("POST /v1/storage/manual-purge", s.authenticated(s.handleManualPurgeStorage))
mux.HandleFunc("GET /v1/stickers/documents/{id}/animation", s.authenticated(s.handleStickerDocumentAnimation))
mux.HandleFunc("GET /v1/gif-catalog/documents/{id}/preview", s.authenticated(s.handleGifCatalogDocumentPreview))
mux.HandleFunc("GET /v1/emoji/{id}/animation", s.authenticated(s.handleEmojiAnimation))
@ -783,6 +785,15 @@ func (s *Server) handleDeleteUncategorizedGifs(w http.ResponseWriter, r *http.Re
writeCommandResult(w, result, err)
}
func (s *Server) handleManualPurgeStorage(w http.ResponseWriter, r *http.Request) {
var req admin.ManualPurgeStorageRequest
if !decodeJSON(w, r, &req) {
return
}
result, err := s.svc.ManualPurgeStorage(r.Context(), req)
writeCommandResult(w, result, err)
}
func (s *Server) handleDeleteGifCatalogEntry(w http.ResponseWriter, r *http.Request) {
var req admin.DeleteGifCatalogEntryRequest
if !decodeJSON(w, r, &req) {

4
internal/adminapi/server_test.go
View file

@ -476,6 +476,10 @@ func (fakeService) DeleteGifCatalogEntry(_ context.Context, req admin.DeleteGifC
return admin.CommandResult{CommandID: req.CommandID, Status: "completed", DryRun: req.DryRun}, nil
}
func (fakeService) ManualPurgeStorage(_ context.Context, req admin.ManualPurgeStorageRequest) (admin.CommandResult, error) {
return admin.CommandResult{CommandID: req.CommandID, Status: "completed", DryRun: req.DryRun}, nil
}
func (fakeService) EmojiAnimation(context.Context, int64) ([]byte, bool, error) {
return []byte(`{"v":"5.7","w":100,"h":100}`), true, nil
}

186
internal/app/files/retention.go
View file

@ -69,10 +69,19 @@ type mediaRetentionStore interface {
// photo still referenced by a live message is exactly as eligible as an
// orphaned one once it's old enough. The delete methods physically
// remove only the file_blobs row(s)/bytes, never the document/photo
// metadata row -- see DeleteFileBlobsForDocument's doc comment.
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error)
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
// metadata row -- see DeleteFileBlobsForDocument's doc comment. cutoff is
// a *time.Time so it can be nil ("no age filter at all") -- the automatic
// sweep below always passes a real cutoff; the manual purge admin action
// (see ManualPurge) is what actually uses nil.
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff *time.Time, limit int) ([]int64, error)
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff *time.Time, limit int) ([]int64, error)
ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff *time.Time, limit int) ([]int64, error)
// Count* are the exact-count counterparts of the List* queries just
// above, used by CountManualPurgeCandidates's dry-run preview: a
// LIMIT-capped list length is not a real total.
CountDocumentsForHardRetention(ctx context.Context, category domain.MediaCategory, cutoff *time.Time) (int, error)
CountPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error)
CountAvatarPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error)
DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)
@ -193,7 +202,7 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time
continue
}
cutoff := now.Add(-age)
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, cutoff, limit)
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, &cutoff, limit)
if err != nil {
return purged, fmt.Errorf("list documents for hard retention (category %d): %w", cat, err)
}
@ -213,7 +222,7 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time
}
if age := s.categoryRetentionAge(domain.MediaCategoryPhoto); age > 0 {
photoCutoff := now.Add(-age)
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, photoCutoff, limit)
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, &photoCutoff, limit)
if err != nil {
return purged, fmt.Errorf("list photos for hard retention: %w", err)
}
@ -233,7 +242,7 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time
}
if age := s.avatarRetentionAge(); age > 0 {
avatarCutoff := now.Add(-age)
avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, avatarCutoff, limit)
avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, &avatarCutoff, limit)
if err != nil {
return purged, fmt.Errorf("list avatar photos for hard retention: %w", err)
}
@ -254,6 +263,169 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time
return purged, nil
}
// manualPurgeCategories enumerates the domain.MediaCategory values the
// manual purge admin action (ManualPurge/CountManualPurgeCandidates) accepts.
// Unlike hardRetentionDocumentCategories (used by the automatic sweep),
// this includes MediaCategoryPhoto -- an operator explicitly choosing
// categories to purge expects "Photo" to be one of the choices, even though
// photos have no documents.category column and are resolved through the
// dedicated ListPhotoIDsForHardRetentionOlderThan query instead. Avatar is
// deliberately NOT a domain.MediaCategory member and is instead its own
// includeAvatars bool parameter, matching how the automatic sweep already
// splits avatar photos out via avatarRetentionAge/ListAvatarPhotoIDsForHardRetentionOlderThan.
var manualPurgeCategories = map[domain.MediaCategory]bool{
domain.MediaCategoryPhoto: true,
domain.MediaCategoryVideo: true,
domain.MediaCategoryRoundVideo: true,
domain.MediaCategoryGif: true,
domain.MediaCategoryMusic: true,
domain.MediaCategoryVoice: true,
domain.MediaCategoryFile: true,
}
// validateManualPurgeCategories rejects any category outside
// manualPurgeCategories (e.g. MediaCategoryNone or MediaCategoryURL, which
// are not meaningful purge targets) rather than silently ignoring it -- an
// operator who fat-fingers a category deserves an error, not a purge that
// quietly did less than they asked for.
func validateManualPurgeCategories(categories []domain.MediaCategory) error {
for _, cat := range categories {
if !manualPurgeCategories[cat] {
return fmt.Errorf("unsupported manual purge category: %d", cat)
}
}
return nil
}
// CountManualPurgeCandidates is the dry-run counterpart of ManualPurge: an
// exact count of how many documents/photos the given selection would purge,
// without deleting anything. before may be nil, meaning no age filter at all
// (every document/photo in the selected categories is a candidate,
// regardless of created_at) -- unlike the automatic sweep, which always
// filters by a configured age.
func (s *Service) CountManualPurgeCandidates(ctx context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time) (docs int, photos int, err error) {
store, ok := s.media.(mediaRetentionStore)
if !ok {
return 0, 0, nil
}
if err := validateManualPurgeCategories(categories); err != nil {
return 0, 0, err
}
for _, cat := range categories {
if cat == domain.MediaCategoryPhoto {
n, err := store.CountPhotosForHardRetention(ctx, before)
if err != nil {
return docs, photos, fmt.Errorf("count photos for manual purge: %w", err)
}
photos += n
continue
}
n, err := store.CountDocumentsForHardRetention(ctx, cat, before)
if err != nil {
return docs, photos, fmt.Errorf("count documents for manual purge (category %d): %w", cat, err)
}
docs += n
}
if includeAvatars {
n, err := store.CountAvatarPhotosForHardRetention(ctx, before)
if err != nil {
return docs, photos, fmt.Errorf("count avatar photos for manual purge: %w", err)
}
photos += n
}
return docs, photos, nil
}
// ManualPurge is the admin-triggered counterpart of DeleteBlobBytesForMediaOlderThan
// ("hard" retention mode's blob-purge primitive): instead of config-derived
// categories and a configured retention age, the operator explicitly chooses
// which categories to purge and an optional cutoff date. before == nil means
// no age filter at all -- everything matching the chosen categories is
// purged, regardless of how recently it was created. Deletion semantics are
// identical to the automatic sweep: only file_blobs bytes are removed, never
// the documents/photos metadata row, and notifyRetentionPurge still turns any
// message still displaying the purged media into the retention-purge notice.
// limit bounds how many documents/photos are purged per category/bucket in
// this single call (the admin action loops/paginates by calling again if the
// dry-run count exceeds one call's limit).
func (s *Service) ManualPurge(ctx context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time, limit int) (purgedDocs int, purgedPhotos int, bytesReclaimed int64, err error) {
store, ok := s.media.(mediaRetentionStore)
if !ok || limit <= 0 {
return 0, 0, 0, nil
}
if err := validateManualPurgeCategories(categories); err != nil {
return 0, 0, 0, err
}
for _, cat := range categories {
if cat == domain.MediaCategoryPhoto {
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, before, limit)
if err != nil {
return purgedDocs, purgedPhotos, bytesReclaimed, fmt.Errorf("list photos for manual purge: %w", err)
}
for _, id := range photoIDs {
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
if err != nil {
s.log.Warn("manual purge photo blob delete failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
for _, b := range blobs {
bytesReclaimed += b.Size
}
s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
purgedPhotos++
}
continue
}
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, before, limit)
if err != nil {
return purgedDocs, purgedPhotos, bytesReclaimed, fmt.Errorf("list documents for manual purge (category %d): %w", cat, err)
}
for _, id := range docIDs {
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
if err != nil {
s.log.Warn("manual purge document blob delete failed", zap.Int64("document_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
for _, b := range blobs {
bytesReclaimed += b.Size
}
s.notifyRetentionPurge(ctx, domain.MediaKindDocument, id)
purgedDocs++
}
}
if includeAvatars {
avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, before, limit)
if err != nil {
return purgedDocs, purgedPhotos, bytesReclaimed, fmt.Errorf("list avatar photos for manual purge: %w", err)
}
for _, id := range avatarIDs {
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
if err != nil {
s.log.Warn("manual purge avatar photo blob delete failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
for _, b := range blobs {
bytesReclaimed += b.Size
}
s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
purgedPhotos++
}
}
return purgedDocs, purgedPhotos, bytesReclaimed, nil
}
// EvictOldestMediaOverBudget implements maintenance.StorageEvictionStore
// (TELESRV_STORAGE_EVICTION_ENABLE): once total physical blob bytes
// (SumFileBlobBytes) exceed TELESRV_STORAGE_MAX_TOTAL_BYTES, purges the

16
internal/app/files/retention_category_zero_test.go
View file

@ -24,21 +24,31 @@ type fakeCategorySweepStore struct {
queriedAvatars bool
}
func (f *fakeCategorySweepStore) ListDocumentIDsForHardRetentionOlderThan(_ context.Context, category domain.MediaCategory, _ time.Time, _ int) ([]int64, error) {
func (f *fakeCategorySweepStore) ListDocumentIDsForHardRetentionOlderThan(_ context.Context, category domain.MediaCategory, _ *time.Time, _ int) ([]int64, error) {
f.queriedDocCategories = append(f.queriedDocCategories, category)
return nil, nil
}
func (f *fakeCategorySweepStore) ListPhotoIDsForHardRetentionOlderThan(context.Context, time.Time, int) ([]int64, error) {
func (f *fakeCategorySweepStore) ListPhotoIDsForHardRetentionOlderThan(context.Context, *time.Time, int) ([]int64, error) {
f.queriedPhotos = true
return nil, nil
}
func (f *fakeCategorySweepStore) ListAvatarPhotoIDsForHardRetentionOlderThan(context.Context, time.Time, int) ([]int64, error) {
func (f *fakeCategorySweepStore) ListAvatarPhotoIDsForHardRetentionOlderThan(context.Context, *time.Time, int) ([]int64, error) {
f.queriedAvatars = true
return nil, nil
}
func (f *fakeCategorySweepStore) CountDocumentsForHardRetention(context.Context, domain.MediaCategory, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeCategorySweepStore) CountPhotosForHardRetention(context.Context, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeCategorySweepStore) CountAvatarPhotosForHardRetention(context.Context, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeCategorySweepStore) ListOrphanedDocumentIDsOlderThan(_ context.Context, category domain.MediaCategory, _ time.Time, _ int) ([]int64, error) {
f.queriedDocCategories = append(f.queriedDocCategories, category)
return nil, nil

314
internal/app/files/retention_manual_purge_test.go Normal file
View file

@ -0,0 +1,314 @@
package files
import (
"context"
"fmt"
"testing"
"time"
"go.uber.org/zap"
"telesrv/internal/domain"
"telesrv/internal/store"
)
// fakeManualPurgeStore is a mediaRetentionStore fake that mimics the SQL
// nullable-cutoff contract in Go: a nil cutoff matches every candidate
// regardless of its createdAt, a non-nil cutoff only matches candidates
// strictly older than it -- same semantics as the real
// ListDocumentIDsForHardRetentionOlderThan/ListPhotoIDsForHardRetentionOlderThan
// queries (see internal/store/postgres/queries/media.sql). Embeds a nil
// store.MediaStore so it satisfies that (large) interface for free, same
// trick as fakeCategorySweepStore in retention_category_zero_test.go.
type fakeManualPurgeStore struct {
store.MediaStore
docsByCategory map[domain.MediaCategory][]int64
docCreatedAt map[int64]time.Time
photos []int64
photoCreatedAt map[int64]time.Time
avatarPhotos []int64
avatarCreated map[int64]time.Time
deletedDocs []int64
deletedPhotos []int64
}
func matchCutoff(createdAt time.Time, cutoff *time.Time) bool {
if cutoff == nil {
return true
}
return createdAt.Before(*cutoff)
}
func (f *fakeManualPurgeStore) ListDocumentIDsForHardRetentionOlderThan(_ context.Context, category domain.MediaCategory, cutoff *time.Time, limit int) ([]int64, error) {
var out []int64
for _, id := range f.docsByCategory[category] {
if !matchCutoff(f.docCreatedAt[id], cutoff) {
continue
}
out = append(out, id)
if len(out) >= limit {
break
}
}
return out, nil
}
func (f *fakeManualPurgeStore) CountDocumentsForHardRetention(ctx context.Context, category domain.MediaCategory, cutoff *time.Time) (int, error) {
ids, err := f.ListDocumentIDsForHardRetentionOlderThan(ctx, category, cutoff, len(f.docsByCategory[category])+1)
return len(ids), err
}
func (f *fakeManualPurgeStore) ListPhotoIDsForHardRetentionOlderThan(_ context.Context, cutoff *time.Time, limit int) ([]int64, error) {
var out []int64
for _, id := range f.photos {
if !matchCutoff(f.photoCreatedAt[id], cutoff) {
continue
}
out = append(out, id)
if len(out) >= limit {
break
}
}
return out, nil
}
func (f *fakeManualPurgeStore) CountPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error) {
ids, err := f.ListPhotoIDsForHardRetentionOlderThan(ctx, cutoff, len(f.photos)+1)
return len(ids), err
}
func (f *fakeManualPurgeStore) ListAvatarPhotoIDsForHardRetentionOlderThan(_ context.Context, cutoff *time.Time, limit int) ([]int64, error) {
var out []int64
for _, id := range f.avatarPhotos {
if !matchCutoff(f.avatarCreated[id], cutoff) {
continue
}
out = append(out, id)
if len(out) >= limit {
break
}
}
return out, nil
}
func (f *fakeManualPurgeStore) CountAvatarPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error) {
ids, err := f.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, cutoff, len(f.avatarPhotos)+1)
return len(ids), err
}
func (f *fakeManualPurgeStore) DeleteFileBlobsForDocument(_ context.Context, id int64) ([]domain.FileBlob, error) {
f.deletedDocs = append(f.deletedDocs, id)
return []domain.FileBlob{{
LocationKey: fmt.Sprintf("doc:%d", id), Backend: domain.MediaBackendLocalFS,
ObjectKey: fmt.Sprintf("obj-doc-%d", id), Size: 100,
}}, nil
}
func (f *fakeManualPurgeStore) DeleteFileBlobsForPhoto(_ context.Context, id int64) ([]domain.FileBlob, error) {
f.deletedPhotos = append(f.deletedPhotos, id)
return []domain.FileBlob{{
LocationKey: fmt.Sprintf("photo:%d", id), Backend: domain.MediaBackendLocalFS,
ObjectKey: fmt.Sprintf("obj-photo-%d", id), Size: 50,
}}, nil
}
// The rest of mediaRetentionStore isn't exercised by ManualPurge/
// CountManualPurgeCandidates -- stub it out, same as
// fakeCategorySweepStore.
func (f *fakeManualPurgeStore) ListOrphanedDocumentIDsOlderThan(context.Context, domain.MediaCategory, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) ListOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) ListAvatarOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
return nil, nil
}
// CountFileBlobRefs deliberately returns 1 (not 0): ManualPurge's deleted
// blobs must still be accounted for (deleteOrphanedBlobs's cache eviction),
// but this fake has no real BlobBackend wired up, so it reports the blob as
// still referenced elsewhere -- skipping the physical backend.Delete call
// that would otherwise need a working s.blobs.
func (f *fakeManualPurgeStore) CountFileBlobRefs(context.Context, string, string) (int, error) {
return 1, nil
}
func (f *fakeManualPurgeStore) DeleteDocumentAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) DeletePhotoAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) OrphanDocumentIfUnreferenced(context.Context, int64) (bool, error) {
return false, nil
}
func (f *fakeManualPurgeStore) SumFileBlobBytes(context.Context) (int64, error) { return 0, nil }
func (f *fakeManualPurgeStore) ListOldestMediaForEviction(context.Context, int) ([]domain.EvictionCandidate, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) ListMediaReferences(context.Context, domain.MediaKind, int64) ([]domain.MediaReference, error) {
return nil, nil
}
func newManualPurgeTestService(fake *fakeManualPurgeStore) *Service {
return &Service{
media: fake,
log: zap.NewNop(),
blobCache: newBlobMetaCache(64),
byteCache: newBlobBytesCache(1 << 20),
}
}
// TestManualPurgeOnlyTouchesSelectedCategory guards ManualPurge's per-category
// scoping: purging just "Video" must never touch a Music document sitting in
// the same fake store, mirroring the real query's category filter (see
// TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory in
// internal/store/postgres).
func TestManualPurgeOnlyTouchesSelectedCategory(t *testing.T) {
fake := &fakeManualPurgeStore{
docsByCategory: map[domain.MediaCategory][]int64{
domain.MediaCategoryVideo: {1},
domain.MediaCategoryMusic: {2},
},
docCreatedAt: map[int64]time.Time{
1: time.Now().Add(-100 * 24 * time.Hour),
2: time.Now().Add(-100 * 24 * time.Hour),
},
}
s := newManualPurgeTestService(fake)
purgedDocs, purgedPhotos, bytes, err := s.ManualPurge(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryVideo}, false, nil, 100)
if err != nil {
t.Fatalf("ManualPurge: %v", err)
}
if purgedDocs != 1 || purgedPhotos != 0 {
t.Fatalf("purgedDocs=%d purgedPhotos=%d, want 1/0", purgedDocs, purgedPhotos)
}
if bytes != 100 {
t.Fatalf("bytesReclaimed = %d, want 100", bytes)
}
if len(fake.deletedDocs) != 1 || fake.deletedDocs[0] != 1 {
t.Fatalf("deletedDocs = %v, want only [1] (the Video document)", fake.deletedDocs)
}
}
// TestManualPurgeNilBeforeIgnoresAge is the unit-level counterpart of the
// Postgres integration test TestManualPurgeNilCutoffIgnoresAge: a document
// created "just now" must still be a purge candidate when before is nil,
// unlike the automatic sweep which always filters by a configured age.
func TestManualPurgeNilBeforeIgnoresAge(t *testing.T) {
fake := &fakeManualPurgeStore{
docsByCategory: map[domain.MediaCategory][]int64{
domain.MediaCategoryFile: {42},
},
docCreatedAt: map[int64]time.Time{
42: time.Now(), // created just now
},
}
s := newManualPurgeTestService(fake)
// Dry-run: nil before must count the fresh document.
docs, photos, err := s.CountManualPurgeCandidates(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryFile}, false, nil)
if err != nil {
t.Fatalf("CountManualPurgeCandidates(nil before): %v", err)
}
if docs != 1 || photos != 0 {
t.Fatalf("CountManualPurgeCandidates(nil before) = (%d, %d), want (1, 0)", docs, photos)
}
// A real cutoff strictly before "now" must exclude the fresh document.
past := time.Now().Add(-time.Hour)
docs, photos, err = s.CountManualPurgeCandidates(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryFile}, false, &past)
if err != nil {
t.Fatalf("CountManualPurgeCandidates(past before): %v", err)
}
if docs != 0 || photos != 0 {
t.Fatalf("CountManualPurgeCandidates(past before) = (%d, %d), want (0, 0)", docs, photos)
}
// Confirm: nil before must actually purge the fresh document.
purgedDocs, _, _, err := s.ManualPurge(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryFile}, false, nil, 100)
if err != nil {
t.Fatalf("ManualPurge(nil before): %v", err)
}
if purgedDocs != 1 {
t.Fatalf("ManualPurge(nil before) purgedDocs = %d, want 1", purgedDocs)
}
}
// TestManualPurgePhotoCategoryUsesPhotoQuery guards that selecting the
// "Photo" category (unlike the other categories, which are documents.category
// buckets) routes through the dedicated photo query rather than being
// silently dropped or misrouted to the document path.
func TestManualPurgePhotoCategoryUsesPhotoQuery(t *testing.T) {
fake := &fakeManualPurgeStore{
photos: []int64{7},
photoCreatedAt: map[int64]time.Time{7: time.Now()},
}
s := newManualPurgeTestService(fake)
purgedDocs, purgedPhotos, _, err := s.ManualPurge(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryPhoto}, false, nil, 100)
if err != nil {
t.Fatalf("ManualPurge: %v", err)
}
if purgedDocs != 0 || purgedPhotos != 1 {
t.Fatalf("purgedDocs=%d purgedPhotos=%d, want 0/1", purgedDocs, purgedPhotos)
}
if len(fake.deletedPhotos) != 1 || fake.deletedPhotos[0] != 7 {
t.Fatalf("deletedPhotos = %v, want only [7]", fake.deletedPhotos)
}
}
// TestManualPurgeIncludeAvatarsIsIndependentOfCategories guards that
// includeAvatars purges avatar photos even when Photo is not among the
// selected categories (and vice versa: selecting Photo alone must not also
// sweep avatars) -- the same split the automatic sweep already has between
// categoryRetentionAge(Photo) and avatarRetentionAge().
func TestManualPurgeIncludeAvatarsIsIndependentOfCategories(t *testing.T) {
fake := &fakeManualPurgeStore{
avatarPhotos: []int64{9},
avatarCreated: map[int64]time.Time{9: time.Now()},
}
s := newManualPurgeTestService(fake)
// No categories selected, but includeAvatars=true must still purge it.
purgedDocs, purgedPhotos, _, err := s.ManualPurge(context.Background(), nil, true, nil, 100)
if err != nil {
t.Fatalf("ManualPurge: %v", err)
}
if purgedDocs != 0 || purgedPhotos != 1 {
t.Fatalf("purgedDocs=%d purgedPhotos=%d, want 0/1", purgedDocs, purgedPhotos)
}
if len(fake.deletedPhotos) != 1 || fake.deletedPhotos[0] != 9 {
t.Fatalf("deletedPhotos = %v, want only [9]", fake.deletedPhotos)
}
}
// TestManualPurgeRejectsUnknownCategory guards validateManualPurgeCategories:
// an operator who fat-fingers a category should get an error, not a purge
// that quietly did less than they asked for.
func TestManualPurgeRejectsUnknownCategory(t *testing.T) {
fake := &fakeManualPurgeStore{}
s := newManualPurgeTestService(fake)
invalid := domain.MediaCategory(99)
if _, _, _, err := s.ManualPurge(context.Background(), []domain.MediaCategory{invalid}, false, nil, 100); err == nil {
t.Fatal("ManualPurge with an unsupported category = nil error, want an error")
}
if _, _, err := s.CountManualPurgeCandidates(context.Background(), []domain.MediaCategory{invalid}, false, nil); err == nil {
t.Fatal("CountManualPurgeCandidates with an unsupported category = nil error, want an error")
}
// MediaCategoryNone is a real category, but not one ManualPurge accepts
// (see manualPurgeCategories's doc comment) -- it must be rejected too,
// not silently treated as "match nothing"/"match everything".
if _, _, _, err := s.ManualPurge(context.Background(), []domain.MediaCategory{domain.MediaCategoryNone}, false, nil, 100); err == nil {
t.Fatal("ManualPurge with MediaCategoryNone = nil error, want an error")
}
}

52
internal/app/files/secret_chat_files.go Normal file
View file

@ -0,0 +1,52 @@
package files
import (
"context"
"fmt"
"go.uber.org/zap"
"telesrv/internal/domain"
)
// DeleteEncryptedFileBlob deletes a secret-chat encrypted file's blob bytes
// (location_key "enc:<id>") once the recipient has fully downloaded it --
// see internal/rpc/upload.go's onUploadGetFile, which calls this after a
// getFile response whose bytes reach the end of the file. Secret chats are
// single-device on both ends (no multi-device sync, unlike ordinary chats),
// so once the one recipient device that will ever ask for this ciphertext
// has it, the server has no further reason to keep it -- unlike ordinary
// message media, there is no "another device might still need to fetch
// this" case to protect against.
//
// No-op unless TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD is enabled
// (WithSecretChatDeleteFileAfterDownload, default true). Best-effort: the
// caller is a fire-and-forget goroutine off the download response, so
// errors are only logged, never propagated -- a failed cleanup here must
// never turn into a failed or delayed file download.
func (s *Service) DeleteEncryptedFileBlob(ctx context.Context, locationKey string) error {
if !s.secretChatDeleteFileAfterDownload {
return nil
}
store, ok := s.media.(mediaRetentionStore)
if !ok {
return nil
}
blob, found, err := s.media.GetFileBlob(ctx, locationKey)
if err != nil {
return fmt.Errorf("get encrypted file blob: %w", err)
}
if !found {
// Already deleted (e.g. a racing duplicate getFile request for the
// same final chunk) -- nothing left to do.
return nil
}
if err := s.media.DeleteFileBlobRow(ctx, locationKey); err != nil {
return fmt.Errorf("delete encrypted file blob row: %w", err)
}
s.blobCache.delete(locationKey)
s.deleteOrphanedBlobs(ctx, store, []domain.FileBlob{blob})
s.log.Info("secret chat file deleted after download",
zap.String("location_key", locationKey), zap.Int64("size", blob.Size))
return nil
}

185
internal/app/files/secret_chat_files_test.go Normal file
View file

@ -0,0 +1,185 @@
package files
import (
"context"
"errors"
"testing"
"time"
"go.uber.org/zap"
"telesrv/internal/domain"
"telesrv/internal/store"
)
// fakeEncryptedBlobStore is a minimal stateful store.MediaStore double for
// DeleteEncryptedFileBlob: a real in-memory map of location_key -> FileBlob
// (so GetFileBlob/DeleteFileBlobRow actually observe each other's effects).
// CountFileBlobRefs always reports "still referenced" so deleteOrphanedBlobs
// never reaches its physical-backend-delete path -- this test's Service has
// no real blob backend configured, and the point here is the file_blobs row
// deletion, which DeleteFileBlobRow already covers directly. Embeds a nil
// store.MediaStore for the rest of that large interface's methods, which
// this test never calls.
type fakeEncryptedBlobStore struct {
store.MediaStore
blobs map[string]domain.FileBlob
deletedRows []string
countRefsErr error
getFileBlobErr error
}
func (f *fakeEncryptedBlobStore) GetFileBlob(_ context.Context, key string) (domain.FileBlob, bool, error) {
if f.getFileBlobErr != nil {
return domain.FileBlob{}, false, f.getFileBlobErr
}
b, ok := f.blobs[key]
return b, ok, nil
}
func (f *fakeEncryptedBlobStore) DeleteFileBlobRow(_ context.Context, key string) error {
f.deletedRows = append(f.deletedRows, key)
delete(f.blobs, key)
return nil
}
func (f *fakeEncryptedBlobStore) CountFileBlobRefs(context.Context, string, string) (int, error) {
if f.countRefsErr != nil {
return 0, f.countRefsErr
}
// Reported as still-referenced (>0) so deleteOrphanedBlobs's physical
// backend-delete path (this test's Service has no real backend
// configured) is never reached -- this test only cares about the
// file_blobs row itself, which DeleteFileBlobRow already removed by the
// time deleteOrphanedBlobs runs.
return 1, nil
}
// The rest of mediaRetentionStore's methods aren't part of store.MediaStore
// (kept out of the hot RPC-facing interface on purpose -- see that
// interface's own doc comment), so embedding store.MediaStore alone doesn't
// satisfy the type assertion DeleteEncryptedFileBlob performs. Stub them
// out; this test never exercises them.
func (f *fakeEncryptedBlobStore) ListOrphanedDocumentIDsOlderThan(context.Context, domain.MediaCategory, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListAvatarOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) DeleteDocumentAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) DeletePhotoAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) OrphanDocumentIfUnreferenced(context.Context, int64) (bool, error) {
return false, nil
}
func (f *fakeEncryptedBlobStore) ListDocumentIDsForHardRetentionOlderThan(context.Context, domain.MediaCategory, *time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListPhotoIDsForHardRetentionOlderThan(context.Context, *time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListAvatarPhotoIDsForHardRetentionOlderThan(context.Context, *time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) CountDocumentsForHardRetention(context.Context, domain.MediaCategory, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeEncryptedBlobStore) CountPhotosForHardRetention(context.Context, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeEncryptedBlobStore) CountAvatarPhotosForHardRetention(context.Context, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeEncryptedBlobStore) DeleteFileBlobsForDocument(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) DeleteFileBlobsForPhoto(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) SumFileBlobBytes(context.Context) (int64, error) {
return 0, nil
}
func (f *fakeEncryptedBlobStore) ListOldestMediaForEviction(context.Context, int) ([]domain.EvictionCandidate, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListMediaReferences(context.Context, domain.MediaKind, int64) ([]domain.MediaReference, error) {
return nil, nil
}
func newTestServiceForSecretChatFiles(media store.MediaStore, enabled bool) *Service {
return &Service{
media: media,
log: zap.NewNop(),
blobCache: newBlobMetaCache(8),
secretChatDeleteFileAfterDownload: enabled,
}
}
func TestDeleteEncryptedFileBlobNoOpWhenDisabled(t *testing.T) {
fake := &fakeEncryptedBlobStore{blobs: map[string]domain.FileBlob{
"enc:1": {LocationKey: "enc:1", Backend: domain.MediaBackendS3, ObjectKey: "obj-1", Size: 42},
}}
s := newTestServiceForSecretChatFiles(fake, false)
if err := s.DeleteEncryptedFileBlob(context.Background(), "enc:1"); err != nil {
t.Fatalf("DeleteEncryptedFileBlob: %v", err)
}
if len(fake.deletedRows) != 0 {
t.Fatalf("deleted rows = %v, want none (feature disabled)", fake.deletedRows)
}
if _, ok := fake.blobs["enc:1"]; !ok {
t.Fatal("blob was removed from the store despite the feature being disabled")
}
}
func TestDeleteEncryptedFileBlobDeletesWhenEnabled(t *testing.T) {
fake := &fakeEncryptedBlobStore{blobs: map[string]domain.FileBlob{
"enc:1": {LocationKey: "enc:1", Backend: domain.MediaBackendS3, ObjectKey: "obj-1", Size: 42},
}}
s := newTestServiceForSecretChatFiles(fake, true)
if err := s.DeleteEncryptedFileBlob(context.Background(), "enc:1"); err != nil {
t.Fatalf("DeleteEncryptedFileBlob: %v", err)
}
if len(fake.deletedRows) != 1 || fake.deletedRows[0] != "enc:1" {
t.Fatalf("deleted rows = %v, want [enc:1]", fake.deletedRows)
}
if _, ok := fake.blobs["enc:1"]; ok {
t.Fatal("blob row still present after DeleteEncryptedFileBlob")
}
}
func TestDeleteEncryptedFileBlobNoOpWhenAlreadyGone(t *testing.T) {
fake := &fakeEncryptedBlobStore{blobs: map[string]domain.FileBlob{}}
s := newTestServiceForSecretChatFiles(fake, true)
// A racing duplicate final-chunk request for the same file: the blob is
// already deleted. Must not error.
if err := s.DeleteEncryptedFileBlob(context.Background(), "enc:404"); err != nil {
t.Fatalf("DeleteEncryptedFileBlob for a missing blob: %v", err)
}
if len(fake.deletedRows) != 0 {
t.Fatalf("deleted rows = %v, want none", fake.deletedRows)
}
}
func TestDeleteEncryptedFileBlobPropagatesLookupError(t *testing.T) {
fake := &fakeEncryptedBlobStore{
blobs: map[string]domain.FileBlob{"enc:1": {LocationKey: "enc:1"}},
getFileBlobErr: errors.New("db unavailable"),
}
s := newTestServiceForSecretChatFiles(fake, true)
if err := s.DeleteEncryptedFileBlob(context.Background(), "enc:1"); err == nil {
t.Fatal("DeleteEncryptedFileBlob = nil error, want the underlying lookup failure surfaced")
}
if len(fake.deletedRows) != 0 {
t.Fatalf("deleted rows = %v, want none (lookup failed before any delete)", fake.deletedRows)
}
}

7
internal/app/files/seed_test.go
View file

@ -157,6 +157,13 @@ func (f *fakeMediaStore) GetFileBlob(_ context.Context, key string) (domain.File
return b, ok, nil
}
func (f *fakeMediaStore) DeleteFileBlobRow(_ context.Context, key string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.blobs, key)
return nil
}
func (f *fakeMediaStore) GetFileBlobs(_ context.Context, keys []string) (map[string]domain.FileBlob, error) {
f.mu.Lock()
defer f.mu.Unlock()

16
internal/app/files/service.go
View file

@ -112,6 +112,10 @@ type Service struct {
// EvictOldestMediaOverBudget as the active-eviction trigger threshold
// (<=0 disables eviction regardless of TELESRV_STORAGE_EVICTION_ENABLE).
storageMaxTotalBytes int64
// secretChatDeleteFileAfterDownload is
// TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD -- see
// DeleteEncryptedFileBlob's doc comment.
secretChatDeleteFileAfterDownload bool
// retentionNotifyMu guards retentionMessages/retentionChannels: they are
// set post-construction (see SetRetentionPurgeNotifier) from
@ -242,6 +246,18 @@ func WithStorageMaxTotalBytes(maxBytes int64) Option {
}
}
// WithSecretChatDeleteFileAfterDownload enables DeleteEncryptedFileBlob's
// actual deletion (TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD). Default
// true -- see that env var's doc comment in internal/config/config.go for
// why (secret-chat blobs have no other cleanup path at all otherwise).
// Passing false opts back out for a self-hoster who wants the ciphertext
// kept around regardless.
func WithSecretChatDeleteFileAfterDownload(enabled bool) Option {
return func(s *Service) {
s.secretChatDeleteFileAfterDownload = enabled
}
}
// WithGifSeedDir records the gif seed directory (cfg.GifSeedDir) so
// AdminDeleteUncategorizedGifs can remove a seed-imported entry's source
// file alongside its DB row -- see the field's doc comment for why that

14
internal/config/config.go
View file

@ -383,6 +383,19 @@ type Config struct {
// StorageMaxTotalBytes has meant so far (block new uploads only ->
// block-and-reclaim from existing ones too).
StorageEvictionEnable bool
// SecretChatDeleteFileAfterDownload deletes a secret-chat encrypted
// file's blob bytes (location_key "enc:<id>") as soon as the recipient
// has fully downloaded it. Secret chats are single-device on both ends
// (no multi-device sync), so once delivered there is no other device
// that will ever ask for it again -- unlike ordinary message media,
// which stays downloadable for re-fetch/sync/forward. Default true:
// none of the automatic retention/eviction machinery touches these
// blobs at all (they have no documents/photos row), so leaving this off
// means they simply accumulate forever with no way to reclaim the
// space -- an explicit TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD=false
// opts back out for a self-hoster who wants the ciphertext kept around
// regardless.
SecretChatDeleteFileAfterDownload bool
// StickerSeedDir 是 reaction / sticker 资源种子目录（导入到 documents/sticker_sets + blob）。
StickerSeedDir string
// StickerSeedMaxSets 限制导入的常规贴纸集数量（避免启动时导入过多包），<=0 表示不限。
@ -1009,6 +1022,7 @@ func Load() (Config, error) {
StorageRetentionMaxAgeByCategory: storageRetentionMaxAgeByCategoryFromEnv(envDurationOr),
StorageRetentionMaxAgeAvatar: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR", 0),
StorageEvictionEnable: envBoolOr("TELESRV_STORAGE_EVICTION_ENABLE", false),
SecretChatDeleteFileAfterDownload: envBoolOr("TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD", true),
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"),

51
internal/hoststats/hoststats.go
View file

@ -7,6 +7,8 @@ package hoststats
import (
"context"
"os"
"path/filepath"
"sync"
"time"
)
@ -14,12 +16,18 @@ import (
// Snapshot is the last successfully sampled host-resource reading. Ready is
// false until the first sample completes, so callers can distinguish "0% CPU"
// from "no data yet" instead of rendering a misleading zero on startup.
// DiskReady is a separate flag: disk stats are sampled from a configured
// path (see NewPoller) that can fail independently of CPU/memory sampling
// (wrong working directory, path not created yet, etc) -- without it, a
// failed disk read looked identical to "this server's disk is completely
// full" (0 free bytes) instead of "we don't have a reading right now".
type Snapshot struct {
CPUPercent float64
MemUsedBytes int64
MemTotalBytes int64
DiskFreeBytes int64
DiskTotalBytes int64
DiskReady bool
Ready bool
}
@ -29,6 +37,10 @@ type Snapshot struct {
// for the local blob-storage free-space guard.
type Poller struct {
diskPath string
// diskFreeBytesFn defaults to the platform diskFreeBytes function;
// overridable in tests to simulate a failing/succeeding disk read
// without touching the real filesystem/OS call.
diskFreeBytesFn func(path string) (free, total int64, err error)
mu sync.RWMutex
snap Snapshot
@ -38,12 +50,31 @@ type Poller struct {
// NewPoller creates a poller that reports free/total disk space for the
// filesystem containing diskPath (pass the server's data/blob directory, or
// "." if it doesn't matter which volume).
// "." if it doesn't matter which volume). diskPath is resolved to an
// absolute path up front (a relative path depends on the process's current
// directory, which callers shouldn't have to reason about here) and, if it
// doesn't exist yet -- e.g. an S3-backend deployment whose local blob
// staging directory is only created on first upload -- walked up to the
// nearest existing ancestor, since GetDiskFreeSpaceEx/statfs need a real
// path and every ancestor is on the same volume anyway.
func NewPoller(diskPath string) *Poller {
if diskPath == "" {
diskPath = "."
}
return &Poller{diskPath: diskPath}
if abs, err := filepath.Abs(diskPath); err == nil {
diskPath = abs
}
for {
if _, err := os.Stat(diskPath); err == nil {
break
}
parent := filepath.Dir(diskPath)
if parent == diskPath {
break
}
diskPath = parent
}
return &Poller{diskPath: diskPath, diskFreeBytesFn: diskFreeBytes}
}
// Snapshot returns the last sample. Safe to call concurrently with Run.
@ -72,13 +103,27 @@ func (p *Poller) Run(ctx context.Context, interval time.Duration) {
}
func (p *Poller) sampleOnce() {
p.mu.RLock()
prevFree, prevTotal := p.snap.DiskFreeBytes, p.snap.DiskTotalBytes
p.mu.RUnlock()
var snap Snapshot
snap.CPUPercent = p.cpu.sample()
if used, total, err := memStats(); err == nil {
snap.MemUsedBytes, snap.MemTotalBytes = used, total
}
if free, total, err := diskFreeBytes(p.diskPath); err == nil {
if free, total, err := p.diskFreeBytesFn(p.diskPath); err == nil {
snap.DiskFreeBytes, snap.DiskTotalBytes = free, total
snap.DiskReady = true
} else {
// Keep the last known-good byte values stored (harmless, and a
// reasonable fallback for any future caller that wants "last known"
// over nothing) but DiskReady reflects THIS sample, not a stale one
// -- a failure must show as "no current reading", not silently keep
// claiming Ready while quietly reusing old numbers forever if the
// underlying path became permanently unreadable.
snap.DiskFreeBytes, snap.DiskTotalBytes = prevFree, prevTotal
snap.DiskReady = false
}
snap.Ready = true

106
internal/hoststats/hoststats_test.go Normal file
View file

@ -0,0 +1,106 @@
package hoststats
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
)
// TestSampleOncePreservesLastGoodDiskReadingOnFailure guards the bug
// reported live: a failed disk-space sample used to silently reset
// DiskFreeBytes/DiskTotalBytes to 0 while still marking the overall
// snapshot Ready -- rendering as "0 bytes free" (indistinguishable from an
// actually-full disk) instead of "no reading right now". A failed sample
// must keep the last known-good reading and report DiskReady=false.
func TestSampleOncePreservesLastGoodDiskReadingOnFailure(t *testing.T) {
p := &Poller{diskFreeBytesFn: func(string) (int64, int64, error) {
return 1234, 5678, nil
}}
p.sampleOnce()
first := p.Snapshot()
if !first.DiskReady || first.DiskFreeBytes != 1234 || first.DiskTotalBytes != 5678 {
t.Fatalf("first snapshot = %+v, want a successful disk reading", first)
}
p.diskFreeBytesFn = func(string) (int64, int64, error) {
return 0, 0, errors.New("disk stat failed")
}
p.sampleOnce()
second := p.Snapshot()
if second.DiskReady {
t.Fatal("DiskReady = true after a failed sample, want false")
}
if second.DiskFreeBytes != 1234 || second.DiskTotalBytes != 5678 {
t.Fatalf("disk fields after failed sample = (%d, %d), want the preserved (1234, 5678)", second.DiskFreeBytes, second.DiskTotalBytes)
}
// CPU/memory sampling must still complete and mark Ready, independent
// of the disk failure.
if !second.Ready {
t.Fatal("Ready = false after a disk-only failure, want true (CPU/mem still sampled)")
}
}
// TestSampleOnceNeverReadyWithoutAnyPriorSuccess confirms a disk read that
// has NEVER succeeded (not just failed after a prior success) still reports
// DiskReady=false with zero-value fields, not a fabricated 0-bytes-free
// reading.
func TestSampleOnceNeverReadyWithoutAnyPriorSuccess(t *testing.T) {
p := &Poller{diskFreeBytesFn: func(string) (int64, int64, error) {
return 0, 0, errors.New("never worked")
}}
p.sampleOnce()
snap := p.Snapshot()
if snap.DiskReady {
t.Fatal("DiskReady = true with no successful sample ever, want false")
}
if !snap.Ready {
t.Fatal("Ready = false, want true (CPU/mem sampling is independent of disk)")
}
}
// TestNewPollerWalksUpToNearestExistingAncestor guards the other half of the
// fix: a configured disk path that doesn't exist yet (e.g. an S3-backend
// deployment's local blob staging directory, only created on first upload)
// must not permanently break disk stats -- NewPoller walks up to the
// nearest existing ancestor instead of handing GetDiskFreeSpaceEx/statfs a
// path they will always fail to stat.
func TestNewPollerWalksUpToNearestExistingAncestor(t *testing.T) {
tmp := t.TempDir()
missing := filepath.Join(tmp, "not-created-yet", "nested", "deeper")
p := NewPoller(missing)
if p.diskPath != tmp {
t.Fatalf("resolved disk path = %q, want the nearest existing ancestor %q", p.diskPath, tmp)
}
if _, err := os.Stat(p.diskPath); err != nil {
t.Fatalf("resolved disk path %q does not exist: %v", p.diskPath, err)
}
}
// TestNewPollerResolvesRelativePathToAbsolute confirms a relative diskPath
// (as configured today, e.g. "data/blobs") no longer silently depends on
// whatever the process's current directory happens to be at NewPoller time.
func TestNewPollerResolvesRelativePathToAbsolute(t *testing.T) {
p := NewPoller(".")
if !filepath.IsAbs(p.diskPath) {
t.Fatalf("resolved disk path %q is not absolute", p.diskPath)
}
}
// TestRunSamplesImmediatelyOnStart confirms Run's documented behavior
// (sampleOnce before entering the ticker loop) without relying on any
// ticker firing or a busy-wait: cancel the context right away and check the
// synchronous first sample already landed by the time Run returns.
func TestRunSamplesImmediatelyOnStart(t *testing.T) {
p := &Poller{diskFreeBytesFn: func(string) (int64, int64, error) { return 1, 1, nil }}
ctx, cancel := context.WithCancel(context.Background())
cancel()
p.Run(ctx, time.Hour)
if !p.Snapshot().Ready {
t.Fatal("Run did not produce a ready snapshot from its initial sample")
}
}

46
internal/procctl/env_sensitive_test.go Normal file
View file

@ -0,0 +1,46 @@
package procctl
import (
"os"
"path/filepath"
"testing"
)
// TestReadEnvGroupsSensitiveFlag guards against sensitiveKeyRe false-positive
// matches on the word "SECRET" in a name that means Telegram's secret-chat
// feature, not a credential (e.g. TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD),
// while still flagging real credential-shaped names as sensitive.
func TestReadEnvGroupsSensitiveFlag(t *testing.T) {
root := t.TempDir()
tmpl := "## Storage & Media -- test group\n" +
"# desc\n" +
"TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD=true\n" +
"# desc\n" +
"TELESRV_ADMIN_PASSWORD=\n" +
"# desc\n" +
"TELESRV_BOT_API_KEY=\n"
if err := os.WriteFile(filepath.Join(root, ".env.example"), []byte(tmpl), 0o644); err != nil {
t.Fatalf("write .env.example: %v", err)
}
groups, err := NewManager(root).ReadEnvGroups()
if err != nil {
t.Fatalf("ReadEnvGroups: %v", err)
}
got := map[string]bool{}
for _, g := range groups {
for _, f := range g.Fields {
got[f.Key] = f.Sensitive
}
}
if got["TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD"] {
t.Errorf("TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD marked sensitive, want not (it's a boolean toggle, not a credential)")
}
if !got["TELESRV_ADMIN_PASSWORD"] {
t.Errorf("TELESRV_ADMIN_PASSWORD not marked sensitive, want sensitive")
}
if !got["TELESRV_BOT_API_KEY"] {
t.Errorf("TELESRV_BOT_API_KEY not marked sensitive, want sensitive")
}
}

7
internal/procctl/procctl.go
View file

@ -490,6 +490,11 @@ var (
activeFieldRe = regexp.MustCompile(`^(TELESRV_[A-Z0-9_]+)=(.*)$`)
commentedFieldRe = regexp.MustCompile(`^#\s*(TELESRV_[A-Z0-9_]+)=(.*)$`)
sensitiveKeyRe = regexp.MustCompile(`(PASSWORD|SECRET|_TOKEN|API_KEY)`)
// sensitiveKeyExceptRe excludes names that trip sensitiveKeyRe on the
// word "SECRET" while meaning Telegram's secret-chat feature, not a
// credential (e.g. TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD) --
// there's nothing to mask there, it's a plain boolean toggle.
sensitiveKeyExceptRe = regexp.MustCompile(`SECRET_CHAT`)
groupHeaderRe = regexp.MustCompile(`^##\s*(.+?)\s*--\s*(.+)$`)
sectionBreakRe = regexp.MustCompile(`^#\s*={10,}\s*$`)
)
@ -554,7 +559,7 @@ func (m *Manager) ReadEnvGroups() ([]EnvGroup, error) {
DefaultValue: defaultValue,
Description: description,
EnabledByDefault: enabledByDefault,
Sensitive: sensitiveKeyRe.MatchString(key),
Sensitive: sensitiveKeyRe.MatchString(key) && !sensitiveKeyExceptRe.MatchString(key),
Value: value,
})
}

4
internal/store/postgres/media_hard_retention_integration_test.go
View file

@ -64,7 +64,7 @@ func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
})
cutoff := time.Now().Add(-24 * time.Hour)
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, cutoff, 1000)
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, &cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
}
@ -95,7 +95,7 @@ func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
// Idempotent / self-terminating: a second sweep pass no longer selects
// this document, since it no longer owns any file_blobs row.
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, cutoff, 1000)
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, &cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
}

60
internal/store/postgres/media_refs.go
View file

@ -236,48 +236,90 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
return blobs, nil
}
// nullableTimestamptz converts an optional cutoff into the nullable
// pgtype.Timestamptz the hard-retention List*/Count* queries expect: nil
// means "no age filter at all" (sqlc.narg(cutoff) IS NULL in the query),
// matching the manual-purge admin action's "no date = everything" contract.
// The automatic sweep (files.Service.DeleteBlobBytesForMediaOlderThan)
// always passes a non-nil cutoff.
func nullableTimestamptz(cutoff *time.Time) pgtype.Timestamptz {
if cutoff == nil {
return pgtype.Timestamptz{}
}
return pgtype.Timestamptz{Time: *cutoff, Valid: true}
}
// ListDocumentIDsForHardRetentionOlderThan returns document ids in the given
// category older than cutoff (by upload/created_at) that still own at least
// one file_blobs row, oldest first, up to limit -- the "hard" retention
// sweep's candidate list. Unlike ListOrphanedDocumentIDsOlderThan, this
// ignores media_references entirely: a document still referenced by a live
// message is exactly as eligible as an orphaned one.
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error) {
// message is exactly as eligible as an orphaned one. cutoff may be nil,
// meaning no age filter at all (every document in the category is a
// candidate) -- used by the manual purge admin action.
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff *time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
Cutoff: nullableTimestamptz(cutoff),
Category: int16(category),
BatchLimit: int32(limit),
})
}
// CountDocumentsForHardRetention is the exact-count counterpart of
// ListDocumentIDsForHardRetentionOlderThan, for the manual purge admin
// action's dry-run preview.
func (s *MediaStore) CountDocumentsForHardRetention(ctx context.Context, category domain.MediaCategory, cutoff *time.Time) (int, error) {
n, err := s.q.CountDocumentsForHardRetention(ctx, sqlcgen.CountDocumentsForHardRetentionParams{
Cutoff: nullableTimestamptz(cutoff),
Category: int16(category),
})
return int(n), err
}
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
// ListDocumentIDsForHardRetentionOlderThan (excluding a live avatar -- see
// ListAvatarPhotoIDsForHardRetentionOlderThan).
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
// ListAvatarPhotoIDsForHardRetentionOlderThan). cutoff may be nil -- see
// ListDocumentIDsForHardRetentionOlderThan.
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff *time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListPhotoIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
Cutoff: nullableTimestamptz(cutoff),
BatchLimit: int32(limit),
})
}
// CountPhotosForHardRetention is the exact-count counterpart of
// ListPhotoIDsForHardRetentionOlderThan.
func (s *MediaStore) CountPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error) {
n, err := s.q.CountPhotosForHardRetention(ctx, nullableTimestamptz(cutoff))
return int(n), err
}
// ListAvatarPhotoIDsForHardRetentionOlderThan is the "Avatar" category
// counterpart of ListPhotoIDsForHardRetentionOlderThan.
func (s *MediaStore) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
// counterpart of ListPhotoIDsForHardRetentionOlderThan. cutoff may be nil --
// see ListDocumentIDsForHardRetentionOlderThan.
func (s *MediaStore) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff *time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListAvatarPhotoIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
Cutoff: nullableTimestamptz(cutoff),
BatchLimit: int32(limit),
})
}
// CountAvatarPhotosForHardRetention is the exact-count counterpart of
// ListAvatarPhotoIDsForHardRetentionOlderThan.
func (s *MediaStore) CountAvatarPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error) {
n, err := s.q.CountAvatarPhotosForHardRetention(ctx, nullableTimestamptz(cutoff))
return int(n), err
}
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
// (main body + thumbnail variants), returning what was deleted so the caller
// can physically remove each object from its backend once confirming (via

54
internal/store/postgres/queries/media.sql
View file

@ -277,9 +277,14 @@ LIMIT sqlc.arg(batch_limit)::int;
-- rows but deliberately leaves this documents row in place), it naturally
-- drops out of this query on the next pass. category scopes to one
-- documents.category bucket per sweep tick -- see
-- ListOrphanedDocumentIDsOlderThan for the 0/None fallback note.
-- ListOrphanedDocumentIDsOlderThan for the 0/None fallback note. cutoff is
-- nullable: NULL means no age filter at all (every document in the category
-- is a candidate, regardless of created_at) -- used by the manual purge admin
-- action, which unlike the automatic sweep can target "everything" with no
-- age cutoff. The automatic sweep (files.Service.DeleteBlobBytesForMediaOlderThan)
-- always passes a real, non-null cutoff.
SELECT d.id FROM documents d
WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR d.created_at < sqlc.narg(cutoff)::timestamptz)
AND d.category = sqlc.arg(category)::smallint
AND EXISTS (
SELECT 1 FROM file_blobs fb
@ -289,12 +294,26 @@ WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
ORDER BY d.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: CountDocumentsForHardRetention :one
-- Exact-count counterpart of ListDocumentIDsForHardRetentionOlderThan, used
-- by the manual purge admin action's dry-run preview: a LIMIT-capped list
-- length is not good enough there, the operator needs the real total.
SELECT COUNT(*)::int FROM documents d
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR d.created_at < sqlc.narg(cutoff)::timestamptz)
AND d.category = sqlc.arg(category)::smallint
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
);
-- name: ListPhotoIDsForHardRetentionOlderThan :many
-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
-- candidate selection, for photos. Excludes photos currently active as
-- someone's avatar -- see ListAvatarPhotoIDsForHardRetentionOlderThan.
-- candidate selection (including the nullable cutoff), for photos. Excludes
-- photos currently active as someone's avatar -- see
-- ListAvatarPhotoIDsForHardRetentionOlderThan.
SELECT p.id FROM photos p
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR p.created_at < sqlc.narg(cutoff)::timestamptz)
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
@ -304,13 +323,25 @@ WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
ORDER BY p.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: CountPhotosForHardRetention :one
-- Exact-count counterpart of ListPhotoIDsForHardRetentionOlderThan -- see
-- CountDocumentsForHardRetention.
SELECT COUNT(*)::int FROM photos p
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR p.created_at < sqlc.narg(cutoff)::timestamptz)
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
);
-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many
-- Same as ListPhotoIDsForHardRetentionOlderThan but only photos currently
-- active as someone's avatar (profile_photos.active) -- lets the Avatar
-- category carry its own retention age, independent of ordinary shared-media
-- photos.
SELECT p.id FROM photos p
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR p.created_at < sqlc.narg(cutoff)::timestamptz)
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
@ -320,6 +351,17 @@ WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
ORDER BY p.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: CountAvatarPhotosForHardRetention :one
-- Exact-count counterpart of ListAvatarPhotoIDsForHardRetentionOlderThan.
SELECT COUNT(*)::int FROM photos p
WHERE (sqlc.narg(cutoff)::timestamptz IS NULL OR p.created_at < sqlc.narg(cutoff)::timestamptz)
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
);
-- name: ListOldestDocumentsForEviction :many
-- Active eviction (TELESRV_STORAGE_EVICTION_ENABLE): candidates are every
-- document that still owns at least one file_blobs row, oldest-uploaded

121
internal/store/postgres/retention_v2_integration_test.go
View file

@ -279,7 +279,7 @@ func TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory(t *testing.T) {
}
cutoff := time.Now().Add(-24 * time.Hour)
videoIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryVideo, cutoff, 1000)
videoIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryVideo, &cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Video): %v", err)
}
@ -287,7 +287,7 @@ func TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory(t *testing.T) {
t.Fatalf("video-category candidates = %v, want to include %d and exclude %d", videoIDs, videoID, musicID)
}
musicIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryMusic, cutoff, 1000)
musicIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryMusic, &cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Music): %v", err)
}
@ -296,6 +296,123 @@ func TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory(t *testing.T) {
}
}
// TestManualPurgeNilCutoffIgnoresAge guards the manual purge admin action's
// query-layer contract: a nil cutoff on the hard-retention List/Count
// queries must mean "no age filter at all" -- a document created just now is
// still a candidate -- while a real, non-nil cutoff still filters normally.
// This is the query-layer half of files.Service.ManualPurge/CountManualPurgeCandidates's
// "no date = everything matching the categories, regardless of age" contract
// (the automatic sweep, by contrast, never passes a nil cutoff).
func TestManualPurgeNilCutoffIgnoresAge(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
media := NewMediaStore(pool)
freshID := time.Now().UnixNano()
locationKey := "doc:" + strconv.FormatInt(freshID, 10)
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = $1", freshID)
_, _ = pool.Exec(context.Background(), "DELETE FROM file_blobs WHERE location_key = $1", locationKey)
})
if err := media.PutDocument(ctx, domain.Document{ID: freshID, MimeType: "application/octet-stream", Size: 128}); err != nil {
t.Fatalf("PutDocument: %v", err)
}
// Deliberately NOT backdated -- created_at stays "just now", unlike every
// other hard-retention test in this file, since the whole point here is
// that a nil cutoff must not exempt a fresh document.
if _, err := pool.Exec(ctx, "UPDATE documents SET category = $2 WHERE id = $1", freshID, int16(domain.MediaCategoryFile)); err != nil {
t.Fatalf("categorize document: %v", err)
}
blob := postgresTestBlob(locationKey, "manual-purge-nil-cutoff", 128, "application/octet-stream")
if err := media.PutFileBlob(ctx, blob); err != nil {
t.Fatalf("PutFileBlob: %v", err)
}
// nil cutoff: the fresh document IS a candidate.
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryFile, nil, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(nil cutoff): %v", err)
}
if !containsInt64(ids, freshID) {
t.Fatalf("nil-cutoff candidates = %v, want to include fresh document %d", ids, freshID)
}
count, err := media.CountDocumentsForHardRetention(ctx, domain.MediaCategoryFile, nil)
if err != nil {
t.Fatalf("CountDocumentsForHardRetention(nil cutoff): %v", err)
}
if count < 1 {
t.Fatalf("nil-cutoff count = %d, want >= 1 (must include fresh document)", count)
}
// A real cutoff in the past still excludes the fresh document.
pastCutoff := time.Now().Add(-24 * time.Hour)
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryFile, &pastCutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(past cutoff): %v", err)
}
if containsInt64(ids, freshID) {
t.Fatalf("past-cutoff candidates = %v, want to exclude fresh document %d", ids, freshID)
}
count, err = media.CountDocumentsForHardRetention(ctx, domain.MediaCategoryFile, &pastCutoff)
if err != nil {
t.Fatalf("CountDocumentsForHardRetention(past cutoff): %v", err)
}
// Some other, unrelated old File-category document could in principle
// exist in this shared test database, so this only asserts the fresh
// document itself dropped out, via the id-level checks above; the count
// assertion below just guards against the query going the wrong
// direction entirely (e.g. cutoff being ignored and still counting
// everything).
allCount, err := media.CountDocumentsForHardRetention(ctx, domain.MediaCategoryFile, nil)
if err != nil {
t.Fatalf("CountDocumentsForHardRetention(nil, recheck): %v", err)
}
if count > allCount {
t.Fatalf("past-cutoff count %d > nil-cutoff count %d, cutoff filter is backwards", count, allCount)
}
}
// TestManualPurgePhotoAndAvatarNilCutoffIgnoresAge is the photo/avatar
// counterpart of TestManualPurgeNilCutoffIgnoresAge.
func TestManualPurgePhotoAndAvatarNilCutoffIgnoresAge(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
media := NewMediaStore(pool)
photoID := time.Now().UnixNano()
locationKey := "photo:" + strconv.FormatInt(photoID, 10)
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), "DELETE FROM photos WHERE id = $1", photoID)
_, _ = pool.Exec(context.Background(), "DELETE FROM file_blobs WHERE location_key = $1", locationKey)
})
if err := media.PutPhoto(ctx, domain.Photo{ID: photoID, AccessHash: 1}); err != nil {
t.Fatalf("PutPhoto: %v", err)
}
blob := postgresTestBlob(locationKey, "manual-purge-photo-nil-cutoff", 64, "image/jpeg")
if err := media.PutFileBlob(ctx, blob); err != nil {
t.Fatalf("PutFileBlob: %v", err)
}
ids, err := media.ListPhotoIDsForHardRetentionOlderThan(ctx, nil, 1000)
if err != nil {
t.Fatalf("ListPhotoIDsForHardRetentionOlderThan(nil cutoff): %v", err)
}
if !containsInt64(ids, photoID) {
t.Fatalf("nil-cutoff photo candidates = %v, want to include fresh photo %d", ids, photoID)
}
pastCutoff := time.Now().Add(-24 * time.Hour)
ids, err = media.ListPhotoIDsForHardRetentionOlderThan(ctx, &pastCutoff, 1000)
if err != nil {
t.Fatalf("ListPhotoIDsForHardRetentionOlderThan(past cutoff): %v", err)
}
if containsInt64(ids, photoID) {
t.Fatalf("past-cutoff photo candidates = %v, want to exclude fresh photo %d", ids, photoID)
}
}
// TestEvictionListsOldestAcrossDocumentsAndPhotos guards Part 3's eviction
// query layer: ListOldestMediaForEviction must return candidates from both
// tables (interleaving by created_at is done by the caller in Go, see

83
internal/store/postgres/sqlcgen/media.sql.go
View file

@ -79,6 +79,51 @@ func (q *Queries) CountAvailableReactions(ctx context.Context) (int32, error) {
return total, err
}
const countAvatarPhotosForHardRetention = `-- name: CountAvatarPhotosForHardRetention :one
SELECT COUNT(*)::int FROM photos p
WHERE ($1::timestamptz IS NULL OR p.created_at < $1::timestamptz)
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
`
// Exact-count counterpart of ListAvatarPhotoIDsForHardRetentionOlderThan.
func (q *Queries) CountAvatarPhotosForHardRetention(ctx context.Context, cutoff pgtype.Timestamptz) (int32, error) {
row := q.db.QueryRow(ctx, countAvatarPhotosForHardRetention, cutoff)
var column_1 int32
err := row.Scan(&column_1)
return column_1, err
}
const countDocumentsForHardRetention = `-- name: CountDocumentsForHardRetention :one
SELECT COUNT(*)::int FROM documents d
WHERE ($1::timestamptz IS NULL OR d.created_at < $1::timestamptz)
AND d.category = $2::smallint
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
)
`
type CountDocumentsForHardRetentionParams struct {
Cutoff pgtype.Timestamptz
Category int16
}
// Exact-count counterpart of ListDocumentIDsForHardRetentionOlderThan, used
// by the manual purge admin action's dry-run preview: a LIMIT-capped list
// length is not good enough there, the operator needs the real total.
func (q *Queries) CountDocumentsForHardRetention(ctx context.Context, arg CountDocumentsForHardRetentionParams) (int32, error) {
row := q.db.QueryRow(ctx, countDocumentsForHardRetention, arg.Cutoff, arg.Category)
var column_1 int32
err := row.Scan(&column_1)
return column_1, err
}
const countFileBlobRefs = `-- name: CountFileBlobRefs :one
SELECT COUNT(*)::int FROM file_blobs WHERE backend = $1::text AND object_key = $2::text
`
@ -95,6 +140,26 @@ func (q *Queries) CountFileBlobRefs(ctx context.Context, arg CountFileBlobRefsPa
return column_1, err
}
const countPhotosForHardRetention = `-- name: CountPhotosForHardRetention :one
SELECT COUNT(*)::int FROM photos p
WHERE ($1::timestamptz IS NULL OR p.created_at < $1::timestamptz)
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
`
// Exact-count counterpart of ListPhotoIDsForHardRetentionOlderThan -- see
// CountDocumentsForHardRetention.
func (q *Queries) CountPhotosForHardRetention(ctx context.Context, cutoff pgtype.Timestamptz) (int32, error) {
row := q.db.QueryRow(ctx, countPhotosForHardRetention, cutoff)
var column_1 int32
err := row.Scan(&column_1)
return column_1, err
}
const countProfilePhotos = `-- name: CountProfilePhotos :one
SELECT count(*)::int AS total
FROM profile_photos
@ -866,7 +931,7 @@ func (q *Queries) ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, arg L
const listAvatarPhotoIDsForHardRetentionOlderThan = `-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many
SELECT p.id FROM photos p
WHERE p.created_at < $1::timestamptz
WHERE ($1::timestamptz IS NULL OR p.created_at < $1::timestamptz)
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
@ -908,7 +973,7 @@ func (q *Queries) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Contex
const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
SELECT d.id FROM documents d
WHERE d.created_at < $1::timestamptz
WHERE ($1::timestamptz IS NULL OR d.created_at < $1::timestamptz)
AND d.category = $2::smallint
AND EXISTS (
SELECT 1 FROM file_blobs fb
@ -933,7 +998,12 @@ type ListDocumentIDsForHardRetentionOlderThanParams struct {
// rows but deliberately leaves this documents row in place), it naturally
// drops out of this query on the next pass. category scopes to one
// documents.category bucket per sweep tick -- see
// ListOrphanedDocumentIDsOlderThan for the 0/None fallback note.
// ListOrphanedDocumentIDsOlderThan for the 0/None fallback note. cutoff is
// nullable: NULL means no age filter at all (every document in the category
// is a candidate, regardless of created_at) -- used by the manual purge admin
// action, which unlike the automatic sweep can target "everything" with no
// age cutoff. The automatic sweep (files.Service.DeleteBlobBytesForMediaOlderThan)
// always passes a real, non-null cutoff.
func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.Category, arg.BatchLimit)
if err != nil {
@ -1207,7 +1277,7 @@ func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrp
const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
SELECT p.id FROM photos p
WHERE p.created_at < $1::timestamptz
WHERE ($1::timestamptz IS NULL OR p.created_at < $1::timestamptz)
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
@ -1224,8 +1294,9 @@ type ListPhotoIDsForHardRetentionOlderThanParams struct {
}
// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
// candidate selection, for photos. Excludes photos currently active as
// someone's avatar -- see ListAvatarPhotoIDsForHardRetentionOlderThan.
// candidate selection (including the nullable cutoff), for photos. Excludes
// photos currently active as someone's avatar -- see
// ListAvatarPhotoIDsForHardRetentionOlderThan.
func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {