s3 support

This commit is contained in:
onysd 2026-08-04 23:09:11 +03:00
parent fa5cfaf14d
commit 03f10b66ee
53 changed files with 2796 additions and 102 deletions

View file

@ -239,6 +239,33 @@ TELESRV_BOT_VERIFICATION_ENABLED=true
# weights and recompute timing live in the Advanced section below.
TELESRV_RATING_ENABLED=true
## Storage & Media -- Where uploaded media is stored, and how the server reacts to running low on space.
# Where uploaded media (photos, documents, stickers) is physically stored.
# "localfs" writes to TELESRV_BLOB_DIR on this machine's disk. "s3" writes to
# an S3-compatible object store -- self-hosted MinIO or AWS S3 -- configured
# below. Switching only affects new uploads; existing files stay wherever
# they were written and remain readable.
TELESRV_BLOB_BACKEND=localfs
# Only used when TELESRV_BLOB_BACKEND=s3.
TELESRV_S3_ENDPOINT=
TELESRV_S3_REGION=us-east-1
TELESRV_S3_BUCKET=
TELESRV_S3_ACCESS_KEY_ID=
TELESRV_S3_SECRET_ACCESS_KEY=
TELESRV_S3_USE_SSL=true
# MinIO typically needs this on (bucket in the URL path); AWS S3 does not.
TELESRV_S3_PATH_STYLE=false
# Reject new uploads once storage is nearly full, instead of letting the disk
# fill up. Thresholds live in the Advanced section below.
TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE=true
# Automatically delete old media once it's no longer referenced by any
# message, profile photo, or sticker set (never deletes media still visible
# in a conversation). Off by default -- storage usage is tracked and shown
# in the admin panel either way; enable this once you're comfortable with
# what it will reclaim. Retention age lives in the Advanced section below.
TELESRV_STORAGE_RETENTION_ENABLE=false
# ==============================================================================
# Advanced / internal tuning
@ -367,6 +394,23 @@ TELESRV_STICKER_SEED_MAX_SETS=300
# Sticker set auto-installed for every newly registered account; <=0 disables this.
TELESRV_DEFAULT_STICKER_SET_ID=0
# Storage low-space guard thresholds (master toggle is TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE above).
# localfs: reject new uploads once real free disk bytes fall below this; <=0 disables.
TELESRV_STORAGE_MIN_FREE_BYTES=1073741824
# Reject new uploads once total tracked blob bytes would exceed this. The only
# meaningful "low space" signal on the s3 backend (no OS free-space concept);
# optional soft cap on localfs too. <=0 disables.
TELESRV_STORAGE_MAX_TOTAL_BYTES=0
# How often the cached free-space/usage gauge behind the guard above refreshes.
TELESRV_STORAGE_USAGE_REFRESH_INTERVAL=1m
# Storage retention sweep tuning (master toggle is TELESRV_STORAGE_RETENTION_ENABLE above).
# How long a document/photo must have had zero references before the sweep
# deletes it -- not how old the media itself is, and it never touches media
# still referenced by a live message/profile-photo/sticker-set. The sweep
# itself runs alongside every other retention check on the shared
# TELESRV_RETENTION_INTERVAL/TELESRV_RETENTION_BATCH cadence above.
TELESRV_STORAGE_RETENTION_MAX_AGE=720h
# New-account perks: free Telegram Premium months and starting Stars balance.
TELESRV_PREMIUM_GRANT_MONTHS=3
TELESRV_STARS_STARTING_GRANT=1000

View file

@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"os"
"strconv"
"strings"
"time"
@ -2541,3 +2542,133 @@ func clampBotVerificationLimit(limit int) int {
}
return limit
}
// storageUsageListDefaultLimit/storageUsageListMaxLimit bound the per-account
// storage breakdown page, same convention as the account rating list above.
const (
storageUsageListDefaultLimit = 50
storageUsageListMaxLimit = 200
)
// perOwnerMediaSizeSQL is shared between StorageStats and
// ListAccountStorageUsage: each document row's size is its stored column;
// each photo row has no single size column (JSONB sizes holds one entry per
// rendition), so its "attributed size" is the largest rendition -- the
// dominant cost, thumbnails are comparatively tiny. This is an
// approximation (not the exact sum of every rendition's blob bytes, which
// would require joining file_blobs by location_key prefix) chosen for admin
// visibility, not billing precision.
const perOwnerMediaSizeSQL = `
SELECT owner_user_id, size FROM documents
UNION ALL
SELECT p.owner_user_id, COALESCE((
SELECT MAX((elem->>'size')::bigint) FROM jsonb_array_elements(p.sizes) elem
), 0) AS size
FROM photos p
`
// StorageStatsRow is the admin panel's storage overview: physical bytes
// (from file_blobs, backend-dedup-aware -- what's actually consuming disk
// or S3) versus logical bytes (sum of the same approximate per-row
// attribution the per-account breakdown uses, which can legitimately be
// higher than physical when identical content is shared by more than one
// document/photo).
type StorageStatsRow struct {
PhysicalBytes int64 `json:"PhysicalBytes,string"`
LogicalBytes int64 `json:"LogicalBytes,string"`
UnattributedBytes int64 `json:"UnattributedBytes,string"`
DocumentCount int64 `json:"DocumentCount,string"`
PhotoCount int64 `json:"PhotoCount,string"`
AccountCount int64 `json:"AccountCount,string"`
BackendKind string
}
// StorageStats returns the admin panel's storage overview.
func (s *readStore) StorageStats(ctx context.Context) (StorageStatsRow, error) {
var stats StorageStatsRow
if err := s.pool.QueryRow(ctx, `SELECT COALESCE(SUM(size), 0)::bigint FROM file_blobs`).Scan(&stats.PhysicalBytes); err != nil {
return StorageStatsRow{}, fmt.Errorf("sum physical blob bytes: %w", err)
}
if err := s.pool.QueryRow(ctx, `
SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x`).Scan(&stats.LogicalBytes); err != nil {
return StorageStatsRow{}, fmt.Errorf("sum logical media bytes: %w", err)
}
if err := s.pool.QueryRow(ctx, `
SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id = 0`).Scan(&stats.UnattributedBytes); err != nil {
return StorageStatsRow{}, fmt.Errorf("sum unattributed media bytes: %w", err)
}
if err := s.pool.QueryRow(ctx, `SELECT count(*)::bigint FROM documents`).Scan(&stats.DocumentCount); err != nil {
return StorageStatsRow{}, fmt.Errorf("count documents: %w", err)
}
if err := s.pool.QueryRow(ctx, `SELECT count(*)::bigint FROM photos`).Scan(&stats.PhotoCount); err != nil {
return StorageStatsRow{}, fmt.Errorf("count photos: %w", err)
}
if err := s.pool.QueryRow(ctx, `
SELECT count(DISTINCT owner_user_id)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id <> 0`).Scan(&stats.AccountCount); err != nil {
return StorageStatsRow{}, fmt.Errorf("count storage accounts: %w", err)
}
stats.BackendKind = strings.ToLower(strings.TrimSpace(os.Getenv("TELESRV_BLOB_BACKEND")))
if stats.BackendKind == "" {
stats.BackendKind = "localfs"
}
return stats, nil
}
// AccountStorageRow is one account's row in the per-account storage
// breakdown table.
type AccountStorageRow struct {
UserID int64 `json:"UserID,string"`
Username string
FirstName string
Bytes int64 `json:"Bytes,string"`
FileCount int64 `json:"FileCount,string"`
}
// ListAccountStorageUsage pages the per-account storage breakdown, largest
// user first. Offset-based (not keyset): storage administration on a
// self-hosted deployment doesn't need to support arbitrarily deep pages
// efficiently the way an infinite-scroll feed does.
func (s *readStore) ListAccountStorageUsage(ctx context.Context, offset, limit int) ([]AccountStorageRow, bool, error) {
if limit <= 0 {
limit = storageUsageListDefaultLimit
}
if limit > storageUsageListMaxLimit {
limit = storageUsageListMaxLimit
}
if offset < 0 {
offset = 0
}
rows, err := s.pool.Query(ctx, `
WITH totals AS (
SELECT owner_user_id, SUM(size)::bigint AS bytes, COUNT(*)::bigint AS file_count
FROM (`+perOwnerMediaSizeSQL+`) x
WHERE owner_user_id <> 0
GROUP BY owner_user_id
)
SELECT t.owner_user_id, COALESCE(u.username, ''), COALESCE(u.first_name, ''), t.bytes, t.file_count
FROM totals t
LEFT JOIN users u ON u.id = t.owner_user_id
ORDER BY t.bytes DESC, t.owner_user_id
OFFSET $1
LIMIT $2`, offset, limit+1)
if err != nil {
return nil, false, fmt.Errorf("list account storage usage: %w", err)
}
defer rows.Close()
out := make([]AccountStorageRow, 0, limit+1)
for rows.Next() {
var item AccountStorageRow
if err := rows.Scan(&item.UserID, &item.Username, &item.FirstName, &item.Bytes, &item.FileCount); err != nil {
return nil, false, err
}
out = append(out, item)
}
if err := rows.Err(); err != nil {
return nil, false, err
}
hasMore := len(out) > limit
if hasMore {
out = out[:limit]
}
return out, hasMore, nil
}

View file

@ -78,6 +78,8 @@ func (s *server) routes() http.Handler {
mux.Handle("GET /api/collectible-usernames/{id}", s.requireAuthAPI(http.HandlerFunc(s.handleCollectibleUsernameDetailAPI)))
mux.Handle("GET /api/account-ratings", s.requireAuthAPI(http.HandlerFunc(s.handleAccountRatingsAPI)))
mux.Handle("GET /api/account-ratings/{user_id}", s.requireAuthAPI(http.HandlerFunc(s.handleAccountRatingDetailAPI)))
mux.Handle("GET /api/storage/stats", s.requireAuthAPI(http.HandlerFunc(s.handleStorageStatsAPI)))
mux.Handle("GET /api/storage/accounts", s.requireAuthAPI(http.HandlerFunc(s.handleStorageAccountsAPI)))
mux.Handle("GET /api/moderation/cases", s.requireAuthAPI(http.HandlerFunc(s.handleModerationCasesAPI)))
mux.Handle("GET /api/moderation/cases/{id}", s.requireAuthAPI(http.HandlerFunc(s.handleModerationCaseAPI)))
mux.Handle("GET /api/moderation/reports/{id}", s.requireAuthAPI(http.HandlerFunc(s.handleModerationReportAPI)))
@ -2342,6 +2344,53 @@ func (s *server) handleAccountRatingsAPI(w http.ResponseWriter, r *http.Request)
})
}
func (s *server) handleStorageStatsAPI(w http.ResponseWriter, r *http.Request) {
if s.read == nil {
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")
return
}
stats, err := s.read.StorageStats(r.Context())
if err != nil {
writeAPIError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, stats)
}
// handleStorageAccountsAPI pages the per-account storage breakdown,
// largest first.
func (s *server) handleStorageAccountsAPI(w http.ResponseWriter, r *http.Request) {
if s.read == nil {
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")
return
}
query := r.URL.Query()
offset, err := parseInt(query.Get("offset"))
if err != nil || offset < 0 {
writeAPIError(w, http.StatusBadRequest, "invalid offset")
return
}
limit, err := parseInt(query.Get("limit"))
if err != nil || limit < 0 {
writeAPIError(w, http.StatusBadRequest, "invalid limit")
return
}
rows, hasMore, err := s.read.ListAccountStorageUsage(r.Context(), offset, limit)
if err != nil {
writeAPIError(w, http.StatusInternalServerError, err.Error())
return
}
nextOffset := 0
if hasMore {
nextOffset = offset + len(rows)
}
writeJSON(w, http.StatusOK, map[string]any{
"rows": rows,
"has_more": hasMore,
"next_offset": nextOffset,
})
}
func (s *server) handleAccountRatingDetailAPI(w http.ResponseWriter, r *http.Request) {
if s.read == nil {
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -23,7 +23,7 @@
})();
</script>
<script type="module" crossorigin src="/assets/index-D0qBFBU-.js"></script>
<script type="module" crossorigin src="/assets/index-CmjlUwWc.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-EkAGiEK9.css">
</head>
<body>

View file

@ -4,6 +4,8 @@ import type {
AccountRatingDetail,
AccountRatingListResponse,
AccountStatsResponse,
AccountStorageListResponse,
StorageStatsResponse,
AdminLoginResult,
AdminSession,
BotDetail,
@ -165,6 +167,9 @@ export const api = {
request<AccountRatingListResponse>(`/api/account-ratings?${params.toString()}`),
accountRating: (userID: string) =>
request<AccountRatingDetail>(`/api/account-ratings/${encodeURIComponent(userID)}`),
storageStats: () => request<StorageStatsResponse>("/api/storage/stats"),
storageAccounts: (params: URLSearchParams) =>
request<AccountStorageListResponse>(`/api/storage/accounts?${params.toString()}`),
verificationApplications: (params: URLSearchParams) =>
request<VerificationApplicationListResponse>(`/api/verification/applications?${params.toString()}`),
// The application id is an int64 decimal string end to end, so it is never

View file

@ -99,6 +99,7 @@ export function Shell({
)}
<NavLink icon={<AtSign size={16} />} href="/collectible-usernames" route={route} navigate={navigate}>{"NFT Usernames"}</NavLink>
<NavLink icon={<Trophy size={16} />} href="/account-ratings" route={route} navigate={navigate}>{"Account Rating"}</NavLink>
<NavLink icon={<Database size={16} />} href="/storage" route={route} navigate={navigate}>{"Storage"}</NavLink>
<NavLink icon={<Gift size={16} />} href="/gifts" route={route} navigate={navigate}>{"Star Gifts"}</NavLink>
<NavLink icon={<Send size={16} />} href="/give-gifts" route={route} navigate={navigate}>{"Give Gifts"}</NavLink>
<NavLink icon={<Sticker size={16} />} href="/stickers" route={route} navigate={navigate}>{"Stickers"}</NavLink>

View file

@ -154,6 +154,26 @@ export function toSmallestUnits(value: string, currency: string): string | null
return digits === "" ? "0" : digits;
}
// formatBytes renders a byte count (as the JSON-string int64 the API sends)
// in the largest unit that keeps it readable. Parses the decimal string
// directly rather than through toNumeric first so precision past
// Number.MAX_SAFE_INTEGER isn't silently lost before the division.
export function formatBytes(value: string): string {
const raw = (value ?? "").trim();
if (!raw || !/^\d+$/.test(raw)) return "0 B";
const bytes = Number(raw);
if (!Number.isFinite(bytes)) return `${raw} B`;
const units = ["B", "KB", "MB", "GB", "TB", "PB"];
let size = bytes;
let unit = 0;
while (size >= 1024 && unit < units.length - 1) {
size /= 1024;
unit++;
}
const precision = unit === 0 ? 0 : 1;
return `${size.toFixed(precision)} ${units[unit]}`;
}
export function formatSigned(value: string): string {
const raw = (value ?? "").trim();
if (!raw) return "0";

View file

@ -19,6 +19,7 @@ import { StickerSetsPage } from "./StickerSetsPage";
import { GiveGiftsPage } from "./GiveGiftsPage";
import { ModerationCaseDetailPage } from "./ModerationCaseDetailPage";
import { ModerationCasesPage } from "./ModerationCasesPage";
import { StoragePage } from "./StoragePage";
import { BotVerificationPage } from "./BotVerificationPage";
import { BotVerificationRequestPage } from "./BotVerificationRequestPage";
import { VerificationDetailPage } from "./VerificationDetailPage";
@ -84,6 +85,9 @@ export function Routes({ route, navigate }: { route: RouteState; navigate: Navig
if (route.path === "/account-ratings") {
return <AccountRatingsPage navigate={navigate} />;
}
if (route.path === "/storage") {
return <StoragePage navigate={navigate} />;
}
if (accountID) {
return <AccountDetailPage id={Number(accountID)} navigate={navigate} />;
}

View file

@ -0,0 +1,118 @@
import { ChevronDown, Loader2, RefreshCw } from "lucide-react";
import { useEffect, useState } from "react";
import { api, errorMessage } from "../api";
import { Alert, EmptyRow, Metric, PageFrame } from "../components/ui";
import { displayUsername, formatBytes, formatQuantity } from "../lib/format";
import type { Navigate } from "../routing";
import type { AccountStorageRow, StorageStatsResponse } from "../types";
export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
const [stats, setStats] = useState<StorageStatsResponse | null>(null);
const [rows, setRows] = useState<AccountStorageRow[]>([]);
const [hasMore, setHasMore] = useState(false);
const [offset, setOffset] = useState(0);
const [busy, setBusy] = useState(false);
const [error, setError] = useState("");
async function loadStats() {
try {
setStats(await api.storageStats());
} catch {
// Stats are a header nicety; a failure here shouldn't block the list.
}
}
async function loadAccounts(next = false) {
setBusy(true);
setError("");
const at = next ? offset : 0;
const params = new URLSearchParams({ limit: "50", offset: String(at) });
try {
const result = await api.storageAccounts(params);
const page = result.rows ?? [];
setRows((current) => (next ? [...current, ...page] : page));
setOffset(result.next_offset);
setHasMore(Boolean(result.has_more));
} catch (err) {
setError(errorMessage(err));
} finally {
setBusy(false);
}
}
function refresh() {
void loadStats();
void loadAccounts(false);
}
useEffect(() => {
refresh();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
// Physical is what actually consumes disk/S3 (deduplicated); logical is the
// sum of what the per-account table below adds up to. They legitimately
// differ when the same content is shared by more than one document/photo.
const dedupBytes = stats ? Math.max(0, Number(stats.LogicalBytes) - Number(stats.PhysicalBytes)) : 0;
return (
<PageFrame
title={"Storage"}
eyebrow={"Media / Storage usage"}
actions={
<button className="btn icon-text" type="button" onClick={refresh} disabled={busy}>
<RefreshCw size={15} className={busy ? "spin" : ""} /> {"Refresh"}
</button>
}
>
{error && <Alert>{error}</Alert>}
<div className="metric-row">
<Metric label={"Physical usage (on disk / S3)"} value={stats ? formatBytes(stats.PhysicalBytes) : "-"} />
<Metric label={"Logical usage (sum per account)"} value={stats ? formatBytes(stats.LogicalBytes) : "-"} />
<Metric label={"Saved by dedup"} value={formatBytes(String(dedupBytes))} tone={dedupBytes > 0 ? "good" : "neutral"} />
<Metric label={"Backend"} value={stats?.BackendKind ?? "-"} />
</div>
<div className="metric-row">
<Metric label={"Documents"} value={stats ? formatQuantity(stats.DocumentCount) : "-"} />
<Metric label={"Photos"} value={stats ? formatQuantity(stats.PhotoCount) : "-"} />
<Metric label={"Accounts with media"} value={stats ? formatQuantity(stats.AccountCount) : "-"} />
<Metric
label={"Unattributed"}
value={stats ? formatBytes(stats.UnattributedBytes) : "-"}
tone={stats && Number(stats.UnattributedBytes) > 0 ? "warn" : "neutral"}
/>
</div>
<div className="table-wrap">
<table className="data-table">
<thead>
<tr>
<th>{"User ID"}</th>
<th>{"Account"}</th>
<th>{"Storage used"}</th>
<th>{"Files"}</th>
</tr>
</thead>
<tbody>
{rows.map((row) => (
<tr key={row.UserID}>
<td className="mono">{row.UserID}</td>
<td>{displayUsername(row.Username) || row.FirstName || "-"}</td>
<td className="mono">{formatBytes(row.Bytes)}</td>
<td className="mono">{formatQuantity(row.FileCount)}</td>
</tr>
))}
{rows.length === 0 && <EmptyRow colSpan={4} />}
</tbody>
</table>
</div>
{hasMore && (
<div className="toolbar">
<button className="btn icon-text" type="button" onClick={() => loadAccounts(true)} disabled={busy}>
{busy ? <Loader2 size={15} className="spin" /> : <ChevronDown size={15} />} {"Load more"}
</button>
</div>
)}
</PageFrame>
);
}

View file

@ -21,6 +21,7 @@ export function routeTitle(pathname: string): string {
if (pathname.startsWith("/verification")) return "Official Verification";
if (pathname.startsWith("/collectible-usernames")) return "Collectible Usernames";
if (pathname.startsWith("/account-ratings")) return "Account Rating";
if (pathname.startsWith("/storage")) return "Storage";
if (pathname.startsWith("/accounts")) return "Accounts";
if (pathname.startsWith("/channels")) return "Supergroups and Channels";
if (pathname.startsWith("/bots")) return "Bots";
@ -39,6 +40,7 @@ export function routeSubtitle(pathname: string): string {
if (pathname.startsWith("/verification")) return "Console / Verification";
if (pathname.startsWith("/collectible-usernames")) return "Console / Collectible usernames";
if (pathname.startsWith("/account-ratings")) return "Console / Account rating";
if (pathname.startsWith("/storage")) return "Console / Storage";
if (pathname.startsWith("/accounts")) return "Console / Accounts";
if (pathname.startsWith("/channels")) return "Console / Channels";
if (pathname.startsWith("/bots")) return "Console / Bots";

View file

@ -789,6 +789,30 @@ export type AccountStatsResponse = {
online: number;
};
export type StorageStatsResponse = {
PhysicalBytes: string;
LogicalBytes: string;
UnattributedBytes: string;
DocumentCount: string;
PhotoCount: string;
AccountCount: string;
BackendKind: string;
};
export type AccountStorageRow = {
UserID: string;
Username: string;
FirstName: string;
Bytes: string;
FileCount: string;
};
export type AccountStorageListResponse = {
rows: AccountStorageRow[] | null;
has_more: boolean;
next_offset: number;
};
export type ChannelListResponse = {
query: string;
limit: number;

View file

@ -710,14 +710,36 @@ func run(logger *zap.Logger) error {
cachedPhotos := userprojection.NewCachedPhotoProvider(mediaStore, userprojection.DefaultPhotoCacheTTL)
privacyStore := privacyapp.NewCachedPrivacyStore(postgres.NewPrivacyStore(pool), 0)
storyStore := postgres.NewStoryStore(pool)
blobBackend, err := filesapp.NewLocalFS(cfg.BlobDir)
// Transient upload-part scratch storage always stays on local disk
// (TELESRV_BLOB_DIR) regardless of the permanent blob backend below --
// one S3 round trip per ~512KB chunk isn't worth it for data deleted
// within minutes of assembly.
localBlobFS, err := filesapp.NewLocalFS(cfg.BlobDir)
if err != nil {
return fmt.Errorf("init blob backend: %w", err)
return fmt.Errorf("init local blob dir: %w", err)
}
var blobBackend filesapp.BlobBackend = localBlobFS
var spaceGuard filesapp.SpaceGuard = filesapp.NoopSpaceGuard{}
if cfg.BlobBackendKind == "s3" {
s3Backend, err := filesapp.NewS3FS(ctx, cfg.S3Endpoint, cfg.S3AccessKeyID, cfg.S3SecretAccessKey, cfg.S3Bucket, cfg.S3Region, cfg.S3UseSSL, cfg.S3PathStyle)
if err != nil {
return fmt.Errorf("init s3 blob backend: %w", err)
}
blobBackend = s3Backend
logger.Info("blob backend ready", zap.String("backend", "s3"), zap.String("bucket", cfg.S3Bucket), zap.String("endpoint", cfg.S3Endpoint))
if cfg.StorageLowSpaceGuardEnable && cfg.StorageMaxTotalBytes > 0 {
s3Guard := filesapp.NewS3BudgetSpaceGuard(cfg.StorageMaxTotalBytes)
spaceGuard = s3Guard
go filesapp.NewS3DiskUsageWorker(s3Guard, mediaStore, cfg.StorageUsageRefreshInterval, logger.Named("files").Named("diskusage")).Run(ctx)
}
} else {
logger.Info("blob backend ready", zap.String("backend", "localfs"), zap.String("dir", cfg.BlobDir))
if cfg.StorageLowSpaceGuardEnable && cfg.StorageMinFreeBytes > 0 {
localGuard := filesapp.NewLocalDiskSpaceGuard(cfg.StorageMinFreeBytes)
spaceGuard = localGuard
go filesapp.NewLocalDiskUsageWorker(localGuard, cfg.BlobDir, cfg.StorageUsageRefreshInterval, logger.Named("files").Named("diskusage")).Run(ctx)
}
}
logger.Info("blob backend ready",
zap.String("backend", "localfs"),
zap.String("dir", cfg.BlobDir),
)
filesService := filesapp.NewService(mediaStore, blobBackend, cfg.DC,
filesapp.WithLogger(logger),
filesapp.WithUploadPartQuota(domain.UploadPartQuota{
@ -725,6 +747,8 @@ func run(logger *zap.Logger) error {
MaxParts: cfg.UploadInFlightMaxParts,
MaxFiles: cfg.UploadInFlightMaxFiles,
}),
filesapp.WithUploadPartBackend(localBlobFS),
filesapp.WithSpaceGuard(spaceGuard),
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
externalMediaOption(cfg),
webPagePreviewOption(cfg),
@ -822,6 +846,10 @@ func run(logger *zap.Logger) error {
Restrictions: adminStore,
OfficialGifts: officialgifts.New(cfg.OfficialGiftsDir),
})
storageRetentionMaxAge := cfg.StorageRetentionMaxAge
if !cfg.StorageRetentionEnable {
storageRetentionMaxAge = 0
}
go maintenance.NewRetentionWorker(dispatchOutboxStore, tempAuthKeyStore, logger.Named("maintenance").Named("retention"),
cfg.UpdateEventRetention,
cfg.RetentionInterval,
@ -836,6 +864,7 @@ func run(logger *zap.Logger) error {
WithUserUpdateRetention(updateEventStore).
WithChannelUpdateRetention(channelStore).
WithOrphanAuthKeyRetention(authKeyStore, activeSessions, cfg.OrphanAuthKeyRetention).
WithOrphanedMediaRetention(filesService, storageRetentionMaxAge).
Run(ctx)
go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"),
cfg.UploadPartTTL,

View file

@ -0,0 +1,11 @@
DROP INDEX IF EXISTS idx_photos_orphaned_at;
DROP INDEX IF EXISTS idx_documents_orphaned_at;
ALTER TABLE public.photos DROP COLUMN IF EXISTS orphaned_at;
ALTER TABLE public.documents DROP COLUMN IF EXISTS orphaned_at;
DROP TABLE IF EXISTS public.media_references;
DROP INDEX IF EXISTS idx_photos_owner_user_id;
DROP INDEX IF EXISTS idx_documents_owner_user_id;
ALTER TABLE public.photos DROP COLUMN IF EXISTS owner_user_id;
ALTER TABLE public.documents DROP COLUMN IF EXISTS owner_user_id;

View file

@ -0,0 +1,32 @@
-- Storage management: track who uploaded each document/photo, and track
-- every live reference to it (message boxes, channel messages, profile
-- photos, sticker material, gifts) so orphaned media (nothing left pointing
-- at it) can be identified safely for later cleanup. A media row's
-- orphaned_at is set the instant its last reference is removed, and cleared
-- if a new reference appears -- retention sweeps only ever act on rows where
-- orphaned_at is set, so media still visible in a live conversation is never
-- touched regardless of age.
ALTER TABLE public.documents ADD COLUMN owner_user_id bigint DEFAULT 0 NOT NULL;
ALTER TABLE public.photos ADD COLUMN owner_user_id bigint DEFAULT 0 NOT NULL;
-- Pre-existing rows predate ownership tracking and have no recorded
-- uploader; they stay at 0 ("unattributed") rather than being backfilled
-- with a guess, and are surfaced as their own bucket in the admin UI.
CREATE INDEX idx_documents_owner_user_id ON public.documents (owner_user_id) WHERE owner_user_id <> 0;
CREATE INDEX idx_photos_owner_user_id ON public.photos (owner_user_id) WHERE owner_user_id <> 0;
CREATE TABLE public.media_references (
media_kind text NOT NULL,
media_id bigint NOT NULL,
ref_kind text NOT NULL,
ref_key text NOT NULL,
created_at timestamp with time zone DEFAULT now() NOT NULL,
PRIMARY KEY (media_kind, media_id, ref_kind, ref_key)
);
CREATE INDEX idx_media_references_media ON public.media_references (media_kind, media_id);
ALTER TABLE public.documents ADD COLUMN orphaned_at timestamp with time zone;
ALTER TABLE public.photos ADD COLUMN orphaned_at timestamp with time zone;
CREATE INDEX idx_documents_orphaned_at ON public.documents (orphaned_at) WHERE orphaned_at IS NOT NULL;
CREATE INDEX idx_photos_orphaned_at ON public.photos (orphaned_at) WHERE orphaned_at IS NOT NULL;

View file

@ -0,0 +1 @@
DROP INDEX IF EXISTS idx_file_blobs_object_key_backend;

View file

@ -0,0 +1,5 @@
-- Supports the reference-count check ("is any other row still using this
-- object_key on this backend?") that runs before a blob is physically
-- deleted from its backend during storage retention sweeps.
CREATE INDEX idx_file_blobs_object_key_backend ON public.file_blobs (backend, object_key);

18
go.mod
View file

@ -13,6 +13,7 @@ require (
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa
github.com/jackc/pgx/v5 v5.9.2
github.com/lestrrat-go/jwx/v3 v3.1.1
github.com/minio/minio-go/v7 v7.2.1
github.com/pion/datachannel v1.6.2
github.com/pion/dtls/v3 v3.1.5
github.com/pion/ice/v4 v4.3.0
@ -31,6 +32,7 @@ require (
golang.org/x/image v0.31.0
golang.org/x/net v0.57.0
golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0
golang.org/x/text v0.40.0
)
@ -40,6 +42,7 @@ require (
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/coder/websocket v1.8.15 // indirect
github.com/dlclark/regexp2 v1.12.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.19.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-faster/jx v1.2.0 // indirect
@ -55,6 +58,8 @@ require (
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.19.1 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/lestrrat-go/blackmagic v1.0.4 // indirect
github.com/lestrrat-go/dsig v1.2.1 // indirect
github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect
@ -63,30 +68,37 @@ require (
github.com/lestrrat-go/option/v2 v2.0.0 // indirect
github.com/mattn/go-colorable v0.1.15 // indirect
github.com/mattn/go-isatty v0.0.22 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/ogen-go/ogen v1.23.0 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pion/mdns/v2 v2.1.0 // indirect
github.com/pion/randutil v0.1.0 // indirect
github.com/pion/stun/v3 v3.1.6 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/refraction-networking/utls v1.8.2 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/segmentio/asm v1.2.1 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/tinylib/msgp v1.6.1 // indirect
github.com/valyala/fastjson v1.6.10 // indirect
github.com/wlynxg/anet v0.0.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/yuin/goldmark v1.8.4 // indirect
github.com/yutopp/go-amf0 v0.1.0 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
golang.org/x/mod v0.38.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/time v0.14.0 // indirect
golang.org/x/tools v0.48.0 // indirect
gopkg.in/ini.v1 v1.67.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
rsc.io/qr v0.2.0 // indirect
)

35
go.sum
View file

@ -36,6 +36,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
@ -95,8 +97,11 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@ -121,8 +126,15 @@ github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.2.1 h1:PfBfwvKB/MmqyN8Vb1G9voWisaM9OrLv+WwOvMwS9Dw=
github.com/minio/minio-go/v7 v7.2.1/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
@ -135,6 +147,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pion/datachannel v1.6.2 h1:7EXQ8TH3vTouBUdRWYbcX2edSx9Yj6k5zl5P+qyxEPc=
github.com/pion/datachannel v1.6.2/go.mod h1:pzbdAZvyGtXbcHM1hBbsFaOTf40lZizU/dNlvVOak6E=
github.com/pion/dtls/v3 v3.1.5 h1:9xJtVsHwMYeSjPp5Hh1FTis4DchnQWtnOa5o+6ygqfc=
@ -174,24 +188,30 @@ github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEv
github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM=
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0=
github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADTh4=
github.com/valyala/fastjson v1.6.10/go.mod h1:e6FubmQouUNP73jtMLmcbxS6ydWIpOfhz34TSfO3JaE=
github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=
@ -207,6 +227,8 @@ github.com/yutopp/go-amf0 v0.1.0/go.mod h1:QzDOBr9RV6sQh6E5GFEJROZbU0iQKijORBmpr
github.com/yutopp/go-flv v0.3.1/go.mod h1:pAlHPSVRMv5aCUKmGOS/dZn/ooTgnc09qOPmiUNMubs=
github.com/yutopp/go-rtmp v0.0.7 h1:sKKm1MVV3ANbJHZlf3Kq8ecq99y5U7XnDUDxSjuK7KU=
github.com/yutopp/go-rtmp v0.0.7/go.mod h1:KSwrC9Xj5Kf18EUlk1g7CScecjXfIqc0J5q+S0u6Irc=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
@ -243,7 +265,6 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
@ -256,6 +277,8 @@ golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss=
gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View file

@ -27,6 +27,12 @@ type BlobBackend interface {
// GetRange 只读 [offset, offset+limit) 段并返回该段字节与文件总大小limit<=0 读到末尾),
// 避免大文件每个 chunk 都整文件读入内存getFile 按 chunk 多次请求 ⇒ 否则 O(N²) 放大)。
GetRange(ctx context.Context, objectKey string, offset, limit int64) (data []byte, total int64, err error)
// Delete removes the object at objectKey. Callers (storage retention GC)
// must first confirm no remaining file_blobs row on this backend still
// references objectKey -- content-addressed storage means the same
// object can be shared by multiple documents/photos. Deleting an
// already-absent object is not an error.
Delete(ctx context.Context, objectKey string) error
}
// UploadPartBackend 保存 upload.saveFilePart/saveBigFilePart 的临时分片字节。
@ -205,6 +211,16 @@ func (l *LocalFS) GetRange(_ context.Context, objectKey string, offset, limit in
return buf[:read], total, nil
}
// Delete removes the on-disk object for objectKey. Missing objects are not
// an error (idempotent, safe to retry). Callers must have already confirmed
// no other file_blobs row on this backend still references objectKey.
func (l *LocalFS) Delete(_ context.Context, objectKey string) error {
if err := os.Remove(l.pathFor(objectKey)); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("delete blob: %w", err)
}
return nil
}
func (l *LocalFS) openBlobFile(objectKey string) (*sharedBlobFile, error) {
l.mu.Lock()
defer l.mu.Unlock()

View file

@ -44,6 +44,32 @@ func TestLocalFSPutGetRoundTrip(t *testing.T) {
}
}
func TestLocalFSDelete(t *testing.T) {
fs, err := NewLocalFS(t.TempDir())
if err != nil {
t.Fatalf("new local fs: %v", err)
}
ctx := context.Background()
key, err := fs.Put(ctx, []byte("delete me"))
if err != nil {
t.Fatalf("put: %v", err)
}
if _, err := fs.Get(ctx, key); err != nil {
t.Fatalf("get before delete: %v", err)
}
if err := fs.Delete(ctx, key); err != nil {
t.Fatalf("delete: %v", err)
}
if _, err := fs.Get(ctx, key); err == nil {
t.Fatal("expected get after delete to fail")
}
// Deleting an already-absent object must be idempotent, not an error --
// the retention sweep can legitimately retry after a partial failure.
if err := fs.Delete(ctx, key); err != nil {
t.Fatalf("delete already-missing object: %v", err)
}
}
func TestLocalFSPutReaderRoundTrip(t *testing.T) {
fs, err := NewLocalFS(t.TempDir())
if err != nil {

View file

@ -0,0 +1,179 @@
package files
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
// S3FS stores blob bytes in an S3-compatible object store (self-hosted MinIO
// or AWS S3). It implements BlobBackend using the same content-addressed
// sha256-hex object key scheme as LocalFS, so file_blobs.object_key stays
// comparable regardless of which backend wrote a given row and a deployment
// can run with rows split across both backends (see TELESRV_BLOB_BACKEND).
//
// UploadPartBackend is intentionally NOT implemented here: transient upload
// parts stay on local disk (see cmd/telesrv/main.go) even when the
// permanent blob backend is s3 -- one S3 round trip per ~512KB chunk isn't
// worth it for scratch data that's deleted within minutes of assembly.
type S3FS struct {
client *minio.Client
bucket string
}
// NewS3FS creates an S3-compatible blob backend. endpoint is host[:port]
// without a scheme (e.g. "minio.internal:9000" or "s3.amazonaws.com");
// useSSL selects http vs https. pathStyle forces path-style addressing
// (bucket in the URL path rather than as a subdomain), which self-hosted
// MinIO typically requires and AWS S3 does not.
func NewS3FS(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucket, region string, useSSL, pathStyle bool) (*S3FS, error) {
if endpoint == "" || bucket == "" {
return nil, fmt.Errorf("s3 blob backend: endpoint and bucket are required")
}
lookup := minio.BucketLookupAuto
if pathStyle {
lookup = minio.BucketLookupPath
}
client, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Secure: useSSL,
Region: region,
BucketLookup: lookup,
})
if err != nil {
return nil, fmt.Errorf("create s3 client: %w", err)
}
exists, err := client.BucketExists(ctx, bucket)
if err != nil {
return nil, fmt.Errorf("check s3 bucket %q: %w", bucket, err)
}
if !exists {
if err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: region}); err != nil {
return nil, fmt.Errorf("create s3 bucket %q: %w", bucket, err)
}
}
return &S3FS{client: client, bucket: bucket}, nil
}
// Name 返回后端标识,与 file_blobs.backend 一致。
func (s *S3FS) Name() string { return "s3" }
func (s *S3FS) Put(ctx context.Context, data []byte) (string, error) {
key, _, _, err := s.PutReader(ctx, bytes.NewReader(data))
return key, err
}
// PutReader hashes the stream to a local temp file first (so the sha256 key
// and exact size are known before the S3 PUT, matching LocalFS's
// content-addressed dedup semantics -- an unknown-length streaming PUT would
// need a second copy operation to rename-by-hash after the fact, which S3
// has no equivalent of), then uploads it and checks for an existing object
// with that key first to skip a redundant PUT.
func (s *S3FS) PutReader(ctx context.Context, r io.Reader) (string, int64, []byte, error) {
tmp, err := os.CreateTemp("", "blob-s3-*.tmp")
if err != nil {
return "", 0, nil, fmt.Errorf("create s3 blob staging file: %w", err)
}
tmpPath := tmp.Name()
defer os.Remove(tmpPath)
h := sha256.New()
size, err := copyWithContext(ctx, io.MultiWriter(tmp, h), r)
closeErr := tmp.Close()
if err != nil {
return "", 0, nil, fmt.Errorf("stage s3 blob: %w", err)
}
if closeErr != nil {
return "", 0, nil, fmt.Errorf("close s3 blob staging file: %w", closeErr)
}
sum := h.Sum(nil)
key := hex.EncodeToString(sum)
if _, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}); err == nil {
return key, size, append([]byte(nil), sum...), nil
} else if !isS3NotFound(err) {
return "", 0, nil, fmt.Errorf("stat s3 blob: %w", err)
}
f, err := os.Open(tmpPath)
if err != nil {
return "", 0, nil, fmt.Errorf("reopen s3 blob staging file: %w", err)
}
defer f.Close()
if _, err := s.client.PutObject(ctx, s.bucket, key, f, size, minio.PutObjectOptions{}); err != nil {
return "", 0, nil, fmt.Errorf("put s3 blob: %w", err)
}
return key, size, append([]byte(nil), sum...), nil
}
func (s *S3FS) Get(ctx context.Context, objectKey string) ([]byte, error) {
obj, err := s.client.GetObject(ctx, s.bucket, objectKey, minio.GetObjectOptions{})
if err != nil {
return nil, fmt.Errorf("get s3 blob: %w", err)
}
defer obj.Close()
data, err := io.ReadAll(obj)
if err != nil {
return nil, fmt.Errorf("read s3 blob: %w", err)
}
return data, nil
}
// GetRange 语义与 LocalFS.GetRange 一致:只读 [offset, offset+limit) 段limit<=0 读到末尾,
// total 取自对象实际大小。
func (s *S3FS) GetRange(ctx context.Context, objectKey string, offset, limit int64) ([]byte, int64, error) {
info, err := s.client.StatObject(ctx, s.bucket, objectKey, minio.StatObjectOptions{})
if err != nil {
return nil, 0, fmt.Errorf("stat s3 blob: %w", err)
}
total := info.Size
if offset < 0 {
offset = 0
}
if offset >= total {
return []byte{}, total, nil
}
opts := minio.GetObjectOptions{}
end := total - 1
if limit > 0 && offset+limit-1 < end {
end = offset + limit - 1
}
if err := opts.SetRange(offset, end); err != nil {
return nil, 0, fmt.Errorf("set s3 range: %w", err)
}
obj, err := s.client.GetObject(ctx, s.bucket, objectKey, opts)
if err != nil {
return nil, 0, fmt.Errorf("get s3 blob range: %w", err)
}
defer obj.Close()
data, err := io.ReadAll(obj)
if err != nil {
return nil, 0, fmt.Errorf("read s3 blob range: %w", err)
}
return data, total, nil
}
// Delete removes the object at objectKey. A missing object is not an error
// (idempotent, safe to retry). Callers must have already confirmed no other
// file_blobs row on this backend still references objectKey.
func (s *S3FS) Delete(ctx context.Context, objectKey string) error {
if err := s.client.RemoveObject(ctx, s.bucket, objectKey, minio.RemoveObjectOptions{}); err != nil {
if isS3NotFound(err) {
return nil
}
return fmt.Errorf("delete s3 blob: %w", err)
}
return nil
}
func isS3NotFound(err error) bool {
resp := minio.ToErrorResponse(err)
return resp.Code == "NoSuchKey" || resp.Code == "NotFound" || resp.StatusCode == 404
}

View file

@ -0,0 +1,99 @@
package files
import "sync/atomic"
// SpaceGuard bounds how much more may be written to the permanent blob
// backend. LocalDiskSpaceGuard checks real OS free disk bytes;
// S3BudgetSpaceGuard compares a cached tracked-bytes total against a
// configured budget (S3 has no OS-level "free space" concept). Both are
// refreshed periodically by DiskUsageWorker rather than recomputed on every
// upload chunk.
type SpaceGuard interface {
// Allow reports whether writing `additional` more bytes is currently
// permitted. false should surface to the client as domain.ErrStorageFull.
Allow(additional int64) (bool, error)
// Usage returns the last-refreshed (used, total) byte snapshot for the
// admin panel. ok is false if no successful refresh has happened yet.
Usage() (used, total int64, ok bool)
}
// NoopSpaceGuard always allows writes; the default when the low-space guard
// is disabled by configuration.
type NoopSpaceGuard struct{}
func (NoopSpaceGuard) Allow(int64) (bool, error) { return true, nil }
func (NoopSpaceGuard) Usage() (int64, int64, bool) { return 0, 0, false }
// LocalDiskSpaceGuard rejects writes once cached free disk bytes fall below
// minFreeBytes (<=0 disables the check). The free-bytes figure is
// refreshed by DiskUsageWorker, not recomputed per call, to avoid a statfs
// syscall on every upload chunk -- reads are lock-free.
type LocalDiskSpaceGuard struct {
minFreeBytes int64
free atomic.Int64
total atomic.Int64
ready atomic.Bool
}
func NewLocalDiskSpaceGuard(minFreeBytes int64) *LocalDiskSpaceGuard {
return &LocalDiskSpaceGuard{minFreeBytes: minFreeBytes}
}
func (g *LocalDiskSpaceGuard) Allow(additional int64) (bool, error) {
// Before the first refresh completes, allow rather than reject: a
// startup race shouldn't turn into spurious upload failures.
if g.minFreeBytes <= 0 || !g.ready.Load() {
return true, nil
}
return g.free.Load()-additional >= g.minFreeBytes, nil
}
func (g *LocalDiskSpaceGuard) Usage() (used, total int64, ok bool) {
if !g.ready.Load() {
return 0, 0, false
}
total = g.total.Load()
used = total - g.free.Load()
if used < 0 {
used = 0
}
return used, total, true
}
func (g *LocalDiskSpaceGuard) setFree(free, total int64) {
g.free.Store(free)
g.total.Store(total)
g.ready.Store(true)
}
// S3BudgetSpaceGuard rejects writes once a cached tracked-bytes total
// (refreshed periodically from file_blobs) would exceed maxTotalBytes
// (<=0 disables the check).
type S3BudgetSpaceGuard struct {
maxTotalBytes int64
used atomic.Int64
ready atomic.Bool
}
func NewS3BudgetSpaceGuard(maxTotalBytes int64) *S3BudgetSpaceGuard {
return &S3BudgetSpaceGuard{maxTotalBytes: maxTotalBytes}
}
func (g *S3BudgetSpaceGuard) Allow(additional int64) (bool, error) {
if g.maxTotalBytes <= 0 || !g.ready.Load() {
return true, nil
}
return g.used.Load()+additional <= g.maxTotalBytes, nil
}
func (g *S3BudgetSpaceGuard) Usage() (used, total int64, ok bool) {
if !g.ready.Load() {
return 0, 0, false
}
return g.used.Load(), g.maxTotalBytes, true
}
func (g *S3BudgetSpaceGuard) setUsed(used int64) {
g.used.Store(used)
g.ready.Store(true)
}

View file

@ -0,0 +1,76 @@
package files
import "testing"
func TestNoopSpaceGuardAlwaysAllows(t *testing.T) {
g := NoopSpaceGuard{}
if allowed, err := g.Allow(1 << 40); err != nil || !allowed {
t.Fatalf("expected noop guard to always allow, got allowed=%v err=%v", allowed, err)
}
if _, _, ok := g.Usage(); ok {
t.Fatal("expected noop guard to report no usage snapshot")
}
}
func TestLocalDiskSpaceGuardBeforeFirstRefresh(t *testing.T) {
g := NewLocalDiskSpaceGuard(1 << 30)
// Before setFree has ever run, the guard must not reject -- a startup
// race shouldn't turn into spurious upload failures.
if allowed, err := g.Allow(1 << 40); err != nil || !allowed {
t.Fatalf("expected pre-refresh guard to allow, got allowed=%v err=%v", allowed, err)
}
if _, _, ok := g.Usage(); ok {
t.Fatal("expected no usage snapshot before first refresh")
}
}
func TestLocalDiskSpaceGuardThreshold(t *testing.T) {
const minFree = int64(1000)
g := NewLocalDiskSpaceGuard(minFree)
g.setFree(1500, 10000)
if allowed, err := g.Allow(400); err != nil || !allowed {
t.Fatalf("writing 400 bytes leaves 1100 free (>= 1000 min): want allow, got allowed=%v err=%v", allowed, err)
}
if allowed, err := g.Allow(600); err != nil || allowed {
t.Fatalf("writing 600 bytes leaves 900 free (< 1000 min): want reject, got allowed=%v err=%v", allowed, err)
}
used, total, ok := g.Usage()
if !ok || total != 10000 || used != 8500 {
t.Fatalf("usage snapshot = used=%d total=%d ok=%v, want used=8500 total=10000 ok=true", used, total, ok)
}
}
func TestLocalDiskSpaceGuardDisabled(t *testing.T) {
g := NewLocalDiskSpaceGuard(0)
g.setFree(10, 1000)
if allowed, err := g.Allow(1 << 40); err != nil || !allowed {
t.Fatalf("minFreeBytes<=0 must disable the check, got allowed=%v err=%v", allowed, err)
}
}
func TestS3BudgetSpaceGuardThreshold(t *testing.T) {
const maxTotal = int64(10000)
g := NewS3BudgetSpaceGuard(maxTotal)
g.setUsed(9000)
if allowed, err := g.Allow(1000); err != nil || !allowed {
t.Fatalf("9000+1000 == budget: want allow, got allowed=%v err=%v", allowed, err)
}
if allowed, err := g.Allow(1001); err != nil || allowed {
t.Fatalf("9000+1001 exceeds budget: want reject, got allowed=%v err=%v", allowed, err)
}
used, total, ok := g.Usage()
if !ok || total != maxTotal || used != 9000 {
t.Fatalf("usage snapshot = used=%d total=%d ok=%v, want used=9000 total=%d ok=true", used, total, ok, maxTotal)
}
}
func TestS3BudgetSpaceGuardBeforeFirstRefresh(t *testing.T) {
g := NewS3BudgetSpaceGuard(100)
if allowed, err := g.Allow(1 << 40); err != nil || !allowed {
t.Fatalf("expected pre-refresh guard to allow, got allowed=%v err=%v", allowed, err)
}
}

View file

@ -0,0 +1,15 @@
//go:build !windows
package files
import "golang.org/x/sys/unix"
// localDiskFreeBytes returns free (available to an unprivileged writer, not
// counting reserved blocks) and total bytes for the filesystem containing path.
func localDiskFreeBytes(path string) (free, total int64, err error) {
var st unix.Statfs_t
if err := unix.Statfs(path, &st); err != nil {
return 0, 0, err
}
return int64(st.Bavail) * int64(st.Bsize), int64(st.Blocks) * int64(st.Bsize), nil
}

View file

@ -0,0 +1,19 @@
//go:build windows
package files
import "golang.org/x/sys/windows"
// localDiskFreeBytes returns free (available to the calling user) and total
// bytes for the volume containing path.
func localDiskFreeBytes(path string) (free, total int64, err error) {
ptr, err := windows.UTF16PtrFromString(path)
if err != nil {
return 0, 0, err
}
var freeAvail, totalBytes, totalFree uint64
if err := windows.GetDiskFreeSpaceEx(ptr, &freeAvail, &totalBytes, &totalFree); err != nil {
return 0, 0, err
}
return int64(freeAvail), int64(totalBytes), nil
}

View file

@ -0,0 +1,81 @@
package files
import (
"context"
"time"
"go.uber.org/zap"
)
// SumFileBlobBytesStore is the minimal store dependency DiskUsageWorker
// needs to refresh an S3BudgetSpaceGuard.
type SumFileBlobBytesStore interface {
SumFileBlobBytes(ctx context.Context) (int64, error)
}
// DiskUsageWorker periodically refreshes one SpaceGuard's cached usage
// snapshot, so the upload path never pays a statfs syscall / SUM(size)
// query per chunk.
type DiskUsageWorker struct {
interval time.Duration
log *zap.Logger
refresh func(ctx context.Context) error
}
// NewLocalDiskUsageWorker refreshes a LocalDiskSpaceGuard from real OS free
// disk bytes under root (the blob backend's storage directory).
func NewLocalDiskUsageWorker(guard *LocalDiskSpaceGuard, root string, interval time.Duration, log *zap.Logger) *DiskUsageWorker {
return newDiskUsageWorker(interval, log, func(context.Context) error {
free, total, err := localDiskFreeBytes(root)
if err != nil {
return err
}
guard.setFree(free, total)
return nil
})
}
// NewS3DiskUsageWorker refreshes an S3BudgetSpaceGuard from the tracked
// file_blobs byte total.
func NewS3DiskUsageWorker(guard *S3BudgetSpaceGuard, media SumFileBlobBytesStore, interval time.Duration, log *zap.Logger) *DiskUsageWorker {
return newDiskUsageWorker(interval, log, func(ctx context.Context) error {
used, err := media.SumFileBlobBytes(ctx)
if err != nil {
return err
}
guard.setUsed(used)
return nil
})
}
func newDiskUsageWorker(interval time.Duration, log *zap.Logger, refresh func(context.Context) error) *DiskUsageWorker {
if interval <= 0 {
interval = time.Minute
}
if log == nil {
log = zap.NewNop()
}
return &DiskUsageWorker{interval: interval, log: log, refresh: refresh}
}
// Run refreshes once immediately (so the guard isn't stuck "not ready" for
// a full interval after startup), then on every tick until ctx is done.
func (w *DiskUsageWorker) Run(ctx context.Context) {
w.refreshOnce(ctx)
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
w.refreshOnce(ctx)
}
}
}
func (w *DiskUsageWorker) refreshOnce(ctx context.Context) {
if err := w.refresh(ctx); err != nil {
w.log.Warn("refresh storage usage snapshot failed", zap.Error(err))
}
}

View file

@ -63,7 +63,7 @@ func (s *Service) CreatePhotoFromUpload(ctx context.Context, file domain.Uploade
if len(data) == 0 {
return domain.Photo{}, domain.ErrPhotoInvalid
}
photo, err := s.createPhoto(ctx, data, photoSizeSpecsForMessage(data))
photo, err := s.createPhoto(ctx, data, photoSizeSpecsForMessage(data), file.OwnerUserID)
if err != nil {
return domain.Photo{}, err
}
@ -86,11 +86,13 @@ func (s *Service) CreatePhotoFromUpload(ctx context.Context, file domain.Uploade
}
// CreatePhotoFromBytes stores already-fetched image bytes as a message Photo.
// There is no uploader (e.g. a server-fetched webpage preview image), so the
// photo is attributed to the system (owner_user_id 0) for storage accounting.
func (s *Service) CreatePhotoFromBytes(ctx context.Context, data []byte) (domain.Photo, error) {
if len(data) == 0 {
return domain.Photo{}, domain.ErrPhotoInvalid
}
return s.createPhoto(ctx, data, photoSizeSpecsForMessage(data))
return s.createPhoto(ctx, data, photoSizeSpecsForMessage(data), 0)
}
// GetPhoto 按 id 返回已存储照片。
@ -148,7 +150,7 @@ func (s *Service) CreateAvatarFromUpload(ctx context.Context, file domain.Upload
if len(data) == 0 {
return domain.Photo{}, domain.ErrPhotoInvalid
}
return s.createAvatarPhoto(ctx, data)
return s.createAvatarPhoto(ctx, data, file.OwnerUserID)
}
// CreateAvatarVideoFromUpload stores an animated profile video as photo.video_sizes.
@ -206,6 +208,7 @@ func (s *Service) createAvatarVideoFromUpload(ctx context.Context, file domain.U
Date: int(time.Now().Unix()),
DCID: s.dc,
Sizes: sizes,
OwnerUserID: file.OwnerUserID,
}
if err := s.media.PutPhoto(ctx, photo); err != nil {
return domain.Photo{}, err
@ -306,6 +309,7 @@ func (s *Service) CreateDocumentFromUpload(ctx context.Context, file domain.Uplo
Size: body.Size,
DCID: s.dc,
Attributes: spec.Attributes,
OwnerUserID: file.OwnerUserID,
}
thumbMaterialized := false
if spec.Thumb != nil {
@ -620,7 +624,7 @@ func (s *Service) DeleteProfilePhotosKind(ctx context.Context, ownerType domain.
}
// createPhoto 把字节落 blob每个尺寸一个 location_key指向同一内容并写 photos 表。
func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSizeSpec) (domain.Photo, error) {
func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSizeSpec, ownerUserID int64) (domain.Photo, error) {
photoID := randomID()
sizes, err := s.putPhotoStaticSizes(ctx, photoID, data, specs)
if err != nil {
@ -633,6 +637,7 @@ func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSiz
Date: int(time.Now().Unix()),
DCID: s.dc,
Sizes: sizes,
OwnerUserID: ownerUserID,
}
if err := s.media.PutPhoto(ctx, photo); err != nil {
return domain.Photo{}, err
@ -640,7 +645,7 @@ func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSiz
return photo, nil
}
func (s *Service) createAvatarPhoto(ctx context.Context, data []byte) (domain.Photo, error) {
func (s *Service) createAvatarPhoto(ctx context.Context, data []byte, ownerUserID int64) (domain.Photo, error) {
photoID := randomID()
sizes, err := s.putAvatarStaticSizes(ctx, photoID, data, photoSizeSpecsForAvatar(data))
if err != nil {
@ -653,6 +658,7 @@ func (s *Service) createAvatarPhoto(ctx context.Context, data []byte) (domain.Ph
Date: int(time.Now().Unix()),
DCID: s.dc,
Sizes: sizes,
OwnerUserID: ownerUserID,
}
if err := s.media.PutPhoto(ctx, photo); err != nil {
return domain.Photo{}, err

View file

@ -0,0 +1,94 @@
package files
import (
"context"
"fmt"
"time"
"go.uber.org/zap"
"telesrv/internal/domain"
)
// mediaRetentionStore is implemented by store.MediaStore backends that
// support the storage retention sweep (currently only the Postgres store).
// A type assertion, not a MediaStore interface method, keeps these
// admin/maintenance-only queries out of the hot RPC-facing interface --
// same convention as photoBatchStore above.
type mediaRetentionStore interface {
ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
CountFileBlobRefs(ctx context.Context, backend, objectKey string) (int, error)
DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
DeletePhotoAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
}
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
// permanently deletes documents/photos that have had no live reference
// (message/profile-photo/sticker-set, see media_references) since at least
// cutoff, along with their blob(s) -- but only physically removes bytes
// from the backend once confirming no other file_blobs row still needs the
// object, since content-addressed storage means the same bytes can be
// shared across documents/photos.
func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
store, ok := s.media.(mediaRetentionStore)
if !ok || limit <= 0 {
return 0, nil
}
deleted := 0
docIDs, err := store.ListOrphanedDocumentIDsOlderThan(ctx, cutoff, limit)
if err != nil {
return deleted, fmt.Errorf("list orphaned documents: %w", err)
}
for _, id := range docIDs {
blobs, err := store.DeleteDocumentAndBlobs(ctx, id)
if err != nil {
s.log.Warn("delete orphaned document failed", zap.Int64("document_id", id), zap.Error(err))
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
deleted++
}
photoIDs, err := store.ListOrphanedPhotoIDsOlderThan(ctx, cutoff, limit)
if err != nil {
return deleted, fmt.Errorf("list orphaned photos: %w", err)
}
for _, id := range photoIDs {
blobs, err := store.DeletePhotoAndBlobs(ctx, id)
if err != nil {
s.log.Warn("delete orphaned photo failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
deleted++
}
return deleted, nil
}
// deleteOrphanedBlobs removes each blob from its backend once confirming
// (via CountFileBlobRefs) no other file_blobs row still references
// (backend, object_key). Only blobs on the currently active backend are
// physically removed -- a blob left over from a previously active backend
// (deployment switched TELESRV_BLOB_BACKEND at some point; switching back
// isn't supported) is logged and skipped rather than silently dropped,
// since there's no configured client to reach it right now anyway.
func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionStore, blobs []domain.FileBlob) {
for _, b := range blobs {
refs, err := store.CountFileBlobRefs(ctx, string(b.Backend), b.ObjectKey)
if err != nil {
s.log.Warn("count file blob refs failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
continue
}
if refs > 0 {
continue
}
if string(b.Backend) != s.blobs.Name() {
s.log.Warn("orphaned blob is on an inactive backend, skipping physical delete",
zap.String("backend", string(b.Backend)), zap.String("object_key", b.ObjectKey))
continue
}
if err := s.blobs.Delete(ctx, b.ObjectKey); err != nil {
s.log.Warn("delete orphaned blob failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
}
}
}

View file

@ -168,6 +168,16 @@ func (f *fakeMediaStore) GetFileBlobs(_ context.Context, keys []string) (map[str
return out, nil
}
func (f *fakeMediaStore) SumFileBlobBytes(_ context.Context) (int64, error) {
f.mu.Lock()
defer f.mu.Unlock()
var total int64
for _, b := range f.blobs {
total += b.Size
}
return total, nil
}
func (f *fakeMediaStore) GetSeedState(_ context.Context, key string) (string, bool, error) {
f.mu.Lock()
defer f.mu.Unlock()

View file

@ -64,6 +64,7 @@ type Service struct {
stickerSetCache *stickerSetFullCache
stickerSetNegCache *stickerSetNegativeCache
uploadQuota domain.UploadPartQuota
spaceGuard SpaceGuard
mapTiles *mapTileProxy
externalMedia *externalMediaFetcher
webpage *webpageFetcher
@ -116,6 +117,30 @@ func WithUploadPartQuota(quota domain.UploadPartQuota) Option {
}
}
// WithSpaceGuard installs the low-disk-space upload guard. Not calling this
// (or passing nil) leaves the default NoopSpaceGuard, which never rejects.
func WithSpaceGuard(guard SpaceGuard) Option {
return func(s *Service) {
if guard != nil {
s.spaceGuard = guard
}
}
}
// WithUploadPartBackend overrides where transient upload-part chunks are
// staged before assembly, independent of the permanent blob backend passed
// to NewService. Needed when the permanent backend is s3 (S3FS doesn't
// implement UploadPartBackend -- chunk-per-request S3 round trips aren't
// worth it for scratch data deleted within minutes): pass a LocalFS here so
// uploads keep working, while permanent blobs still land in s3.
func WithUploadPartBackend(backend UploadPartBackend) Option {
return func(s *Service) {
if backend != nil {
s.uploadParts = backend
}
}
}
// NewService 创建 files 服务。dc 是本 server 的 DC id写入新建 document/photo 的 dc_id。
func NewService(media store.MediaStore, blobs BlobBackend, dc int, opts ...Option) *Service {
s := &Service{
@ -132,6 +157,7 @@ func NewService(media store.MediaStore, blobs BlobBackend, dc int, opts ...Optio
MaxParts: DefaultUploadInFlightMaxParts,
MaxFiles: DefaultUploadInFlightMaxFiles,
},
spaceGuard: NoopSpaceGuard{},
}
if partBackend, ok := blobs.(UploadPartBackend); ok {
s.uploadParts = partBackend
@ -205,6 +231,16 @@ func (s *Service) saveFilePart(ctx context.Context, part domain.UploadPart, byte
if s.uploadParts == nil {
return fmt.Errorf("upload part backend not configured")
}
// Cheapest possible rejection point: reject before any disk write once
// the permanent blob backend is low on space. Upload parts themselves
// always land on local scratch disk (see UploadPartBackend), but a
// low-space condition on the permanent backend means assembly will
// fail anyway, so there's no point accepting more chunks toward it.
if allowed, err := s.spaceGuard.Allow(int64(len(bytes))); err != nil {
return err
} else if !allowed {
return domain.ErrStorageFull
}
slot, err := s.checkUploadPartQuota(ctx, part)
if err != nil {
return err
@ -584,10 +620,19 @@ type assembledUploadBlob struct {
// assembleUploadBlob 把上传分片流式写入正式 blob。调用方应在 durable media 元数据
// 成功提交后调用 cleanupUploadParts避免 metadata 写失败时丢失可重试的上传分片。
func (s *Service) assembleUploadBlob(ctx context.Context, ownerUserID, fileID int64, expectedParts int) (assembledUploadBlob, error) {
parts, _, err := s.loadAndValidateUploadParts(ctx, ownerUserID, fileID, expectedParts)
parts, total, err := s.loadAndValidateUploadParts(ctx, ownerUserID, fileID, expectedParts)
if err != nil {
return assembledUploadBlob{}, err
}
// Re-check free space against the full assembled size right before
// committing to the permanent blob backend: SaveFilePart already
// checked each chunk, but free space may have dropped since then over
// the lifetime of a large multi-part upload.
if allowed, err := s.spaceGuard.Allow(total); err != nil {
return assembledUploadBlob{}, err
} else if !allowed {
return assembledUploadBlob{}, domain.ErrStorageFull
}
if s.uploadParts == nil {
return assembledUploadBlob{}, fmt.Errorf("upload part backend not configured")
}

View file

@ -0,0 +1,27 @@
package maintenance
import (
"context"
"time"
)
// OrphanedMediaRetentionStore deletes documents/photos that have been
// orphaned (no live message/profile-photo/sticker-set reference remains,
// tracked via media_references + orphaned_at) for at least the configured
// age, along with their underlying blob once no other file_blobs row on
// its backend still needs it. Never touches media that still has a live
// reference, regardless of age.
type OrphanedMediaRetentionStore interface {
DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error)
}
// WithOrphanedMediaRetention enables the storage retention sweep. maxAge is
// how long a document/photo must have been orphaned before it's actually
// deleted (not how old the media itself is) -- <=0 leaves the sweep
// disabled even if a store is provided, matching
// TELESRV_STORAGE_RETENTION_ENABLE=false being the safe default.
func (w *RetentionWorker) WithOrphanedMediaRetention(store OrphanedMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
w.orphanedMedia = store
w.orphanedMediaMaxAge = maxAge
return w
}

View file

@ -118,6 +118,7 @@ type RetentionWorker struct {
orphanAuthKeys OrphanAuthKeyRetentionStore
activeAuthKeys ActiveRawAuthKeyProvider
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
orphanedMedia OrphanedMediaRetentionStore
logger *zap.Logger
retention time.Duration
botAPIRetention time.Duration
@ -126,6 +127,7 @@ type RetentionWorker struct {
authDeliveryReportRetention time.Duration
outboxPoisonRetention time.Duration
outboxPoisonInterval time.Duration
orphanedMediaMaxAge time.Duration
interval time.Duration
batch int
}
@ -412,6 +414,17 @@ func (w *RetentionWorker) runRetentionOnce(ctx context.Context) {
w.logger.Info("expired channel_update_events contiguous-prefix cleanup complete", zap.Int("deleted", channelDeleted))
}
}
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
// Only ever touches documents/photos already marked orphaned (no live
// message/profile-photo/sticker-set reference remains) -- media still
// visible in a conversation is never a candidate, regardless of age.
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
if err != nil {
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
} else if mediaDeleted > 0 {
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
}
}
}
func (w *RetentionWorker) orphanHeartbeatInterval() time.Duration {

View file

@ -253,6 +253,45 @@ type Config struct {
StarGiftTONStartingGrant int64
// BlobDir 是本地磁盘 blob backend 根目录(媒体文件字节内容)。
BlobDir string
// BlobBackendKind selects the blob storage backend: "localfs" (default)
// or "s3". Transient upload parts always stay on local disk (BlobDir)
// regardless of this setting -- only the permanent blob store moves.
BlobBackendKind string
// S3Endpoint/S3Region/S3Bucket/S3AccessKeyID/S3SecretAccessKey/S3UseSSL/
// S3PathStyle configure the s3 blob backend; only used when
// BlobBackendKind == "s3". Works against self-hosted MinIO or AWS S3.
S3Endpoint string
S3Region string
S3Bucket string
S3AccessKeyID string
S3SecretAccessKey string
S3UseSSL bool
S3PathStyle bool
// StorageLowSpaceGuardEnable turns on the pre-upload free-space check.
StorageLowSpaceGuardEnable bool
// StorageMinFreeBytes: for the localfs backend, reject new uploads once
// real free disk bytes fall below this. <=0 disables this check.
StorageMinFreeBytes int64
// StorageMaxTotalBytes: reject new uploads once total tracked blob
// bytes would exceed this -- the only meaningful "low space" signal for
// the s3 backend (no OS-level free space concept), and usable as an
// optional soft budget cap on localfs too. <=0 disables this check.
StorageMaxTotalBytes int64
// StorageUsageRefreshInterval is how often the cached free-space/budget
// usage gauge refreshes; <=0 uses a 1 minute default.
StorageUsageRefreshInterval time.Duration
// StorageRetentionEnable turns on the orphaned-media age sweep. Off by
// default: orphaned media (no live message/profile-photo/sticker-set
// reference) is tracked and visible in the admin panel, but nothing is
// auto-deleted until the operator explicitly opts in.
StorageRetentionEnable bool
// StorageRetentionMaxAge is how long a document/photo must have been
// orphaned (not how old the media itself is) before the sweep deletes
// it. Never deletes media that still has a live reference, regardless
// of age. <=0 uses a 30 day default. The sweep itself runs on the
// shared RetentionInterval/RetentionBatch cadence alongside every other
// retention check (see maintenance.RetentionWorker).
StorageRetentionMaxAge time.Duration
// StickerSeedDir 是 reaction / sticker 资源种子目录(导入到 documents/sticker_sets + blob
StickerSeedDir string
// StickerSeedMaxSets 限制导入的常规贴纸集数量(避免启动时导入过多包),<=0 表示不限。
@ -756,6 +795,20 @@ func Load() (Config, error) {
OfficialGiftsDir: envOr("TELESRV_OFFICIAL_GIFTS_DIR", "data/official-gifts"),
StarGiftTONStartingGrant: envInt64Or("TELESRV_STARGIFT_TON_STARTING_GRANT", 10_000_000_000),
BlobDir: envOr("TELESRV_BLOB_DIR", "data/blobs"),
BlobBackendKind: strings.ToLower(strings.TrimSpace(envOr("TELESRV_BLOB_BACKEND", "localfs"))),
S3Endpoint: envOr("TELESRV_S3_ENDPOINT", ""),
S3Region: envOr("TELESRV_S3_REGION", "us-east-1"),
S3Bucket: envOr("TELESRV_S3_BUCKET", ""),
S3AccessKeyID: envOr("TELESRV_S3_ACCESS_KEY_ID", ""),
S3SecretAccessKey: envOr("TELESRV_S3_SECRET_ACCESS_KEY", ""),
S3UseSSL: envBoolOr("TELESRV_S3_USE_SSL", true),
S3PathStyle: envBoolOr("TELESRV_S3_PATH_STYLE", false),
StorageLowSpaceGuardEnable: envBoolOr("TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE", true),
StorageMinFreeBytes: envInt64Or("TELESRV_STORAGE_MIN_FREE_BYTES", 1<<30),
StorageMaxTotalBytes: envInt64Or("TELESRV_STORAGE_MAX_TOTAL_BYTES", 0),
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
StorageRetentionEnable: envBoolOr("TELESRV_STORAGE_RETENTION_ENABLE", false),
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"),
@ -930,6 +983,9 @@ func Load() (Config, error) {
if err := validateTelegramLoginConfig(cfg); err != nil {
return Config{}, err
}
if err := validateStorageConfig(cfg); err != nil {
return Config{}, err
}
return cfg, nil
}
@ -1132,6 +1188,40 @@ func validateVerificationConfig(cfg Config) error {
return nil
}
// validateStorageConfig checks the blob backend selection and storage
// management (low-space guard, retention sweep) settings.
func validateStorageConfig(cfg Config) error {
switch cfg.BlobBackendKind {
case "localfs":
// no extra requirements
case "s3":
if strings.TrimSpace(cfg.S3Endpoint) == "" {
return fmt.Errorf("TELESRV_S3_ENDPOINT is required when TELESRV_BLOB_BACKEND=s3")
}
if strings.TrimSpace(cfg.S3Bucket) == "" {
return fmt.Errorf("TELESRV_S3_BUCKET is required when TELESRV_BLOB_BACKEND=s3")
}
default:
return fmt.Errorf("TELESRV_BLOB_BACKEND must be \"localfs\" or \"s3\", got %q", cfg.BlobBackendKind)
}
if cfg.StorageMinFreeBytes < 0 {
return fmt.Errorf("TELESRV_STORAGE_MIN_FREE_BYTES must be non-negative")
}
if cfg.StorageMaxTotalBytes < 0 {
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
}
if cfg.StorageUsageRefreshInterval < 0 {
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be non-negative")
}
if cfg.StorageRetentionMaxAge < 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
}
if cfg.StorageRetentionEnable && cfg.StorageRetentionMaxAge <= 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_ENABLE is true")
}
return nil
}
// validateAdminRBACConfig checks the panel/adminapi permission configuration.
// An unparsable permission name is refused rather than ignored: a silently
// dropped permission is either a lockout or an unintended grant.

View file

@ -18,6 +18,8 @@ type MediaBackend string
const (
// MediaBackendLocalFS 表示 blob 字节存在本地磁盘object_key 为相对路径)。
MediaBackendLocalFS MediaBackend = "localfs"
// MediaBackendS3 表示 blob 字节存在 S3 兼容对象存储MinIO 或 AWS S3
MediaBackendS3 MediaBackend = "s3"
)
// FileBlob 是一个可下载的二进制对象的索引项location_key → 后端/对象键/大小/mime。
@ -233,6 +235,10 @@ type Document struct {
DCID int `json:"dc_id,omitempty"`
Attributes []DocumentAttribute `json:"attributes,omitempty"`
Thumbs []PhotoSize `json:"thumbs,omitempty"`
// OwnerUserID is who uploaded this document; 0 for rows predating storage
// accounting or for system-originated documents. Used for per-account
// storage usage reporting, not part of the tg wire protocol.
OwnerUserID int64 `json:"owner_user_id,omitempty"`
}
// StickerSetRef 返回该文档归属的贴纸集引用(若有 sticker/custom_emoji 属性)。
@ -384,6 +390,9 @@ type Photo struct {
DCID int `json:"dc_id,omitempty"`
HasStickers bool `json:"has_stickers,omitempty"`
Sizes []PhotoSize `json:"sizes,omitempty"`
// OwnerUserID is who uploaded this photo; 0 for rows predating storage
// accounting or for system-originated photos (e.g. webpage previews).
OwnerUserID int64 `json:"owner_user_id,omitempty"`
}
func ClonePhotoPtr(photo *Photo) *Photo {

View file

@ -11,4 +11,8 @@ var (
ErrUploadQuotaExceeded = errors.New("upload quota exceeded")
ErrPhotoInvalid = errors.New("photo invalid")
ErrDocumentInvalid = errors.New("document invalid")
// ErrStorageFull is returned when the configured low-space guard rejects a
// write: local disk free bytes (or, on the s3 backend, the configured
// total-bytes budget) has fallen below the configured threshold.
ErrStorageFull = errors.New("storage full")
)

View file

@ -0,0 +1,90 @@
package domain
// MediaKind identifies which table a media_references row points into.
type MediaKind string
const (
MediaKindDocument MediaKind = "document"
MediaKindPhoto MediaKind = "photo"
)
// MediaRefKind identifies why a document/photo is still considered "live" --
// each distinct kind of place that can hold a durable pointer to media gets
// its own ref_kind, so removing one kind of reference (e.g. a deleted
// message) doesn't accidentally drop a still-live reference of another kind
// (e.g. the same document also set as someone's profile photo).
type MediaRefKind string
const (
MediaRefKindMessageBox MediaRefKind = "message_box"
MediaRefKindChannelMessage MediaRefKind = "channel_message"
MediaRefKindProfilePhoto MediaRefKind = "profile_photo"
MediaRefKindStickerSet MediaRefKind = "sticker_set"
MediaRefKindGift MediaRefKind = "gift"
)
// MediaReference is one live pointer to a document/photo. GC (the storage
// retention sweep) only considers a document/photo eligible for deletion
// once every reference to it has been removed.
type MediaReference struct {
Kind MediaKind
MediaID int64
RefKind MediaRefKind
RefKey string
}
// OrphanCandidate is a document/photo whose last reference has been removed
// (orphaned_at is set) and is old enough to be considered for the storage
// retention sweep.
type OrphanCandidate struct {
Kind MediaKind
MediaID int64
Backend MediaBackend
ObjectKey string
Size int64
OrphanedAt int64 // unix seconds
}
// MediaRefTarget identifies one document/photo embedded in a message's media
// snapshot.
type MediaRefTarget struct {
Kind MediaKind
ID int64
}
// ExtractMediaRefTargets returns every document/photo id embedded in a
// message's media snapshot (including nested ones -- a live photo's video
// document, a webpage preview's photo), deduplicated. Used to register/drop
// media_references rows when a message carrying this media is
// created/edited/deleted.
func ExtractMediaRefTargets(media *MessageMedia) []MediaRefTarget {
if media == nil {
return nil
}
seen := make(map[MediaRefTarget]struct{}, 2)
var out []MediaRefTarget
add := func(kind MediaKind, id int64) {
if id == 0 {
return
}
t := MediaRefTarget{Kind: kind, ID: id}
if _, ok := seen[t]; ok {
return
}
seen[t] = struct{}{}
out = append(out, t)
}
if media.Photo != nil {
add(MediaKindPhoto, media.Photo.ID)
}
if media.Document != nil {
add(MediaKindDocument, media.Document.ID)
}
if media.LivePhotoVideo != nil {
add(MediaKindDocument, media.LivePhotoVideo.ID)
}
if media.WebPage != nil && media.WebPage.Photo != nil {
add(MediaKindPhoto, media.WebPage.Photo.ID)
}
return out
}

View file

@ -107,6 +107,11 @@ func locationInvalidErr() error { return tgerr.New(400, "LOCATION_INVALID")
func fileIDInvalidErr() error { return tgerr.New(400, "FILE_ID_INVALID") }
func documentInvalidErr() error { return tgerr.New(400, "DOCUMENT_INVALID") }
// storageFullErr surfaces the low-disk-space upload guard. Deliberately not
// a flood-wait: a full disk won't resolve itself in 60 seconds, and telling
// the client to retry shortly would be misleading.
func storageFullErr() error { return tgerr.New(400, "STORAGE_FULL") }
func mediaEmptyErr() error { return tgerr.New(400, "MEDIA_EMPTY") }
func frozenMethodInvalidErr() error { return tgerr.New(420, "FROZEN_METHOD_INVALID") }

View file

@ -684,6 +684,8 @@ func photoUploadErr(err error) error {
return filePartsInvalidErr()
case errors.Is(err, domain.ErrPhotoInvalid):
return photoInvalidErr()
case errors.Is(err, domain.ErrStorageFull):
return storageFullErr()
default:
return internalErr()
}

View file

@ -1144,6 +1144,8 @@ func mediaUploadErr(err error) error {
return photoInvalidErr()
case errors.Is(err, domain.ErrDocumentInvalid):
return mediaInvalidErr()
case errors.Is(err, domain.ErrStorageFull):
return storageFullErr()
default:
return internalErr()
}

View file

@ -327,6 +327,8 @@ func fileSaveErr(err error) error {
return filePartTooBigErr()
case errors.Is(err, domain.ErrUploadQuotaExceeded):
return floodWaitErr(60)
case errors.Is(err, domain.ErrStorageFull):
return storageFullErr()
default:
return internalErr()
}

View file

@ -30,6 +30,10 @@ type MediaStore interface {
// GetFileBlobs 批量按 location_key 取 FileBlob 元数据(缺失的 key 不出现在返回 map 中)。
// 供启动预热等需一次性加载大量 blob 的路径,替代逐个 GetFileBlob 的 N+1 往返。
GetFileBlobs(ctx context.Context, locationKeys []string) (map[string]domain.FileBlob, error)
// SumFileBlobBytes 返回全部 blob 的物理字节总量(去重后,同内容只算一次)。
// 供低磁盘空间守卫的周期性用量刷新使用(尤其是 s3 backend 的预算模式,没有
// 操作系统级"剩余空间"概念,只能靠这个累计值和配置的预算比较)。
SumFileBlobBytes(ctx context.Context) (int64, error)
// seed 状态。只记录静态资源 catalog 的内容 hash用于启动时跳过未变化的重复导入
// 真实可服务性仍由 documents/file_blobs 校验保证,不能只相信这里的 hash。

View file

@ -479,6 +479,11 @@ ORDER BY id`, channel.ID, id32)
// 删除入口统一静默跳过(官方客户端对它禁用删除)。
continue
}
// Drop this message's media_references (storage GC); orphans the
// document/photo if this was its last live reference anywhere.
if err := removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindChannelMessage, channelMessageRefKey(channel.ID, id)); err != nil {
return nil, domain.ChannelUpdateEvent{}, channel, fmt.Errorf("remove deleted channel message media references: %w", err)
}
deleted = append(deleted, id)
if err := s.insertChannelAdminLogTx(ctx, tx, domain.ChannelAdminLogEvent{
ChannelID: channel.ID,

View file

@ -272,6 +272,10 @@ WHERE location_key = ANY($1::text[])`, locationKeys)
return out, nil
}
func (s *MediaStore) SumFileBlobBytes(ctx context.Context) (int64, error) {
return s.q.SumFileBlobBytes(ctx)
}
func (s *MediaStore) GetSeedState(ctx context.Context, key string) (string, bool, error) {
var hash string
if err := s.db.QueryRow(ctx, `
@ -332,6 +336,7 @@ func putDocumentParams(doc domain.Document) (sqlcgen.PutDocumentParams, error) {
DcID: int32(doc.DCID),
AttributesJson: attrs,
ThumbsJson: thumbs,
OwnerUserID: doc.OwnerUserID,
}, nil
}
@ -472,6 +477,20 @@ func (c *documentMetaCache) put(id int64, doc domain.Document) {
}
}
// remove evicts id, e.g. after the row is permanently deleted (storage
// retention sweep) so a stale cache hit can't outlive the row.
func (c *documentMetaCache) remove(id int64) {
if c == nil || id == 0 {
return
}
c.mu.Lock()
defer c.mu.Unlock()
if el, ok := c.m[id]; ok {
c.ll.Remove(el)
delete(c.m, id)
}
}
func cloneDocument(doc domain.Document) domain.Document {
doc.FileReference = append([]byte(nil), doc.FileReference...)
if len(doc.Attributes) > 0 {
@ -514,6 +533,7 @@ func documentFromRow(row sqlcgen.GetDocumentRow) (domain.Document, error) {
DCID: int(row.DcID),
Attributes: attrs,
Thumbs: thumbs,
OwnerUserID: row.OwnerUserID,
}, nil
}
@ -532,6 +552,7 @@ func (s *MediaStore) PutPhoto(ctx context.Context, photo domain.Photo) error {
DcID: int32(photo.DCID),
HasStickers: photo.HasStickers,
SizesJson: sizes,
OwnerUserID: photo.OwnerUserID,
})
}
@ -543,7 +564,7 @@ func (s *MediaStore) GetPhoto(ctx context.Context, id int64) (domain.Photo, bool
}
return domain.Photo{}, false, err
}
photo, err := photoFromFields(row.ID, row.AccessHash, row.FileReference, int(row.Date), int(row.DcID), row.HasStickers, row.SizesJson)
photo, err := photoFromFields(row.ID, row.AccessHash, row.FileReference, int(row.Date), int(row.DcID), row.HasStickers, row.SizesJson, row.OwnerUserID)
if err != nil {
return domain.Photo{}, false, err
}
@ -572,7 +593,7 @@ func (s *MediaStore) GetPhotos(ctx context.Context, ids []int64) ([]domain.Photo
return nil, nil
}
rows, err := s.db.Query(ctx, `
SELECT id, access_hash, file_reference, date, dc_id, has_stickers, sizes::text
SELECT id, access_hash, file_reference, date, dc_id, has_stickers, sizes::text, owner_user_id
FROM photos
WHERE id = ANY($1::bigint[])
`, unique)
@ -613,14 +634,15 @@ func scanPhotoRow(row photoScanner) (domain.Photo, error) {
dcID int32
hasStickers bool
sizesJSON string
ownerUserID int64
)
if err := row.Scan(&id, &accessHash, &fileReference, &date, &dcID, &hasStickers, &sizesJSON); err != nil {
if err := row.Scan(&id, &accessHash, &fileReference, &date, &dcID, &hasStickers, &sizesJSON, &ownerUserID); err != nil {
return domain.Photo{}, err
}
return photoFromFields(id, accessHash, fileReference, int(date), int(dcID), hasStickers, sizesJSON)
return photoFromFields(id, accessHash, fileReference, int(date), int(dcID), hasStickers, sizesJSON, ownerUserID)
}
func photoFromFields(id, accessHash int64, fileReference []byte, date, dcID int, hasStickers bool, sizesJSON string) (domain.Photo, error) {
func photoFromFields(id, accessHash int64, fileReference []byte, date, dcID int, hasStickers bool, sizesJSON string, ownerUserID int64) (domain.Photo, error) {
sizes, err := decodePhotoSizes(sizesJSON)
if err != nil {
return domain.Photo{}, err
@ -633,6 +655,7 @@ func photoFromFields(id, accessHash int64, fileReference []byte, date, dcID int,
DCID: dcID,
HasStickers: hasStickers,
Sizes: sizes,
OwnerUserID: ownerUserID,
}, nil
}
@ -697,7 +720,7 @@ func (s *MediaStore) CreateStickerSet(ctx context.Context, set domain.StickerSet
return err
}
}
return nil
return addStickerSetMediaReferencesTx(ctx, qtx, set.ID, docs)
})
if err != nil {
if stickerSetShortNameConflict(err) {
@ -726,7 +749,13 @@ func (s *MediaStore) UpdateStickerSet(ctx context.Context, set domain.StickerSet
return err
}
}
return nil
// Re-register from scratch: drops references for any document no
// longer in the set (candidate for storage GC once orphaned long
// enough) and refreshes the rest.
if err := removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindStickerSet, stickerSetRefKey(set.ID)); err != nil {
return err
}
return addStickerSetMediaReferencesTx(ctx, qtx, set.ID, docs)
})
if err != nil {
return err
@ -751,8 +780,10 @@ WHERE id = $1
if tag.RowsAffected() == 0 {
return domain.ErrStickerSetInvalid
}
_, err = tx.Exec(ctx, `DELETE FROM user_sticker_sets WHERE sticker_set_id = $1`, setID)
return err
if _, err := tx.Exec(ctx, `DELETE FROM user_sticker_sets WHERE sticker_set_id = $1`, setID); err != nil {
return err
}
return removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindStickerSet, stickerSetRefKey(setID))
})
}
@ -769,11 +800,40 @@ WHERE id = $1
if tag.RowsAffected() == 0 {
return domain.ErrStickerSetInvalid
}
_, err = tx.Exec(ctx, `DELETE FROM user_sticker_sets WHERE sticker_set_id = $1`, setID)
return err
if _, err := tx.Exec(ctx, `DELETE FROM user_sticker_sets WHERE sticker_set_id = $1`, setID); err != nil {
return err
}
return removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindStickerSet, stickerSetRefKey(setID))
})
}
func stickerSetRefKey(setID int64) string {
return fmt.Sprintf("stickerset:%d", setID)
}
// addStickerSetMediaReferencesTx registers every document belonging to a
// sticker set as referenced (storage GC), clearing orphaned_at on each.
func addStickerSetMediaReferencesTx(ctx context.Context, qtx *sqlcgen.Queries, setID int64, docs []domain.Document) error {
refKey := stickerSetRefKey(setID)
for _, doc := range docs {
if doc.ID == 0 {
continue
}
if err := qtx.InsertMediaReference(ctx, sqlcgen.InsertMediaReferenceParams{
MediaKind: string(domain.MediaKindDocument),
MediaID: doc.ID,
RefKind: string(domain.MediaRefKindStickerSet),
RefKey: refKey,
}); err != nil {
return fmt.Errorf("register sticker set media reference: %w", err)
}
if err := qtx.ClearDocumentOrphan(ctx, doc.ID); err != nil {
return fmt.Errorf("clear sticker document orphan: %w", err)
}
}
return nil
}
func insertStickerSet(ctx context.Context, db sqlcgen.DBTX, set domain.StickerSet) error {
thumbs, err := jsonArrayOrEmpty(set.Thumbs)
if err != nil {
@ -1197,15 +1257,29 @@ func (s *MediaStore) AddProfilePhotoKind(ctx context.Context, ownerType domain.P
if err != nil {
return err
}
_, err = s.db.Exec(ctx, `
if _, err := s.db.Exec(ctx, `
INSERT INTO profile_photos (owner_peer_type, owner_peer_id, kind, photo_id, date, active, sort_order)
VALUES ($1, $2, $3, $4, $5, true, $6)
ON CONFLICT (owner_peer_type, owner_peer_id, kind, photo_id) DO UPDATE SET
date = EXCLUDED.date,
active = true,
sort_order = EXCLUDED.sort_order
`, string(ownerType), ownerID, string(kind), photoID, date, next+1)
return err
`, string(ownerType), ownerID, string(kind), photoID, date, next+1); err != nil {
return err
}
if err := s.q.InsertMediaReference(ctx, sqlcgen.InsertMediaReferenceParams{
MediaKind: string(domain.MediaKindPhoto),
MediaID: photoID,
RefKind: string(domain.MediaRefKindProfilePhoto),
RefKey: profilePhotoRefKey(ownerType, ownerID, kind, photoID),
}); err != nil {
return fmt.Errorf("register profile photo reference: %w", err)
}
return s.q.ClearPhotoOrphan(ctx, photoID)
}
func profilePhotoRefKey(ownerType domain.PeerType, ownerID int64, kind domain.ProfilePhotoKind, photoID int64) string {
return fmt.Sprintf("%s:%d:kind:%s:photo:%d", ownerType, ownerID, kind, photoID)
}
func (s *MediaStore) CurrentProfilePhotoKind(ctx context.Context, ownerType domain.PeerType, ownerID int64, kind domain.ProfilePhotoKind) (int64, bool, error) {
@ -1336,7 +1410,7 @@ func (s *MediaStore) ListProfilePhotoDetailsKind(ctx context.Context, ownerType
var err error
if offset < 0 && maxID > 0 {
rows, err = s.db.Query(ctx, `
SELECT ph.id, ph.access_hash, ph.file_reference, ph.date, ph.dc_id, ph.has_stickers, ph.sizes::text AS sizes_json
SELECT ph.id, ph.access_hash, ph.file_reference, ph.date, ph.dc_id, ph.has_stickers, ph.sizes::text AS sizes_json, ph.owner_user_id
FROM profile_photos pp
JOIN photos ph ON ph.id = pp.photo_id
WHERE pp.owner_peer_type = $1
@ -1352,7 +1426,7 @@ LIMIT $5
offset = 0
}
rows, err = s.db.Query(ctx, `
SELECT ph.id, ph.access_hash, ph.file_reference, ph.date, ph.dc_id, ph.has_stickers, ph.sizes::text AS sizes_json
SELECT ph.id, ph.access_hash, ph.file_reference, ph.date, ph.dc_id, ph.has_stickers, ph.sizes::text AS sizes_json, ph.owner_user_id
FROM profile_photos pp
JOIN photos ph ON ph.id = pp.photo_id
WHERE pp.owner_peer_type = $1
@ -1429,6 +1503,19 @@ RETURNING photo_id
if err := rows.Err(); err != nil {
return nil, err
}
for _, id := range deleted {
if err := s.q.RemoveMediaReference(ctx, sqlcgen.RemoveMediaReferenceParams{
MediaKind: string(domain.MediaKindPhoto),
MediaID: id,
RefKind: string(domain.MediaRefKindProfilePhoto),
RefKey: profilePhotoRefKey(ownerType, ownerID, kind, id),
}); err != nil {
return nil, fmt.Errorf("remove profile photo reference: %w", err)
}
if err := s.q.OrphanPhotoIfUnreferenced(ctx, id); err != nil {
return nil, fmt.Errorf("orphan check profile photo: %w", err)
}
}
return deleted, nil
}

View file

@ -13,6 +13,8 @@ import (
// 写路径只在「创建」和「编辑改媒体」两处维护,删除靠读查询 JOIN 过滤 deleted、不在此维护。
// insertChannelMediaIndexTx 为一条频道消息按其媒体类别写索引行(无类别则 no-op)。
// 同时登记该消息内嵌 document/photo 的 media_references(存储回收用),与分类
// 索引共用同一事务。
func insertChannelMediaIndexTx(ctx context.Context, tx pgx.Tx, channelID int64, id, date int, media *domain.MessageMedia, entities []domain.MessageEntity) error {
for _, c := range domain.ClassifyMediaCategories(media, entities) {
if _, err := tx.Exec(ctx, `
@ -22,15 +24,18 @@ ON CONFLICT (channel_id, id, category) DO NOTHING`, channelID, id, int16(c), dat
return fmt.Errorf("insert channel media index: %w", err)
}
}
return nil
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindChannelMessage, channelMessageRefKey(channelID, id))
}
// deleteChannelMediaIndexTx 清掉一条频道消息的全部索引行(编辑改媒体前先清后插)。
// 只在 replaceChannelMediaIndexTx 内被调用;真正的消息删除不清 *_media 分类索引
// (读时靠 JOIN deleted 过滤,见文件头注释),但仍需在此清掉 media_references,
// 否则 replace 场景下旧媒体永远不会被判定为孤儿。
func deleteChannelMediaIndexTx(ctx context.Context, tx pgx.Tx, channelID int64, id int) error {
if _, err := tx.Exec(ctx, `DELETE FROM channel_message_media WHERE channel_id = $1 AND id = $2`, channelID, id); err != nil {
return fmt.Errorf("delete channel media index: %w", err)
}
return nil
return removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindChannelMessage, channelMessageRefKey(channelID, id))
}
// replaceChannelMediaIndexTx 在编辑替换媒体后重建索引行(类别可能变化)。
@ -41,7 +46,8 @@ func replaceChannelMediaIndexTx(ctx context.Context, tx pgx.Tx, channelID int64,
return insertChannelMediaIndexTx(ctx, tx, channelID, id, date, media, entities)
}
// insertMessageBoxMediaIndexTx 为一条私聊 owner box 按其媒体类别写索引行。
// insertMessageBoxMediaIndexTx 为一条私聊 owner box 按其媒体类别写索引行。同时登记
// media_references(存储回收用)。
func insertMessageBoxMediaIndexTx(ctx context.Context, tx pgx.Tx, ownerUserID, peerID int64, boxID, date int, media *domain.MessageMedia, entities []domain.MessageEntity) error {
for _, c := range domain.ClassifyMediaCategories(media, entities) {
if _, err := tx.Exec(ctx, `
@ -51,15 +57,18 @@ ON CONFLICT (owner_user_id, box_id, category) DO NOTHING`, ownerUserID, boxID, p
return fmt.Errorf("insert message box media index: %w", err)
}
}
return nil
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindMessageBox, messageBoxRefKey(ownerUserID, boxID))
}
// deleteMessageBoxMediaIndexTx 清掉一条私聊 owner box 的全部索引行。
// deleteMessageBoxMediaIndexTx 清掉一条私聊 owner box 的全部索引行。只在
// replaceMessageBoxMediaIndexTx 内被调用(编辑改媒体前先清后插);真正的消息
// 删除不清 message_box_media(读时靠 JOIN deleted 过滤),但仍需在此清掉
// media_references,否则 replace 场景下旧媒体永远不会被判定为孤儿。
func deleteMessageBoxMediaIndexTx(ctx context.Context, tx pgx.Tx, ownerUserID int64, boxID int) error {
if _, err := tx.Exec(ctx, `DELETE FROM message_box_media WHERE owner_user_id = $1 AND box_id = $2`, ownerUserID, boxID); err != nil {
return fmt.Errorf("delete message box media index: %w", err)
}
return nil
return removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindMessageBox, messageBoxRefKey(ownerUserID, boxID))
}
// replaceMessageBoxMediaIndexTx 在编辑替换媒体后重建索引行。

View file

@ -0,0 +1,212 @@
package postgres
import (
"context"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"telesrv/internal/domain"
"telesrv/internal/store/postgres/sqlcgen"
)
// messageBoxRefKey/channelMessageRefKey are the ref_key encodings used by
// media_references rows registered from the private-mailbox and channel
// message write paths, respectively. Kept as named helpers so the add and
// remove sides can never drift apart on format.
func messageBoxRefKey(ownerUserID int64, boxID int) string {
return fmt.Sprintf("user:%d:box:%d", ownerUserID, boxID)
}
func channelMessageRefKey(channelID int64, messageID int) string {
return fmt.Sprintf("channel:%d:msg:%d", channelID, messageID)
}
// addMediaReferencesTx registers every document/photo embedded in media as
// referenced by refKind/refKey, clearing orphaned_at on each if it had been
// set by an earlier removal. Must run in the same transaction as the write
// that creates the reference (message send/edit).
func addMediaReferencesTx(ctx context.Context, tx sqlcgen.DBTX, media *domain.MessageMedia, refKind domain.MediaRefKind, refKey string) error {
targets := domain.ExtractMediaRefTargets(media)
if len(targets) == 0 {
return nil
}
q := sqlcgen.New(tx)
for _, t := range targets {
if err := q.InsertMediaReference(ctx, sqlcgen.InsertMediaReferenceParams{
MediaKind: string(t.Kind),
MediaID: t.ID,
RefKind: string(refKind),
RefKey: refKey,
}); err != nil {
return fmt.Errorf("insert media reference: %w", err)
}
var clearErr error
switch t.Kind {
case domain.MediaKindDocument:
clearErr = q.ClearDocumentOrphan(ctx, t.ID)
case domain.MediaKindPhoto:
clearErr = q.ClearPhotoOrphan(ctx, t.ID)
}
if clearErr != nil {
return fmt.Errorf("clear media orphan: %w", clearErr)
}
}
return nil
}
// removeMediaReferencesByKeyTx drops every media_references row registered
// under refKind/refKey (no need to know which document/photo ids those were
// -- the delete finds them) and, for each one that becomes fully
// unreferenced as a result, marks it orphaned so the storage retention
// sweep can consider it once old enough. Must run in the same transaction
// as the write that removes the reference (a message being soft-deleted).
func removeMediaReferencesByKeyTx(ctx context.Context, tx sqlcgen.DBTX, refKind domain.MediaRefKind, refKey string) error {
rows, err := tx.Query(ctx, `
DELETE FROM media_references
WHERE ref_kind = $1 AND ref_key = $2
RETURNING media_kind, media_id`, string(refKind), refKey)
if err != nil {
return fmt.Errorf("remove media references: %w", err)
}
type removedRef struct {
kind string
id int64
}
var removed []removedRef
for rows.Next() {
var r removedRef
if err := rows.Scan(&r.kind, &r.id); err != nil {
rows.Close()
return fmt.Errorf("scan removed media reference: %w", err)
}
removed = append(removed, r)
}
if err := rows.Err(); err != nil {
return fmt.Errorf("remove media references: %w", err)
}
rows.Close()
q := sqlcgen.New(tx)
for _, r := range removed {
var orphanErr error
switch domain.MediaKind(r.kind) {
case domain.MediaKindDocument:
orphanErr = q.OrphanDocumentIfUnreferenced(ctx, r.id)
case domain.MediaKindPhoto:
orphanErr = q.OrphanPhotoIfUnreferenced(ctx, r.id)
}
if orphanErr != nil {
return fmt.Errorf("orphan check media: %w", orphanErr)
}
}
return nil
}
// ---- storage retention sweep ----
// ListOrphanedDocumentIDsOlderThan returns document ids whose orphaned_at is
// set and older than cutoff, oldest first, up to limit.
func (s *MediaStore) ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListOrphanedDocumentIDsOlderThan(ctx, sqlcgen.ListOrphanedDocumentIDsOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
BatchLimit: int32(limit),
})
}
// ListOrphanedPhotoIDsOlderThan returns photo ids whose orphaned_at is set
// and older than cutoff, oldest first, up to limit.
func (s *MediaStore) ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListOrphanedPhotoIDsOlderThan(ctx, sqlcgen.ListOrphanedPhotoIDsOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
BatchLimit: int32(limit),
})
}
// CountFileBlobRefs reports how many file_blobs rows still point at
// (backend, objectKey) -- the caller must not physically delete the object
// from that backend while this is > 0 (content-addressed storage: the same
// object can be shared by multiple documents/photos).
func (s *MediaStore) CountFileBlobRefs(ctx context.Context, backend, objectKey string) (int, error) {
n, err := s.q.CountFileBlobRefs(ctx, sqlcgen.CountFileBlobRefsParams{Backend: backend, ObjectKey: objectKey})
return int(n), err
}
// DeleteDocumentAndBlobs deletes a document row and every file_blobs row it
// owns (main body + thumbnail variants), returning what was deleted so the
// caller can physically remove each object from its backend once confirming
// (via CountFileBlobRefs, after this call) no other row still needs it.
// Assumes the document is already orphaned -- does not check references.
func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error) {
var blobs []domain.FileBlob
err := withTx(ctx, s.db, "delete document and blobs", func(tx pgx.Tx) error {
qtx := s.q.WithTx(tx)
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
ExactKey: fmt.Sprintf("doc:%d", id),
PrefixPattern: fmt.Sprintf("doc:%d:%%", id),
})
if err != nil {
return fmt.Errorf("list document blobs: %w", err)
}
for _, r := range rows {
blobs = append(blobs, domain.FileBlob{
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
})
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
return fmt.Errorf("delete file blob row: %w", err)
}
}
if err := qtx.DeleteDocumentRow(ctx, id); err != nil {
return fmt.Errorf("delete document row: %w", err)
}
return nil
})
if err != nil {
return nil, err
}
s.documents.remove(id)
return blobs, nil
}
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
// (one per rendition size), returning what was deleted so the caller can
// physically remove each object from its backend once confirming (via
// CountFileBlobRefs, after this call) no other row still needs it. Assumes
// the photo is already orphaned -- does not check references.
func (s *MediaStore) DeletePhotoAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error) {
var blobs []domain.FileBlob
err := withTx(ctx, s.db, "delete photo and blobs", func(tx pgx.Tx) error {
qtx := s.q.WithTx(tx)
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
ExactKey: fmt.Sprintf("photo:%d", id),
PrefixPattern: fmt.Sprintf("photo:%d:%%", id),
})
if err != nil {
return fmt.Errorf("list photo blobs: %w", err)
}
for _, r := range rows {
blobs = append(blobs, domain.FileBlob{
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
})
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
return fmt.Errorf("delete file blob row: %w", err)
}
}
if err := qtx.DeletePhotoRow(ctx, id); err != nil {
return fmt.Errorf("delete photo row: %w", err)
}
return nil
})
if err != nil {
return nil, err
}
return blobs, nil
}

View file

@ -0,0 +1,109 @@
package postgres
import (
"context"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
"telesrv/internal/domain"
)
// TestMediaReferenceOrphanTransitions proves the core storage-retention
// safety invariant: a document's orphaned_at is set only once every
// reference to it is gone, and cleared the instant a new one appears --
// so the retention sweep never targets media still visible in a
// conversation, regardless of how many places reference it.
func TestMediaReferenceOrphanTransitions(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
s := NewMediaStore(pool)
const docID = int64(9100000000000000101)
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), `DELETE FROM media_references WHERE media_kind = 'document' AND media_id = $1`, docID)
_, _ = pool.Exec(context.Background(), `DELETE FROM documents WHERE id = $1`, docID)
})
if err := s.PutDocument(ctx, domain.Document{ID: docID, MimeType: "text/plain", Size: 10}); err != nil {
t.Fatalf("put document: %v", err)
}
media := &domain.MessageMedia{Kind: domain.MessageMediaKindDocument, Document: &domain.Document{ID: docID}}
// A freshly created document has no orphaned_at yet either way -- it's
// simply unreferenced until a message send registers the first
// reference, at which point normal tracking takes over.
orphaned, err := documentOrphanedAt(ctx, pool, docID)
if err != nil {
t.Fatalf("query orphaned_at: %v", err)
}
if orphaned {
t.Fatal("expected a freshly inserted document to not be marked orphaned yet")
}
// Adding a reference (as if a message carrying it was sent) clears it.
mustAddRef(t, pool, media, domain.MediaRefKindMessageBox, "user:1:box:1")
if orphaned, err := documentOrphanedAt(ctx, pool, docID); err != nil || orphaned {
t.Fatalf("expected referenced document to not be orphaned, orphaned=%v err=%v", orphaned, err)
}
// A second, independent reference (e.g. forwarded to another box).
mustAddRef(t, pool, media, domain.MediaRefKindMessageBox, "user:2:box:5")
// Removing only one of the two references must NOT orphan the document.
mustRemoveRefsByKey(t, pool, domain.MediaRefKindMessageBox, "user:1:box:1")
if orphaned, err := documentOrphanedAt(ctx, pool, docID); err != nil || orphaned {
t.Fatalf("expected document with a remaining reference to survive, orphaned=%v err=%v", orphaned, err)
}
// Removing the last reference orphans it.
mustRemoveRefsByKey(t, pool, domain.MediaRefKindMessageBox, "user:2:box:5")
if orphaned, err := documentOrphanedAt(ctx, pool, docID); err != nil || !orphaned {
t.Fatalf("expected document with no remaining reference to be orphaned, orphaned=%v err=%v", orphaned, err)
}
// A reference reappearing after orphaning (e.g. re-sent) clears it again.
mustAddRef(t, pool, media, domain.MediaRefKindMessageBox, "user:3:box:9")
if orphaned, err := documentOrphanedAt(ctx, pool, docID); err != nil || orphaned {
t.Fatalf("expected re-referenced document to no longer be orphaned, orphaned=%v err=%v", orphaned, err)
}
}
func mustAddRef(t *testing.T, pool *pgxpool.Pool, media *domain.MessageMedia, refKind domain.MediaRefKind, refKey string) {
t.Helper()
ctx := context.Background()
tx, err := pool.Begin(ctx)
if err != nil {
t.Fatalf("begin tx: %v", err)
}
defer func() { _ = tx.Rollback(ctx) }()
if err := addMediaReferencesTx(ctx, tx, media, refKind, refKey); err != nil {
t.Fatalf("add media reference: %v", err)
}
if err := tx.Commit(ctx); err != nil {
t.Fatalf("commit: %v", err)
}
}
func mustRemoveRefsByKey(t *testing.T, pool *pgxpool.Pool, refKind domain.MediaRefKind, refKey string) {
t.Helper()
ctx := context.Background()
tx, err := pool.Begin(ctx)
if err != nil {
t.Fatalf("begin tx: %v", err)
}
defer func() { _ = tx.Rollback(ctx) }()
if err := removeMediaReferencesByKeyTx(ctx, tx, refKind, refKey); err != nil {
t.Fatalf("remove media references: %v", err)
}
if err := tx.Commit(ctx); err != nil {
t.Fatalf("commit: %v", err)
}
}
func documentOrphanedAt(ctx context.Context, pool *pgxpool.Pool, id int64) (bool, error) {
var orphaned bool
err := pool.QueryRow(ctx, `SELECT orphaned_at IS NOT NULL FROM documents WHERE id = $1`, id).Scan(&orphaned)
return orphaned, err
}

View file

@ -174,6 +174,11 @@ func (s *MessageStore) finishDeleteMessagesTx(ctx context.Context, db sqlcgen.DB
if row.ownerUserID == 0 || row.boxID == 0 {
continue
}
// Drop this box's media_references (storage GC); orphans the
// document/photo if this was its last live reference anywhere.
if err := removeMediaReferencesByKeyTx(ctx, db, domain.MediaRefKindMessageBox, messageBoxRefKey(row.ownerUserID, row.boxID)); err != nil {
return res, fmt.Errorf("remove deleted message media references: %w", err)
}
idsByOwner[row.ownerUserID] = append(idsByOwner[row.ownerUserID], row.boxID)
if row.peer.ID != 0 {
if peersByOwner[row.ownerUserID] == nil {

View file

@ -108,7 +108,7 @@ WHERE location_key = sqlc.arg(location_key)::text;
-- documents -------------------------------------------------------------------
-- name: PutDocument :exec
INSERT INTO documents (id, access_hash, file_reference, date, mime_type, size, dc_id, attributes, thumbs)
INSERT INTO documents (id, access_hash, file_reference, date, mime_type, size, dc_id, attributes, thumbs, owner_user_id)
VALUES (
sqlc.arg(id)::bigint,
sqlc.arg(access_hash)::bigint,
@ -118,8 +118,13 @@ VALUES (
sqlc.arg(size)::bigint,
sqlc.arg(dc_id)::int,
sqlc.arg(attributes_json)::jsonb,
sqlc.arg(thumbs_json)::jsonb
sqlc.arg(thumbs_json)::jsonb,
sqlc.arg(owner_user_id)::bigint
)
-- owner_user_id is intentionally NOT in the UPDATE SET list: a document id is
-- only ever (re-)upserted by its original uploader's own request replay, and
-- keeping the first-write owner sticky avoids any risk of a later call
-- (e.g. a forward re-touching the row) reassigning ownership.
ON CONFLICT (id) DO UPDATE SET
access_hash = EXCLUDED.access_hash,
file_reference = EXCLUDED.file_reference,
@ -133,21 +138,26 @@ ON CONFLICT (id) DO UPDATE SET
-- name: GetDocument :one
SELECT id, access_hash, file_reference, date, mime_type, size, dc_id,
attributes::text AS attributes_json,
thumbs::text AS thumbs_json
thumbs::text AS thumbs_json,
owner_user_id
FROM documents
WHERE id = sqlc.arg(id)::bigint;
-- name: GetDocuments :many
SELECT id, access_hash, file_reference, date, mime_type, size, dc_id,
attributes::text AS attributes_json,
thumbs::text AS thumbs_json
thumbs::text AS thumbs_json,
owner_user_id
FROM documents
WHERE id = ANY(sqlc.arg(ids)::bigint[]);
-- name: DeleteDocumentRow :exec
DELETE FROM documents WHERE id = sqlc.arg(id)::bigint;
-- photos ----------------------------------------------------------------------
-- name: PutPhoto :exec
INSERT INTO photos (id, access_hash, file_reference, date, dc_id, has_stickers, sizes)
INSERT INTO photos (id, access_hash, file_reference, date, dc_id, has_stickers, sizes, owner_user_id)
VALUES (
sqlc.arg(id)::bigint,
sqlc.arg(access_hash)::bigint,
@ -155,8 +165,10 @@ VALUES (
sqlc.arg(date)::int,
sqlc.arg(dc_id)::int,
sqlc.arg(has_stickers)::boolean,
sqlc.arg(sizes_json)::jsonb
sqlc.arg(sizes_json)::jsonb,
sqlc.arg(owner_user_id)::bigint
)
-- owner_user_id intentionally not updated on conflict, see PutDocument.
ON CONFLICT (id) DO UPDATE SET
access_hash = EXCLUDED.access_hash,
file_reference = EXCLUDED.file_reference,
@ -167,10 +179,87 @@ ON CONFLICT (id) DO UPDATE SET
-- name: GetPhoto :one
SELECT id, access_hash, file_reference, date, dc_id, has_stickers,
sizes::text AS sizes_json
sizes::text AS sizes_json,
owner_user_id
FROM photos
WHERE id = sqlc.arg(id)::bigint;
-- name: DeletePhotoRow :exec
DELETE FROM photos WHERE id = sqlc.arg(id)::bigint;
-- media_references / storage retention -----------------------------------------
-- name: InsertMediaReference :exec
INSERT INTO media_references (media_kind, media_id, ref_kind, ref_key)
VALUES (sqlc.arg(media_kind)::text, sqlc.arg(media_id)::bigint, sqlc.arg(ref_kind)::text, sqlc.arg(ref_key)::text)
ON CONFLICT DO NOTHING;
-- name: ClearDocumentOrphan :exec
UPDATE documents SET orphaned_at = NULL
WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
-- name: ClearPhotoOrphan :exec
UPDATE photos SET orphaned_at = NULL
WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
-- name: RemoveMediaReference :exec
DELETE FROM media_references
WHERE media_kind = sqlc.arg(media_kind)::text
AND media_id = sqlc.arg(media_id)::bigint
AND ref_kind = sqlc.arg(ref_kind)::text
AND ref_key = sqlc.arg(ref_key)::text;
-- name: OrphanDocumentIfUnreferenced :exec
UPDATE documents SET orphaned_at = now()
WHERE id = sqlc.arg(media_id)::bigint
AND orphaned_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM media_references WHERE media_kind = 'document' AND media_id = sqlc.arg(media_id)::bigint
);
-- name: OrphanPhotoIfUnreferenced :exec
UPDATE photos SET orphaned_at = now()
WHERE id = sqlc.arg(media_id)::bigint
AND orphaned_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM media_references WHERE media_kind = 'photo' AND media_id = sqlc.arg(media_id)::bigint
);
-- name: ListOrphanedDocumentIDsOlderThan :many
SELECT id FROM documents
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
ORDER BY orphaned_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: ListOrphanedPhotoIDsOlderThan :many
SELECT id FROM photos
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
ORDER BY orphaned_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: CountFileBlobRefs :one
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;
-- name: DeleteFileBlobRow :exec
DELETE FROM file_blobs WHERE location_key = sqlc.arg(location_key)::text;
-- name: ListFileBlobsByLocationPrefix :many
-- Matches a media's main blob (exact_key, e.g. "doc:123") plus every
-- variant keyed off it (prefix_pattern, e.g. "doc:123:%" for thumbnails /
-- "photo:456:%" for each rendition size) -- a document/photo can own
-- multiple file_blobs rows.
SELECT location_key, backend, object_key, size
FROM file_blobs
WHERE location_key = sqlc.arg(exact_key)::text
OR location_key LIKE sqlc.arg(prefix_pattern)::text;
-- name: SumFileBlobBytes :one
-- Physical bytes actually held by the blob backend (dedup-aware: identical
-- content uploaded by different users is one row here). Used by the
-- low-space guard's cached usage gauge and the admin panel's "physical
-- usage" stat.
SELECT COALESCE(SUM(size), 0)::bigint FROM file_blobs;
-- sticker_sets ----------------------------------------------------------------
-- name: PutStickerSet :exec

View file

@ -48,6 +48,26 @@ func (q *Queries) AddProfilePhoto(ctx context.Context, arg AddProfilePhotoParams
return err
}
const clearDocumentOrphan = `-- name: ClearDocumentOrphan :exec
UPDATE documents SET orphaned_at = NULL
WHERE id = $1::bigint AND orphaned_at IS NOT NULL
`
func (q *Queries) ClearDocumentOrphan(ctx context.Context, mediaID int64) error {
_, err := q.db.Exec(ctx, clearDocumentOrphan, mediaID)
return err
}
const clearPhotoOrphan = `-- name: ClearPhotoOrphan :exec
UPDATE photos SET orphaned_at = NULL
WHERE id = $1::bigint AND orphaned_at IS NOT NULL
`
func (q *Queries) ClearPhotoOrphan(ctx context.Context, mediaID int64) error {
_, err := q.db.Exec(ctx, clearPhotoOrphan, mediaID)
return err
}
const countAvailableReactions = `-- name: CountAvailableReactions :one
SELECT count(*)::int AS total FROM available_reactions
`
@ -59,6 +79,22 @@ func (q *Queries) CountAvailableReactions(ctx context.Context) (int32, error) {
return total, err
}
const countFileBlobRefs = `-- name: CountFileBlobRefs :one
SELECT COUNT(*)::int FROM file_blobs WHERE backend = $1::text AND object_key = $2::text
`
type CountFileBlobRefsParams struct {
Backend string
ObjectKey string
}
func (q *Queries) CountFileBlobRefs(ctx context.Context, arg CountFileBlobRefsParams) (int32, error) {
row := q.db.QueryRow(ctx, countFileBlobRefs, arg.Backend, arg.ObjectKey)
var column_1 int32
err := row.Scan(&column_1)
return column_1, err
}
const countProfilePhotos = `-- name: CountProfilePhotos :one
SELECT count(*)::int AS total
FROM profile_photos
@ -199,6 +235,15 @@ func (q *Queries) DeactivateProfilePhotos(ctx context.Context, arg DeactivatePro
return items, nil
}
const deleteDocumentRow = `-- name: DeleteDocumentRow :exec
DELETE FROM documents WHERE id = $1::bigint
`
func (q *Queries) DeleteDocumentRow(ctx context.Context, id int64) error {
_, err := q.db.Exec(ctx, deleteDocumentRow, id)
return err
}
const deleteExpiredUploadParts = `-- name: DeleteExpiredUploadParts :many
WITH doomed AS (
SELECT owner_user_id, file_id, part
@ -240,6 +285,24 @@ func (q *Queries) DeleteExpiredUploadParts(ctx context.Context, arg DeleteExpire
return items, nil
}
const deleteFileBlobRow = `-- name: DeleteFileBlobRow :exec
DELETE FROM file_blobs WHERE location_key = $1::text
`
func (q *Queries) DeleteFileBlobRow(ctx context.Context, locationKey string) error {
_, err := q.db.Exec(ctx, deleteFileBlobRow, locationKey)
return err
}
const deletePhotoRow = `-- name: DeletePhotoRow :exec
DELETE FROM photos WHERE id = $1::bigint
`
func (q *Queries) DeletePhotoRow(ctx context.Context, id int64) error {
_, err := q.db.Exec(ctx, deletePhotoRow, id)
return err
}
const deleteUploadParts = `-- name: DeleteUploadParts :many
DELETE FROM upload_parts
WHERE owner_user_id = $1::bigint
@ -275,7 +338,8 @@ func (q *Queries) DeleteUploadParts(ctx context.Context, arg DeleteUploadPartsPa
const getDocument = `-- name: GetDocument :one
SELECT id, access_hash, file_reference, date, mime_type, size, dc_id,
attributes::text AS attributes_json,
thumbs::text AS thumbs_json
thumbs::text AS thumbs_json,
owner_user_id
FROM documents
WHERE id = $1::bigint
`
@ -290,6 +354,7 @@ type GetDocumentRow struct {
DcID int32
AttributesJson string
ThumbsJson string
OwnerUserID int64
}
func (q *Queries) GetDocument(ctx context.Context, id int64) (GetDocumentRow, error) {
@ -305,6 +370,7 @@ func (q *Queries) GetDocument(ctx context.Context, id int64) (GetDocumentRow, er
&i.DcID,
&i.AttributesJson,
&i.ThumbsJson,
&i.OwnerUserID,
)
return i, err
}
@ -312,7 +378,8 @@ func (q *Queries) GetDocument(ctx context.Context, id int64) (GetDocumentRow, er
const getDocuments = `-- name: GetDocuments :many
SELECT id, access_hash, file_reference, date, mime_type, size, dc_id,
attributes::text AS attributes_json,
thumbs::text AS thumbs_json
thumbs::text AS thumbs_json,
owner_user_id
FROM documents
WHERE id = ANY($1::bigint[])
`
@ -327,6 +394,7 @@ type GetDocumentsRow struct {
DcID int32
AttributesJson string
ThumbsJson string
OwnerUserID int64
}
func (q *Queries) GetDocuments(ctx context.Context, ids []int64) ([]GetDocumentsRow, error) {
@ -348,6 +416,7 @@ func (q *Queries) GetDocuments(ctx context.Context, ids []int64) ([]GetDocuments
&i.DcID,
&i.AttributesJson,
&i.ThumbsJson,
&i.OwnerUserID,
); err != nil {
return nil, err
}
@ -390,7 +459,8 @@ func (q *Queries) GetFileBlob(ctx context.Context, locationKey string) (GetFileB
const getPhoto = `-- name: GetPhoto :one
SELECT id, access_hash, file_reference, date, dc_id, has_stickers,
sizes::text AS sizes_json
sizes::text AS sizes_json,
owner_user_id
FROM photos
WHERE id = $1::bigint
`
@ -403,6 +473,7 @@ type GetPhotoRow struct {
DcID int32
HasStickers bool
SizesJson string
OwnerUserID int64
}
func (q *Queries) GetPhoto(ctx context.Context, id int64) (GetPhotoRow, error) {
@ -416,6 +487,7 @@ func (q *Queries) GetPhoto(ctx context.Context, id int64) (GetPhotoRow, error) {
&i.DcID,
&i.HasStickers,
&i.SizesJson,
&i.OwnerUserID,
)
return i, err
}
@ -686,6 +758,31 @@ func (q *Queries) GetUploadPartUsage(ctx context.Context, ownerUserID int64) (Ge
return i, err
}
const insertMediaReference = `-- name: InsertMediaReference :exec
INSERT INTO media_references (media_kind, media_id, ref_kind, ref_key)
VALUES ($1::text, $2::bigint, $3::text, $4::text)
ON CONFLICT DO NOTHING
`
type InsertMediaReferenceParams struct {
MediaKind string
MediaID int64
RefKind string
RefKey string
}
// media_references / storage retention -----------------------------------------
func (q *Queries) InsertMediaReference(ctx context.Context, arg InsertMediaReferenceParams) error {
_, err := q.db.Exec(ctx, insertMediaReference,
arg.MediaKind,
arg.MediaID,
arg.RefKind,
arg.RefKey,
)
return err
}
const listAvailableReactions = `-- name: ListAvailableReactions :many
SELECT
reaction, title, inactive, premium,
@ -728,6 +825,118 @@ func (q *Queries) ListAvailableReactions(ctx context.Context) ([]AvailableReacti
return items, nil
}
const listFileBlobsByLocationPrefix = `-- name: ListFileBlobsByLocationPrefix :many
SELECT location_key, backend, object_key, size
FROM file_blobs
WHERE location_key = $1::text
OR location_key LIKE $2::text
`
type ListFileBlobsByLocationPrefixParams struct {
ExactKey string
PrefixPattern string
}
type ListFileBlobsByLocationPrefixRow struct {
LocationKey string
Backend string
ObjectKey string
Size int64
}
// Matches a media's main blob (exact_key, e.g. "doc:123") plus every
// variant keyed off it (prefix_pattern, e.g. "doc:123:%" for thumbnails /
// "photo:456:%" for each rendition size) -- a document/photo can own
// multiple file_blobs rows.
func (q *Queries) ListFileBlobsByLocationPrefix(ctx context.Context, arg ListFileBlobsByLocationPrefixParams) ([]ListFileBlobsByLocationPrefixRow, error) {
rows, err := q.db.Query(ctx, listFileBlobsByLocationPrefix, arg.ExactKey, arg.PrefixPattern)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListFileBlobsByLocationPrefixRow
for rows.Next() {
var i ListFileBlobsByLocationPrefixRow
if err := rows.Scan(
&i.LocationKey,
&i.Backend,
&i.ObjectKey,
&i.Size,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listOrphanedDocumentIDsOlderThan = `-- name: ListOrphanedDocumentIDsOlderThan :many
SELECT id FROM documents
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
ORDER BY orphaned_at ASC
LIMIT $2::int
`
type ListOrphanedDocumentIDsOlderThanParams struct {
Cutoff pgtype.Timestamptz
BatchLimit int32
}
func (q *Queries) ListOrphanedDocumentIDsOlderThan(ctx context.Context, arg ListOrphanedDocumentIDsOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listOrphanedDocumentIDsOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
}
items = append(items, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listOrphanedPhotoIDsOlderThan = `-- name: ListOrphanedPhotoIDsOlderThan :many
SELECT id FROM photos
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
ORDER BY orphaned_at ASC
LIMIT $2::int
`
type ListOrphanedPhotoIDsOlderThanParams struct {
Cutoff pgtype.Timestamptz
BatchLimit int32
}
func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrphanedPhotoIDsOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listOrphanedPhotoIDsOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
}
items = append(items, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listProfilePhotos = `-- name: ListProfilePhotos :many
SELECT photo_id
FROM profile_photos
@ -925,6 +1134,34 @@ func (q *Queries) NextProfilePhotoOrder(ctx context.Context, arg NextProfilePhot
return max_order, err
}
const orphanDocumentIfUnreferenced = `-- name: OrphanDocumentIfUnreferenced :exec
UPDATE documents SET orphaned_at = now()
WHERE id = $1::bigint
AND orphaned_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM media_references WHERE media_kind = 'document' AND media_id = $1::bigint
)
`
func (q *Queries) OrphanDocumentIfUnreferenced(ctx context.Context, mediaID int64) error {
_, err := q.db.Exec(ctx, orphanDocumentIfUnreferenced, mediaID)
return err
}
const orphanPhotoIfUnreferenced = `-- name: OrphanPhotoIfUnreferenced :exec
UPDATE photos SET orphaned_at = now()
WHERE id = $1::bigint
AND orphaned_at IS NULL
AND NOT EXISTS (
SELECT 1 FROM media_references WHERE media_kind = 'photo' AND media_id = $1::bigint
)
`
func (q *Queries) OrphanPhotoIfUnreferenced(ctx context.Context, mediaID int64) error {
_, err := q.db.Exec(ctx, orphanPhotoIfUnreferenced, mediaID)
return err
}
const putAvailableReaction = `-- name: PutAvailableReaction :exec
INSERT INTO available_reactions (
@ -995,7 +1232,7 @@ func (q *Queries) PutAvailableReaction(ctx context.Context, arg PutAvailableReac
const putDocument = `-- name: PutDocument :exec
INSERT INTO documents (id, access_hash, file_reference, date, mime_type, size, dc_id, attributes, thumbs)
INSERT INTO documents (id, access_hash, file_reference, date, mime_type, size, dc_id, attributes, thumbs, owner_user_id)
VALUES (
$1::bigint,
$2::bigint,
@ -1005,7 +1242,8 @@ VALUES (
$6::bigint,
$7::int,
$8::jsonb,
$9::jsonb
$9::jsonb,
$10::bigint
)
ON CONFLICT (id) DO UPDATE SET
access_hash = EXCLUDED.access_hash,
@ -1028,9 +1266,14 @@ type PutDocumentParams struct {
DcID int32
AttributesJson []byte
ThumbsJson []byte
OwnerUserID int64
}
// documents -------------------------------------------------------------------
// owner_user_id is intentionally NOT in the UPDATE SET list: a document id is
// only ever (re-)upserted by its original uploader's own request replay, and
// keeping the first-write owner sticky avoids any risk of a later call
// (e.g. a forward re-touching the row) reassigning ownership.
func (q *Queries) PutDocument(ctx context.Context, arg PutDocumentParams) error {
_, err := q.db.Exec(ctx, putDocument,
arg.ID,
@ -1042,6 +1285,7 @@ func (q *Queries) PutDocument(ctx context.Context, arg PutDocumentParams) error
arg.DcID,
arg.AttributesJson,
arg.ThumbsJson,
arg.OwnerUserID,
)
return err
}
@ -1089,7 +1333,7 @@ func (q *Queries) PutFileBlob(ctx context.Context, arg PutFileBlobParams) error
const putPhoto = `-- name: PutPhoto :exec
INSERT INTO photos (id, access_hash, file_reference, date, dc_id, has_stickers, sizes)
INSERT INTO photos (id, access_hash, file_reference, date, dc_id, has_stickers, sizes, owner_user_id)
VALUES (
$1::bigint,
$2::bigint,
@ -1097,7 +1341,8 @@ VALUES (
$4::int,
$5::int,
$6::boolean,
$7::jsonb
$7::jsonb,
$8::bigint
)
ON CONFLICT (id) DO UPDATE SET
access_hash = EXCLUDED.access_hash,
@ -1116,9 +1361,11 @@ type PutPhotoParams struct {
DcID int32
HasStickers bool
SizesJson []byte
OwnerUserID int64
}
// photos ----------------------------------------------------------------------
// owner_user_id intentionally not updated on conflict, see PutDocument.
func (q *Queries) PutPhoto(ctx context.Context, arg PutPhotoParams) error {
_, err := q.db.Exec(ctx, putPhoto,
arg.ID,
@ -1128,6 +1375,7 @@ func (q *Queries) PutPhoto(ctx context.Context, arg PutPhotoParams) error {
arg.DcID,
arg.HasStickers,
arg.SizesJson,
arg.OwnerUserID,
)
return err
}
@ -1244,6 +1492,31 @@ func (q *Queries) PutStickerSet(ctx context.Context, arg PutStickerSetParams) er
return err
}
const removeMediaReference = `-- name: RemoveMediaReference :exec
DELETE FROM media_references
WHERE media_kind = $1::text
AND media_id = $2::bigint
AND ref_kind = $3::text
AND ref_key = $4::text
`
type RemoveMediaReferenceParams struct {
MediaKind string
MediaID int64
RefKind string
RefKey string
}
func (q *Queries) RemoveMediaReference(ctx context.Context, arg RemoveMediaReferenceParams) error {
_, err := q.db.Exec(ctx, removeMediaReference,
arg.MediaKind,
arg.MediaID,
arg.RefKind,
arg.RefKey,
)
return err
}
const saveUploadPart = `-- name: SaveUploadPart :exec
INSERT INTO upload_parts (owner_user_id, file_id, part, total_parts, is_big, backend, object_key, size, sha256)
@ -1295,3 +1568,18 @@ func (q *Queries) SaveUploadPart(ctx context.Context, arg SaveUploadPartParams)
)
return err
}
const sumFileBlobBytes = `-- name: SumFileBlobBytes :one
SELECT COALESCE(SUM(size), 0)::bigint FROM file_blobs
`
// Physical bytes actually held by the blob backend (dedup-aware: identical
// content uploaded by different users is one row here). Used by the
// low-space guard's cached usage gauge and the admin panel's "physical
// usage" stat.
func (q *Queries) SumFileBlobBytes(ctx context.Context) (int64, error) {
row := q.db.QueryRow(ctx, sumFileBlobBytes)
var column_1 int64
err := row.Scan(&column_1)
return column_1, err
}

View file

@ -79,6 +79,34 @@ type AccountPrivacyRule struct {
UpdatedAt pgtype.Timestamptz
}
type AccountRating struct {
UserID int64
Level int32
Stars int64
CurrentLevelStars int64
NextLevelStars *int64
StarsComponent int64
ActivityComponent int64
PenaltyComponent int64
ManualComponent int64
PendingStars int64
PendingDate pgtype.Timestamptz
ComputedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
Version int64
}
type AccountRatingEvent struct {
ID int64
UserID int64
Kind string
Amount int64
Reason string
Actor string
CommandKey *string
CreatedAt pgtype.Timestamptz
}
type AccountReactionSetting struct {
UserID int64
MessagesNotifyFrom string
@ -218,6 +246,21 @@ type AttachMenuUserState struct {
UpdatedAt pgtype.Timestamptz
}
type AuthDeliveryReport struct {
ID int64
AuthKeyID []byte
SessionID int64
ClientType string
PhoneHash []byte
CodeHash []byte
IssuedUserID int64
DeliveryID string
Channel string
Mnc string
Fingerprint []byte
CreatedAt pgtype.Timestamptz
}
type AuthKey struct {
AuthKeyID int64
Body []byte
@ -448,6 +491,20 @@ type BotUserPermission struct {
UpdatedAt pgtype.Timestamptz
}
type BotVerifierSetting struct {
BotID int64
IconDocumentID int64
CompanyName string
DefaultDescription string
CanModifyCustomDescription bool
Enabled bool
GrantedBy string
GrantReason string
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
Version int64
}
type BusinessAutomationDelivery struct {
OwnerUserID int64
PeerUserID int64
@ -577,6 +634,18 @@ type ChannelAdminLogEvent struct {
CreatedAt pgtype.Timestamptz
}
type ChannelAntispamDecision struct {
ID int64
ChannelID int64
MessageID int32
AuthorUserID int64
EvidenceSchemaVersion int16
Evidence []byte
EvidenceHash []byte
ReportID *int64
CreatedAt pgtype.Timestamptz
}
type ChannelBoostSlot struct {
UserID int64
Slot int32
@ -689,25 +758,27 @@ type ChannelMediaCategoryCount struct {
}
type ChannelMember struct {
ChannelID int64
UserID int64
InviterUserID int64
Role string
Status string
JoinedAt int32
LeftAt int32
AdminRights []byte
BannedRights []byte
Rank string
AvailableMinID int32
AvailableMinPts int32
ReadInboxMaxID int32
ReadInboxDate int32
ReadOutboxMaxID int32
UnreadMark bool
SlowmodeLastSendDate int32
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
ChannelID int64
UserID int64
InviterUserID int64
Role string
Status string
JoinedAt int32
LeftAt int32
AdminRights []byte
BannedRights []byte
Rank string
AvailableMinID int32
AvailableMinPts int32
ReadInboxMaxID int32
ReadInboxDate int32
ReadOutboxMaxID int32
UnreadMark bool
SlowmodeLastSendDate int32
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
HistoryClearAnchorID int32
HistoryClearAnchorDate int32
}
type ChannelMessage struct {
@ -910,6 +981,55 @@ type ChatlistMembership struct {
UpdatedAt pgtype.Timestamptz
}
type ClientTelemetryEvent struct {
ID int64
UserID int64
Kind string
PeerType string
PeerID int64
SubjectIds []int64
Payload []byte
Fingerprint []byte
CreatedAt pgtype.Timestamptz
}
type CollectibleUsername struct {
ID int64
Username string
UsernameLower string
Status string
OwnerPeerType string
OwnerPeerID int64
PurchaseDate pgtype.Timestamptz
Currency string
Amount int64
CryptoCurrency string
CryptoAmount int64
Url string
OriginalOwnerPeerType string
OriginalOwnerPeerID int64
TransferCount int32
Version int64
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
}
type CollectibleUsernameTransfer struct {
ID int64
CollectibleID int64
Kind string
FromPeerType string
FromPeerID int64
ToPeerType string
ToPeerID int64
Currency string
Amount int64
Actor string
Reason string
CommandKey *string
CreatedAt pgtype.Timestamptz
}
type Community struct {
ID int64
AccessHash int64
@ -1008,6 +1128,41 @@ type CountryCode struct {
OrderIndex int32
}
type CustomVerification struct {
ID int64
VerifierBotID int64
PeerType string
PeerID int64
IconDocumentID int64
Description string
GrantedByUserID int64
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
Version int64
}
type CustomVerificationRequest struct {
ID int64
VerifierBotID int64
ApplicantUserID int64
PeerType string
PeerID int64
PeerTitle string
PeerUsername string
Reason string
RequestedDescription string
Status string
DecidedBy string
DecisionReason string
InternalNote string
CorrelationID string
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
ApprovedAt pgtype.Timestamptz
RejectedAt pgtype.Timestamptz
Version int64
}
type Dialog struct {
UserID int64
PeerType string
@ -1095,6 +1250,8 @@ type Document struct {
Attributes []byte
Thumbs []byte
CreatedAt pgtype.Timestamptz
OwnerUserID int64
OrphanedAt pgtype.Timestamptz
}
type EncryptedFile struct {
@ -1286,6 +1443,14 @@ type LoginCodeMessageDelivery struct {
ExpiresAt pgtype.Timestamptz
}
type MediaReference struct {
MediaKind string
MediaID int64
RefKind string
RefKey string
CreatedAt pgtype.Timestamptz
}
type MessageBox struct {
OwnerUserID int64
BoxID int32
@ -1343,6 +1508,126 @@ type MessageBoxMedium struct {
MessageDate int32
}
type ModerationAction struct {
ID int64
CaseID int64
DecisionID int64
Kind string
Payload []byte
Status string
Attempts int32
AvailableAt pgtype.Timestamptz
LeaseUntil pgtype.Timestamptz
LastError string
CommandID string
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
}
type ModerationAppeal struct {
ID int64
CaseID int64
AppellantUserID int64
AppealText string
TextHash []byte
Fingerprint []byte
Status string
PreviousCaseStatus string
Reviewer string
ReviewReason string
CreatedAt pgtype.Timestamptz
ReviewedAt pgtype.Timestamptz
}
type ModerationAppealLink struct {
ID int64
CaseID int64
AppellantUserID int64
TokenHash []byte
ExpiresAt pgtype.Timestamptz
AppealID *int64
CreatedAt pgtype.Timestamptz
ConsumedAt pgtype.Timestamptz
}
type ModerationCase struct {
ID int64
TargetPeerType string
TargetPeerID int64
Status string
Severity int16
AssignedTo string
Version int64
ReportCount int32
DistinctReporterCount int32
FirstReportAt pgtype.Timestamptz
LastReportAt pgtype.Timestamptz
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
}
type ModerationCaseReport struct {
CaseID int64
ReportID int64
AttachedAt pgtype.Timestamptz
}
type ModerationDecision struct {
ID int64
CaseID int64
AppealID *int64
Kind string
Actor string
Reason string
CommandID string
Fingerprint []byte
CreatedAt pgtype.Timestamptz
}
type ModerationLegacyEphemeralMigration struct {
LegacyReportID int64
ModerationReportID int64
MigratedAt pgtype.Timestamptz
}
type ModerationMediaHold struct {
ReportID int64
ItemOrdinal int16
MediaKind string
StorageKey string
CreatedAt pgtype.Timestamptz
ReleasedAt pgtype.Timestamptz
}
type ModerationReport struct {
ID int64
ReporterUserID int64
Source string
TargetPeerType string
TargetPeerID int64
Reason string
ReportOption string
ReportComment string
CommentHash []byte
Fingerprint []byte
TaxonomyVersion int16
CreatedAt pgtype.Timestamptz
}
type ModerationReportItem struct {
ReportID int64
Ordinal int16
ItemKind string
PeerType string
PeerID int64
ItemID int64
SecondaryID int64
AuthorUserID int64
EvidenceSchemaVersion int16
Evidence []byte
EvidenceHash []byte
}
type NotifySetting struct {
OwnerUserID int64
ScopeKind string
@ -1412,6 +1697,11 @@ type PeerUsername struct {
PeerType string
PeerID int64
UpdatedAt pgtype.Timestamptz
Username string
Active bool
Editable bool
SortOrder int32
CollectibleID *int64
}
type Photo struct {
@ -1423,6 +1713,8 @@ type Photo struct {
HasStickers bool
Sizes []byte
CreatedAt pgtype.Timestamptz
OwnerUserID int64
OrphanedAt pgtype.Timestamptz
}
type Poll struct {
@ -1518,6 +1810,24 @@ type PrivateMessageReaction struct {
UpdatedAt pgtype.Timestamptz
}
type PrivateNoForwardsChat struct {
UserLowID int64
UserHighID int64
EnabledByUserID *int64
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
}
type PrivateNoForwardsRequest struct {
PrivateMessageSenderUserID int64
PrivateMessageID int64
RequesterUserID int64
ResponderUserID int64
ExpiresAt int32
HandledAt int32
CreatedAt pgtype.Timestamptz
}
type ProfilePhoto struct {
OwnerPeerType string
OwnerPeerID int64
@ -1568,6 +1878,16 @@ type SavedDialogPin struct {
CreatedAt pgtype.Timestamptz
}
type SavedMessageReactionTag struct {
UserID int64
MessageBoxID int32
ReactionType string
ReactionValue string
ChosenOrder int32
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
}
type SavedMusic struct {
UserID int64
DocumentID int64
@ -1645,6 +1965,21 @@ type SeedState struct {
UpdatedAt pgtype.Timestamptz
}
type SponsoredMessageImpression struct {
ID int64
UserID int64
RandomIDHash []byte
TargetPeerType string
TargetPeerID int64
AuthorUserID int64
EvidenceSchemaVersion int16
Evidence []byte
EvidenceHash []byte
ReportID *int64
CreatedAt pgtype.Timestamptz
ExpiresAt pgtype.Timestamptz
}
type StarGiftAdminGrantCommand struct {
RecipientUserID int64
CommandKey string
@ -1781,6 +2116,22 @@ type StarGiftCatalogRevision struct {
BackgroundTextColor *int32
}
// Purchase-time snapshot of per-admin channel gift notification intents; delivery uses deterministic private-message replay.
type StarGiftChannelNotificationJob struct {
SavedGiftID int64
TargetUserID int64
GiftDate int32
Action []byte
Attempts int32
NextAttemptAt int32
LeaseUntil int32
DeliveredAt int32
MessageID int32
LastError string
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
}
type StarGiftCollectibleBackdrop struct {
ID int64
CollectibleRevisionID int64
@ -2072,7 +2423,7 @@ type StarGiftUpgradeCommand struct {
SourceEditPts int32
}
// Owner-local service-message aliases (unique outputs and separate prepaid-upgrade notifications) for one saved gift aggregate.
// Viewer-local private service-message aliases to saved gift aggregates; the saved gift owner may be that user or an authorized channel.
type StarGiftUserMessageRef struct {
OwnerUserID int64
MsgID int32
@ -2115,6 +2466,55 @@ type StarsBalance struct {
UpdatedAt pgtype.Timestamptz
}
type StarsGiveaway struct {
ID int64
BuyerUserID int64
FormID int64
ChannelID int64
LaunchMessageID int32
RandomID int64
Stars int64
Users int32
PerUserStars int64
YearlyBoosts int32
UntilDate int32
PurposeJson []byte
State string
CreatedAt int32
}
type StarsPurchaseCommand struct {
BuyerUserID int64
FormID int64
RequestFingerprint []byte
RecipientUserID *int64
Stars int64
Currency string
Amount int64
BalanceAfter int64
TransactionID string
CreatedAt int32
Kind string
SpendPeerType *string
SpendPeerID *int64
PurposeJson []byte
}
type StarsPurchaseForm struct {
BuyerUserID int64
FormID int64
RecipientUserID *int64
Stars int64
Currency string
Amount int64
IssuedAt int32
ExpiresAt int32
Kind string
SpendPeerType *string
SpendPeerID *int64
PurposeJson []byte
}
type StarsTransaction struct {
ID int64
UserID int64
@ -2502,18 +2902,21 @@ type UserBusinessProfile struct {
}
type UserChannelMemberIndex struct {
UserID int64
ChannelID int64
Status string
Megagroup bool
Broadcast bool
Deleted bool
UpdatedAt pgtype.Timestamptz
Role string
LeftAt int32
Forum bool
PublicUsername bool
CanPinMessages bool
UserID int64
ChannelID int64
Status string
Megagroup bool
Broadcast bool
Deleted bool
UpdatedAt pgtype.Timestamptz
Role string
LeftAt int32
Forum bool
PublicUsername bool
CanPinMessages bool
AvailableMinID int32
HistoryClearAnchorID int32
HistoryClearUpdatedAt int32
}
type UserRecentReaction struct {
@ -2530,6 +2933,7 @@ type UserSavedReactionTag struct {
ReactionType string
ReactionValue string
Title string
// Legacy unused column; visible counts are aggregated from saved_message_reaction_tags.
ReactionCount int32
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
@ -2607,6 +3011,67 @@ type UserUpdateWatermark struct {
UpdatedAt pgtype.Timestamptz
}
type VerificationApplication struct {
ID int64
ApplicantUserID int64
TargetType string
TargetID int64
TargetTitle string
TargetUsername string
TargetAccessHash int64
Category string
Description string
OfficialWebsite string
SocialLinks []string
PressLinks []string
AdditionalNote string
Status string
ReviewerAdminID string
DecisionReason string
InternalNote string
CorrelationID string
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
SubmittedAt pgtype.Timestamptz
ReviewedAt pgtype.Timestamptz
Version int64
}
type VerificationApplicationEvent struct {
ID int64
ApplicationID int64
Kind string
FromStatus string
ToStatus string
Actor string
Reason string
Note string
CorrelationID string
CreatedAt pgtype.Timestamptz
}
type VerificationIcon struct {
ID int64
DocumentID int64
OwnerBotID int64
Name string
Active bool
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
}
type VerificationNotificationOutbox struct {
ID int64
ApplicationID int64
RecipientUserID int64
Kind string
Payload []byte
Attempts int32
DeliveredAt pgtype.Timestamptz
LastError string
CreatedAt pgtype.Timestamptz
}
type WebAuthorization struct {
Hash int64
RequestID int64

View file

@ -14,7 +14,7 @@ func TestStarGiftLifecycleMigrationsApply(t *testing.T) {
if err != nil {
t.Fatalf("migrate star gift lifecycle schema: %v", err)
}
if status.Dirty || status.Empty || status.Version != 20260714003127 {
t.Fatalf("migration status = %+v, want clean version 20260714003127", status)
if status.Dirty || status.Empty || status.Version != 20260714003129 {
t.Fatalf("migration status = %+v, want clean version 20260714003129", status)
}
}