s3 support

This commit is contained in:
onysd 2026-08-04 23:09:11 +03:00
parent fa5cfaf14d
commit 03f10b66ee
53 changed files with 2796 additions and 102 deletions

View file

@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"os"
"strconv"
"strings"
"time"
@ -2541,3 +2542,133 @@ func clampBotVerificationLimit(limit int) int {
}
return limit
}
// storageUsageListDefaultLimit/storageUsageListMaxLimit bound the per-account
// storage breakdown page, same convention as the account rating list above.
const (
storageUsageListDefaultLimit = 50
storageUsageListMaxLimit = 200
)
// perOwnerMediaSizeSQL is shared between StorageStats and
// ListAccountStorageUsage: each document row's size is its stored column;
// each photo row has no single size column (JSONB sizes holds one entry per
// rendition), so its "attributed size" is the largest rendition -- the
// dominant cost, thumbnails are comparatively tiny. This is an
// approximation (not the exact sum of every rendition's blob bytes, which
// would require joining file_blobs by location_key prefix) chosen for admin
// visibility, not billing precision.
const perOwnerMediaSizeSQL = `
SELECT owner_user_id, size FROM documents
UNION ALL
SELECT p.owner_user_id, COALESCE((
SELECT MAX((elem->>'size')::bigint) FROM jsonb_array_elements(p.sizes) elem
), 0) AS size
FROM photos p
`
// StorageStatsRow is the admin panel's storage overview: physical bytes
// (from file_blobs, backend-dedup-aware -- what's actually consuming disk
// or S3) versus logical bytes (sum of the same approximate per-row
// attribution the per-account breakdown uses, which can legitimately be
// higher than physical when identical content is shared by more than one
// document/photo).
type StorageStatsRow struct {
PhysicalBytes int64 `json:"PhysicalBytes,string"`
LogicalBytes int64 `json:"LogicalBytes,string"`
UnattributedBytes int64 `json:"UnattributedBytes,string"`
DocumentCount int64 `json:"DocumentCount,string"`
PhotoCount int64 `json:"PhotoCount,string"`
AccountCount int64 `json:"AccountCount,string"`
BackendKind string
}
// StorageStats returns the admin panel's storage overview.
func (s *readStore) StorageStats(ctx context.Context) (StorageStatsRow, error) {
var stats StorageStatsRow
if err := s.pool.QueryRow(ctx, `SELECT COALESCE(SUM(size), 0)::bigint FROM file_blobs`).Scan(&stats.PhysicalBytes); err != nil {
return StorageStatsRow{}, fmt.Errorf("sum physical blob bytes: %w", err)
}
if err := s.pool.QueryRow(ctx, `
SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x`).Scan(&stats.LogicalBytes); err != nil {
return StorageStatsRow{}, fmt.Errorf("sum logical media bytes: %w", err)
}
if err := s.pool.QueryRow(ctx, `
SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id = 0`).Scan(&stats.UnattributedBytes); err != nil {
return StorageStatsRow{}, fmt.Errorf("sum unattributed media bytes: %w", err)
}
if err := s.pool.QueryRow(ctx, `SELECT count(*)::bigint FROM documents`).Scan(&stats.DocumentCount); err != nil {
return StorageStatsRow{}, fmt.Errorf("count documents: %w", err)
}
if err := s.pool.QueryRow(ctx, `SELECT count(*)::bigint FROM photos`).Scan(&stats.PhotoCount); err != nil {
return StorageStatsRow{}, fmt.Errorf("count photos: %w", err)
}
if err := s.pool.QueryRow(ctx, `
SELECT count(DISTINCT owner_user_id)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id <> 0`).Scan(&stats.AccountCount); err != nil {
return StorageStatsRow{}, fmt.Errorf("count storage accounts: %w", err)
}
stats.BackendKind = strings.ToLower(strings.TrimSpace(os.Getenv("TELESRV_BLOB_BACKEND")))
if stats.BackendKind == "" {
stats.BackendKind = "localfs"
}
return stats, nil
}
// AccountStorageRow is one account's row in the per-account storage
// breakdown table.
type AccountStorageRow struct {
UserID int64 `json:"UserID,string"`
Username string
FirstName string
Bytes int64 `json:"Bytes,string"`
FileCount int64 `json:"FileCount,string"`
}
// ListAccountStorageUsage pages the per-account storage breakdown, largest
// user first. Offset-based (not keyset): storage administration on a
// self-hosted deployment doesn't need to support arbitrarily deep pages
// efficiently the way an infinite-scroll feed does.
func (s *readStore) ListAccountStorageUsage(ctx context.Context, offset, limit int) ([]AccountStorageRow, bool, error) {
if limit <= 0 {
limit = storageUsageListDefaultLimit
}
if limit > storageUsageListMaxLimit {
limit = storageUsageListMaxLimit
}
if offset < 0 {
offset = 0
}
rows, err := s.pool.Query(ctx, `
WITH totals AS (
SELECT owner_user_id, SUM(size)::bigint AS bytes, COUNT(*)::bigint AS file_count
FROM (`+perOwnerMediaSizeSQL+`) x
WHERE owner_user_id <> 0
GROUP BY owner_user_id
)
SELECT t.owner_user_id, COALESCE(u.username, ''), COALESCE(u.first_name, ''), t.bytes, t.file_count
FROM totals t
LEFT JOIN users u ON u.id = t.owner_user_id
ORDER BY t.bytes DESC, t.owner_user_id
OFFSET $1
LIMIT $2`, offset, limit+1)
if err != nil {
return nil, false, fmt.Errorf("list account storage usage: %w", err)
}
defer rows.Close()
out := make([]AccountStorageRow, 0, limit+1)
for rows.Next() {
var item AccountStorageRow
if err := rows.Scan(&item.UserID, &item.Username, &item.FirstName, &item.Bytes, &item.FileCount); err != nil {
return nil, false, err
}
out = append(out, item)
}
if err := rows.Err(); err != nil {
return nil, false, err
}
hasMore := len(out) > limit
if hasMore {
out = out[:limit]
}
return out, hasMore, nil
}

View file

@ -78,6 +78,8 @@ func (s *server) routes() http.Handler {
mux.Handle("GET /api/collectible-usernames/{id}", s.requireAuthAPI(http.HandlerFunc(s.handleCollectibleUsernameDetailAPI)))
mux.Handle("GET /api/account-ratings", s.requireAuthAPI(http.HandlerFunc(s.handleAccountRatingsAPI)))
mux.Handle("GET /api/account-ratings/{user_id}", s.requireAuthAPI(http.HandlerFunc(s.handleAccountRatingDetailAPI)))
mux.Handle("GET /api/storage/stats", s.requireAuthAPI(http.HandlerFunc(s.handleStorageStatsAPI)))
mux.Handle("GET /api/storage/accounts", s.requireAuthAPI(http.HandlerFunc(s.handleStorageAccountsAPI)))
mux.Handle("GET /api/moderation/cases", s.requireAuthAPI(http.HandlerFunc(s.handleModerationCasesAPI)))
mux.Handle("GET /api/moderation/cases/{id}", s.requireAuthAPI(http.HandlerFunc(s.handleModerationCaseAPI)))
mux.Handle("GET /api/moderation/reports/{id}", s.requireAuthAPI(http.HandlerFunc(s.handleModerationReportAPI)))
@ -2342,6 +2344,53 @@ func (s *server) handleAccountRatingsAPI(w http.ResponseWriter, r *http.Request)
})
}
func (s *server) handleStorageStatsAPI(w http.ResponseWriter, r *http.Request) {
if s.read == nil {
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")
return
}
stats, err := s.read.StorageStats(r.Context())
if err != nil {
writeAPIError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, stats)
}
// handleStorageAccountsAPI pages the per-account storage breakdown,
// largest first.
func (s *server) handleStorageAccountsAPI(w http.ResponseWriter, r *http.Request) {
if s.read == nil {
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")
return
}
query := r.URL.Query()
offset, err := parseInt(query.Get("offset"))
if err != nil || offset < 0 {
writeAPIError(w, http.StatusBadRequest, "invalid offset")
return
}
limit, err := parseInt(query.Get("limit"))
if err != nil || limit < 0 {
writeAPIError(w, http.StatusBadRequest, "invalid limit")
return
}
rows, hasMore, err := s.read.ListAccountStorageUsage(r.Context(), offset, limit)
if err != nil {
writeAPIError(w, http.StatusInternalServerError, err.Error())
return
}
nextOffset := 0
if hasMore {
nextOffset = offset + len(rows)
}
writeJSON(w, http.StatusOK, map[string]any{
"rows": rows,
"has_more": hasMore,
"next_offset": nextOffset,
})
}
func (s *server) handleAccountRatingDetailAPI(w http.ResponseWriter, r *http.Request) {
if s.read == nil {
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -23,7 +23,7 @@
})();
</script>
<script type="module" crossorigin src="/assets/index-D0qBFBU-.js"></script>
<script type="module" crossorigin src="/assets/index-CmjlUwWc.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-EkAGiEK9.css">
</head>
<body>

View file

@ -4,6 +4,8 @@ import type {
AccountRatingDetail,
AccountRatingListResponse,
AccountStatsResponse,
AccountStorageListResponse,
StorageStatsResponse,
AdminLoginResult,
AdminSession,
BotDetail,
@ -165,6 +167,9 @@ export const api = {
request<AccountRatingListResponse>(`/api/account-ratings?${params.toString()}`),
accountRating: (userID: string) =>
request<AccountRatingDetail>(`/api/account-ratings/${encodeURIComponent(userID)}`),
storageStats: () => request<StorageStatsResponse>("/api/storage/stats"),
storageAccounts: (params: URLSearchParams) =>
request<AccountStorageListResponse>(`/api/storage/accounts?${params.toString()}`),
verificationApplications: (params: URLSearchParams) =>
request<VerificationApplicationListResponse>(`/api/verification/applications?${params.toString()}`),
// The application id is an int64 decimal string end to end, so it is never

View file

@ -99,6 +99,7 @@ export function Shell({
)}
<NavLink icon={<AtSign size={16} />} href="/collectible-usernames" route={route} navigate={navigate}>{"NFT Usernames"}</NavLink>
<NavLink icon={<Trophy size={16} />} href="/account-ratings" route={route} navigate={navigate}>{"Account Rating"}</NavLink>
<NavLink icon={<Database size={16} />} href="/storage" route={route} navigate={navigate}>{"Storage"}</NavLink>
<NavLink icon={<Gift size={16} />} href="/gifts" route={route} navigate={navigate}>{"Star Gifts"}</NavLink>
<NavLink icon={<Send size={16} />} href="/give-gifts" route={route} navigate={navigate}>{"Give Gifts"}</NavLink>
<NavLink icon={<Sticker size={16} />} href="/stickers" route={route} navigate={navigate}>{"Stickers"}</NavLink>

View file

@ -154,6 +154,26 @@ export function toSmallestUnits(value: string, currency: string): string | null
return digits === "" ? "0" : digits;
}
// formatBytes renders a byte count (as the JSON-string int64 the API sends)
// in the largest unit that keeps it readable. Parses the decimal string
// directly rather than through toNumeric first so precision past
// Number.MAX_SAFE_INTEGER isn't silently lost before the division.
export function formatBytes(value: string): string {
const raw = (value ?? "").trim();
if (!raw || !/^\d+$/.test(raw)) return "0 B";
const bytes = Number(raw);
if (!Number.isFinite(bytes)) return `${raw} B`;
const units = ["B", "KB", "MB", "GB", "TB", "PB"];
let size = bytes;
let unit = 0;
while (size >= 1024 && unit < units.length - 1) {
size /= 1024;
unit++;
}
const precision = unit === 0 ? 0 : 1;
return `${size.toFixed(precision)} ${units[unit]}`;
}
export function formatSigned(value: string): string {
const raw = (value ?? "").trim();
if (!raw) return "0";

View file

@ -19,6 +19,7 @@ import { StickerSetsPage } from "./StickerSetsPage";
import { GiveGiftsPage } from "./GiveGiftsPage";
import { ModerationCaseDetailPage } from "./ModerationCaseDetailPage";
import { ModerationCasesPage } from "./ModerationCasesPage";
import { StoragePage } from "./StoragePage";
import { BotVerificationPage } from "./BotVerificationPage";
import { BotVerificationRequestPage } from "./BotVerificationRequestPage";
import { VerificationDetailPage } from "./VerificationDetailPage";
@ -84,6 +85,9 @@ export function Routes({ route, navigate }: { route: RouteState; navigate: Navig
if (route.path === "/account-ratings") {
return <AccountRatingsPage navigate={navigate} />;
}
if (route.path === "/storage") {
return <StoragePage navigate={navigate} />;
}
if (accountID) {
return <AccountDetailPage id={Number(accountID)} navigate={navigate} />;
}

View file

@ -0,0 +1,118 @@
import { ChevronDown, Loader2, RefreshCw } from "lucide-react";
import { useEffect, useState } from "react";
import { api, errorMessage } from "../api";
import { Alert, EmptyRow, Metric, PageFrame } from "../components/ui";
import { displayUsername, formatBytes, formatQuantity } from "../lib/format";
import type { Navigate } from "../routing";
import type { AccountStorageRow, StorageStatsResponse } from "../types";
export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
const [stats, setStats] = useState<StorageStatsResponse | null>(null);
const [rows, setRows] = useState<AccountStorageRow[]>([]);
const [hasMore, setHasMore] = useState(false);
const [offset, setOffset] = useState(0);
const [busy, setBusy] = useState(false);
const [error, setError] = useState("");
async function loadStats() {
try {
setStats(await api.storageStats());
} catch {
// Stats are a header nicety; a failure here shouldn't block the list.
}
}
async function loadAccounts(next = false) {
setBusy(true);
setError("");
const at = next ? offset : 0;
const params = new URLSearchParams({ limit: "50", offset: String(at) });
try {
const result = await api.storageAccounts(params);
const page = result.rows ?? [];
setRows((current) => (next ? [...current, ...page] : page));
setOffset(result.next_offset);
setHasMore(Boolean(result.has_more));
} catch (err) {
setError(errorMessage(err));
} finally {
setBusy(false);
}
}
function refresh() {
void loadStats();
void loadAccounts(false);
}
useEffect(() => {
refresh();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
// Physical is what actually consumes disk/S3 (deduplicated); logical is the
// sum of what the per-account table below adds up to. They legitimately
// differ when the same content is shared by more than one document/photo.
const dedupBytes = stats ? Math.max(0, Number(stats.LogicalBytes) - Number(stats.PhysicalBytes)) : 0;
return (
<PageFrame
title={"Storage"}
eyebrow={"Media / Storage usage"}
actions={
<button className="btn icon-text" type="button" onClick={refresh} disabled={busy}>
<RefreshCw size={15} className={busy ? "spin" : ""} /> {"Refresh"}
</button>
}
>
{error && <Alert>{error}</Alert>}
<div className="metric-row">
<Metric label={"Physical usage (on disk / S3)"} value={stats ? formatBytes(stats.PhysicalBytes) : "-"} />
<Metric label={"Logical usage (sum per account)"} value={stats ? formatBytes(stats.LogicalBytes) : "-"} />
<Metric label={"Saved by dedup"} value={formatBytes(String(dedupBytes))} tone={dedupBytes > 0 ? "good" : "neutral"} />
<Metric label={"Backend"} value={stats?.BackendKind ?? "-"} />
</div>
<div className="metric-row">
<Metric label={"Documents"} value={stats ? formatQuantity(stats.DocumentCount) : "-"} />
<Metric label={"Photos"} value={stats ? formatQuantity(stats.PhotoCount) : "-"} />
<Metric label={"Accounts with media"} value={stats ? formatQuantity(stats.AccountCount) : "-"} />
<Metric
label={"Unattributed"}
value={stats ? formatBytes(stats.UnattributedBytes) : "-"}
tone={stats && Number(stats.UnattributedBytes) > 0 ? "warn" : "neutral"}
/>
</div>
<div className="table-wrap">
<table className="data-table">
<thead>
<tr>
<th>{"User ID"}</th>
<th>{"Account"}</th>
<th>{"Storage used"}</th>
<th>{"Files"}</th>
</tr>
</thead>
<tbody>
{rows.map((row) => (
<tr key={row.UserID}>
<td className="mono">{row.UserID}</td>
<td>{displayUsername(row.Username) || row.FirstName || "-"}</td>
<td className="mono">{formatBytes(row.Bytes)}</td>
<td className="mono">{formatQuantity(row.FileCount)}</td>
</tr>
))}
{rows.length === 0 && <EmptyRow colSpan={4} />}
</tbody>
</table>
</div>
{hasMore && (
<div className="toolbar">
<button className="btn icon-text" type="button" onClick={() => loadAccounts(true)} disabled={busy}>
{busy ? <Loader2 size={15} className="spin" /> : <ChevronDown size={15} />} {"Load more"}
</button>
</div>
)}
</PageFrame>
);
}

View file

@ -21,6 +21,7 @@ export function routeTitle(pathname: string): string {
if (pathname.startsWith("/verification")) return "Official Verification";
if (pathname.startsWith("/collectible-usernames")) return "Collectible Usernames";
if (pathname.startsWith("/account-ratings")) return "Account Rating";
if (pathname.startsWith("/storage")) return "Storage";
if (pathname.startsWith("/accounts")) return "Accounts";
if (pathname.startsWith("/channels")) return "Supergroups and Channels";
if (pathname.startsWith("/bots")) return "Bots";
@ -39,6 +40,7 @@ export function routeSubtitle(pathname: string): string {
if (pathname.startsWith("/verification")) return "Console / Verification";
if (pathname.startsWith("/collectible-usernames")) return "Console / Collectible usernames";
if (pathname.startsWith("/account-ratings")) return "Console / Account rating";
if (pathname.startsWith("/storage")) return "Console / Storage";
if (pathname.startsWith("/accounts")) return "Console / Accounts";
if (pathname.startsWith("/channels")) return "Console / Channels";
if (pathname.startsWith("/bots")) return "Console / Bots";

View file

@ -789,6 +789,30 @@ export type AccountStatsResponse = {
online: number;
};
export type StorageStatsResponse = {
PhysicalBytes: string;
LogicalBytes: string;
UnattributedBytes: string;
DocumentCount: string;
PhotoCount: string;
AccountCount: string;
BackendKind: string;
};
export type AccountStorageRow = {
UserID: string;
Username: string;
FirstName: string;
Bytes: string;
FileCount: string;
};
export type AccountStorageListResponse = {
rows: AccountStorageRow[] | null;
has_more: boolean;
next_offset: number;
};
export type ChannelListResponse = {
query: string;
limit: number;

View file

@ -710,14 +710,36 @@ func run(logger *zap.Logger) error {
cachedPhotos := userprojection.NewCachedPhotoProvider(mediaStore, userprojection.DefaultPhotoCacheTTL)
privacyStore := privacyapp.NewCachedPrivacyStore(postgres.NewPrivacyStore(pool), 0)
storyStore := postgres.NewStoryStore(pool)
blobBackend, err := filesapp.NewLocalFS(cfg.BlobDir)
// Transient upload-part scratch storage always stays on local disk
// (TELESRV_BLOB_DIR) regardless of the permanent blob backend below --
// one S3 round trip per ~512KB chunk isn't worth it for data deleted
// within minutes of assembly.
localBlobFS, err := filesapp.NewLocalFS(cfg.BlobDir)
if err != nil {
return fmt.Errorf("init blob backend: %w", err)
return fmt.Errorf("init local blob dir: %w", err)
}
var blobBackend filesapp.BlobBackend = localBlobFS
var spaceGuard filesapp.SpaceGuard = filesapp.NoopSpaceGuard{}
if cfg.BlobBackendKind == "s3" {
s3Backend, err := filesapp.NewS3FS(ctx, cfg.S3Endpoint, cfg.S3AccessKeyID, cfg.S3SecretAccessKey, cfg.S3Bucket, cfg.S3Region, cfg.S3UseSSL, cfg.S3PathStyle)
if err != nil {
return fmt.Errorf("init s3 blob backend: %w", err)
}
blobBackend = s3Backend
logger.Info("blob backend ready", zap.String("backend", "s3"), zap.String("bucket", cfg.S3Bucket), zap.String("endpoint", cfg.S3Endpoint))
if cfg.StorageLowSpaceGuardEnable && cfg.StorageMaxTotalBytes > 0 {
s3Guard := filesapp.NewS3BudgetSpaceGuard(cfg.StorageMaxTotalBytes)
spaceGuard = s3Guard
go filesapp.NewS3DiskUsageWorker(s3Guard, mediaStore, cfg.StorageUsageRefreshInterval, logger.Named("files").Named("diskusage")).Run(ctx)
}
} else {
logger.Info("blob backend ready", zap.String("backend", "localfs"), zap.String("dir", cfg.BlobDir))
if cfg.StorageLowSpaceGuardEnable && cfg.StorageMinFreeBytes > 0 {
localGuard := filesapp.NewLocalDiskSpaceGuard(cfg.StorageMinFreeBytes)
spaceGuard = localGuard
go filesapp.NewLocalDiskUsageWorker(localGuard, cfg.BlobDir, cfg.StorageUsageRefreshInterval, logger.Named("files").Named("diskusage")).Run(ctx)
}
}
logger.Info("blob backend ready",
zap.String("backend", "localfs"),
zap.String("dir", cfg.BlobDir),
)
filesService := filesapp.NewService(mediaStore, blobBackend, cfg.DC,
filesapp.WithLogger(logger),
filesapp.WithUploadPartQuota(domain.UploadPartQuota{
@ -725,6 +747,8 @@ func run(logger *zap.Logger) error {
MaxParts: cfg.UploadInFlightMaxParts,
MaxFiles: cfg.UploadInFlightMaxFiles,
}),
filesapp.WithUploadPartBackend(localBlobFS),
filesapp.WithSpaceGuard(spaceGuard),
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
externalMediaOption(cfg),
webPagePreviewOption(cfg),
@ -822,6 +846,10 @@ func run(logger *zap.Logger) error {
Restrictions: adminStore,
OfficialGifts: officialgifts.New(cfg.OfficialGiftsDir),
})
storageRetentionMaxAge := cfg.StorageRetentionMaxAge
if !cfg.StorageRetentionEnable {
storageRetentionMaxAge = 0
}
go maintenance.NewRetentionWorker(dispatchOutboxStore, tempAuthKeyStore, logger.Named("maintenance").Named("retention"),
cfg.UpdateEventRetention,
cfg.RetentionInterval,
@ -836,6 +864,7 @@ func run(logger *zap.Logger) error {
WithUserUpdateRetention(updateEventStore).
WithChannelUpdateRetention(channelStore).
WithOrphanAuthKeyRetention(authKeyStore, activeSessions, cfg.OrphanAuthKeyRetention).
WithOrphanedMediaRetention(filesService, storageRetentionMaxAge).
Run(ctx)
go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"),
cfg.UploadPartTTL,