s3 support
This commit is contained in:
parent
fa5cfaf14d
commit
03f10b66ee
53 changed files with 2796 additions and 102 deletions
|
|
@ -5,6 +5,7 @@ import (
|
|||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -2541,3 +2542,133 @@ func clampBotVerificationLimit(limit int) int {
|
|||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
// storageUsageListDefaultLimit/storageUsageListMaxLimit bound the per-account
|
||||
// storage breakdown page, same convention as the account rating list above.
|
||||
const (
|
||||
storageUsageListDefaultLimit = 50
|
||||
storageUsageListMaxLimit = 200
|
||||
)
|
||||
|
||||
// perOwnerMediaSizeSQL is shared between StorageStats and
|
||||
// ListAccountStorageUsage: each document row's size is its stored column;
|
||||
// each photo row has no single size column (JSONB sizes holds one entry per
|
||||
// rendition), so its "attributed size" is the largest rendition -- the
|
||||
// dominant cost, thumbnails are comparatively tiny. This is an
|
||||
// approximation (not the exact sum of every rendition's blob bytes, which
|
||||
// would require joining file_blobs by location_key prefix) chosen for admin
|
||||
// visibility, not billing precision.
|
||||
const perOwnerMediaSizeSQL = `
|
||||
SELECT owner_user_id, size FROM documents
|
||||
UNION ALL
|
||||
SELECT p.owner_user_id, COALESCE((
|
||||
SELECT MAX((elem->>'size')::bigint) FROM jsonb_array_elements(p.sizes) elem
|
||||
), 0) AS size
|
||||
FROM photos p
|
||||
`
|
||||
|
||||
// StorageStatsRow is the admin panel's storage overview: physical bytes
|
||||
// (from file_blobs, backend-dedup-aware -- what's actually consuming disk
|
||||
// or S3) versus logical bytes (sum of the same approximate per-row
|
||||
// attribution the per-account breakdown uses, which can legitimately be
|
||||
// higher than physical when identical content is shared by more than one
|
||||
// document/photo).
|
||||
type StorageStatsRow struct {
|
||||
PhysicalBytes int64 `json:"PhysicalBytes,string"`
|
||||
LogicalBytes int64 `json:"LogicalBytes,string"`
|
||||
UnattributedBytes int64 `json:"UnattributedBytes,string"`
|
||||
DocumentCount int64 `json:"DocumentCount,string"`
|
||||
PhotoCount int64 `json:"PhotoCount,string"`
|
||||
AccountCount int64 `json:"AccountCount,string"`
|
||||
BackendKind string
|
||||
}
|
||||
|
||||
// StorageStats returns the admin panel's storage overview.
|
||||
func (s *readStore) StorageStats(ctx context.Context) (StorageStatsRow, error) {
|
||||
var stats StorageStatsRow
|
||||
if err := s.pool.QueryRow(ctx, `SELECT COALESCE(SUM(size), 0)::bigint FROM file_blobs`).Scan(&stats.PhysicalBytes); err != nil {
|
||||
return StorageStatsRow{}, fmt.Errorf("sum physical blob bytes: %w", err)
|
||||
}
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x`).Scan(&stats.LogicalBytes); err != nil {
|
||||
return StorageStatsRow{}, fmt.Errorf("sum logical media bytes: %w", err)
|
||||
}
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id = 0`).Scan(&stats.UnattributedBytes); err != nil {
|
||||
return StorageStatsRow{}, fmt.Errorf("sum unattributed media bytes: %w", err)
|
||||
}
|
||||
if err := s.pool.QueryRow(ctx, `SELECT count(*)::bigint FROM documents`).Scan(&stats.DocumentCount); err != nil {
|
||||
return StorageStatsRow{}, fmt.Errorf("count documents: %w", err)
|
||||
}
|
||||
if err := s.pool.QueryRow(ctx, `SELECT count(*)::bigint FROM photos`).Scan(&stats.PhotoCount); err != nil {
|
||||
return StorageStatsRow{}, fmt.Errorf("count photos: %w", err)
|
||||
}
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
SELECT count(DISTINCT owner_user_id)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id <> 0`).Scan(&stats.AccountCount); err != nil {
|
||||
return StorageStatsRow{}, fmt.Errorf("count storage accounts: %w", err)
|
||||
}
|
||||
stats.BackendKind = strings.ToLower(strings.TrimSpace(os.Getenv("TELESRV_BLOB_BACKEND")))
|
||||
if stats.BackendKind == "" {
|
||||
stats.BackendKind = "localfs"
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// AccountStorageRow is one account's row in the per-account storage
|
||||
// breakdown table.
|
||||
type AccountStorageRow struct {
|
||||
UserID int64 `json:"UserID,string"`
|
||||
Username string
|
||||
FirstName string
|
||||
Bytes int64 `json:"Bytes,string"`
|
||||
FileCount int64 `json:"FileCount,string"`
|
||||
}
|
||||
|
||||
// ListAccountStorageUsage pages the per-account storage breakdown, largest
|
||||
// user first. Offset-based (not keyset): storage administration on a
|
||||
// self-hosted deployment doesn't need to support arbitrarily deep pages
|
||||
// efficiently the way an infinite-scroll feed does.
|
||||
func (s *readStore) ListAccountStorageUsage(ctx context.Context, offset, limit int) ([]AccountStorageRow, bool, error) {
|
||||
if limit <= 0 {
|
||||
limit = storageUsageListDefaultLimit
|
||||
}
|
||||
if limit > storageUsageListMaxLimit {
|
||||
limit = storageUsageListMaxLimit
|
||||
}
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
WITH totals AS (
|
||||
SELECT owner_user_id, SUM(size)::bigint AS bytes, COUNT(*)::bigint AS file_count
|
||||
FROM (`+perOwnerMediaSizeSQL+`) x
|
||||
WHERE owner_user_id <> 0
|
||||
GROUP BY owner_user_id
|
||||
)
|
||||
SELECT t.owner_user_id, COALESCE(u.username, ''), COALESCE(u.first_name, ''), t.bytes, t.file_count
|
||||
FROM totals t
|
||||
LEFT JOIN users u ON u.id = t.owner_user_id
|
||||
ORDER BY t.bytes DESC, t.owner_user_id
|
||||
OFFSET $1
|
||||
LIMIT $2`, offset, limit+1)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("list account storage usage: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]AccountStorageRow, 0, limit+1)
|
||||
for rows.Next() {
|
||||
var item AccountStorageRow
|
||||
if err := rows.Scan(&item.UserID, &item.Username, &item.FirstName, &item.Bytes, &item.FileCount); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
hasMore := len(out) > limit
|
||||
if hasMore {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out, hasMore, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,8 @@ func (s *server) routes() http.Handler {
|
|||
mux.Handle("GET /api/collectible-usernames/{id}", s.requireAuthAPI(http.HandlerFunc(s.handleCollectibleUsernameDetailAPI)))
|
||||
mux.Handle("GET /api/account-ratings", s.requireAuthAPI(http.HandlerFunc(s.handleAccountRatingsAPI)))
|
||||
mux.Handle("GET /api/account-ratings/{user_id}", s.requireAuthAPI(http.HandlerFunc(s.handleAccountRatingDetailAPI)))
|
||||
mux.Handle("GET /api/storage/stats", s.requireAuthAPI(http.HandlerFunc(s.handleStorageStatsAPI)))
|
||||
mux.Handle("GET /api/storage/accounts", s.requireAuthAPI(http.HandlerFunc(s.handleStorageAccountsAPI)))
|
||||
mux.Handle("GET /api/moderation/cases", s.requireAuthAPI(http.HandlerFunc(s.handleModerationCasesAPI)))
|
||||
mux.Handle("GET /api/moderation/cases/{id}", s.requireAuthAPI(http.HandlerFunc(s.handleModerationCaseAPI)))
|
||||
mux.Handle("GET /api/moderation/reports/{id}", s.requireAuthAPI(http.HandlerFunc(s.handleModerationReportAPI)))
|
||||
|
|
@ -2342,6 +2344,53 @@ func (s *server) handleAccountRatingsAPI(w http.ResponseWriter, r *http.Request)
|
|||
})
|
||||
}
|
||||
|
||||
func (s *server) handleStorageStatsAPI(w http.ResponseWriter, r *http.Request) {
|
||||
if s.read == nil {
|
||||
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")
|
||||
return
|
||||
}
|
||||
stats, err := s.read.StorageStats(r.Context())
|
||||
if err != nil {
|
||||
writeAPIError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, stats)
|
||||
}
|
||||
|
||||
// handleStorageAccountsAPI pages the per-account storage breakdown,
|
||||
// largest first.
|
||||
func (s *server) handleStorageAccountsAPI(w http.ResponseWriter, r *http.Request) {
|
||||
if s.read == nil {
|
||||
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")
|
||||
return
|
||||
}
|
||||
query := r.URL.Query()
|
||||
offset, err := parseInt(query.Get("offset"))
|
||||
if err != nil || offset < 0 {
|
||||
writeAPIError(w, http.StatusBadRequest, "invalid offset")
|
||||
return
|
||||
}
|
||||
limit, err := parseInt(query.Get("limit"))
|
||||
if err != nil || limit < 0 {
|
||||
writeAPIError(w, http.StatusBadRequest, "invalid limit")
|
||||
return
|
||||
}
|
||||
rows, hasMore, err := s.read.ListAccountStorageUsage(r.Context(), offset, limit)
|
||||
if err != nil {
|
||||
writeAPIError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
nextOffset := 0
|
||||
if hasMore {
|
||||
nextOffset = offset + len(rows)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"rows": rows,
|
||||
"has_more": hasMore,
|
||||
"next_offset": nextOffset,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *server) handleAccountRatingDetailAPI(w http.ResponseWriter, r *http.Request) {
|
||||
if s.read == nil {
|
||||
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")
|
||||
|
|
|
|||
10
cmd/telesrv-admin/web/dist/assets/index-CmjlUwWc.js
vendored
Normal file
10
cmd/telesrv-admin/web/dist/assets/index-CmjlUwWc.js
vendored
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
2
cmd/telesrv-admin/web/dist/index.html
vendored
2
cmd/telesrv-admin/web/dist/index.html
vendored
|
|
@ -23,7 +23,7 @@
|
|||
})();
|
||||
</script>
|
||||
|
||||
<script type="module" crossorigin src="/assets/index-D0qBFBU-.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index-CmjlUwWc.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-EkAGiEK9.css">
|
||||
</head>
|
||||
<body>
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ import type {
|
|||
AccountRatingDetail,
|
||||
AccountRatingListResponse,
|
||||
AccountStatsResponse,
|
||||
AccountStorageListResponse,
|
||||
StorageStatsResponse,
|
||||
AdminLoginResult,
|
||||
AdminSession,
|
||||
BotDetail,
|
||||
|
|
@ -165,6 +167,9 @@ export const api = {
|
|||
request<AccountRatingListResponse>(`/api/account-ratings?${params.toString()}`),
|
||||
accountRating: (userID: string) =>
|
||||
request<AccountRatingDetail>(`/api/account-ratings/${encodeURIComponent(userID)}`),
|
||||
storageStats: () => request<StorageStatsResponse>("/api/storage/stats"),
|
||||
storageAccounts: (params: URLSearchParams) =>
|
||||
request<AccountStorageListResponse>(`/api/storage/accounts?${params.toString()}`),
|
||||
verificationApplications: (params: URLSearchParams) =>
|
||||
request<VerificationApplicationListResponse>(`/api/verification/applications?${params.toString()}`),
|
||||
// The application id is an int64 decimal string end to end, so it is never
|
||||
|
|
|
|||
|
|
@ -99,6 +99,7 @@ export function Shell({
|
|||
)}
|
||||
<NavLink icon={<AtSign size={16} />} href="/collectible-usernames" route={route} navigate={navigate}>{"NFT Usernames"}</NavLink>
|
||||
<NavLink icon={<Trophy size={16} />} href="/account-ratings" route={route} navigate={navigate}>{"Account Rating"}</NavLink>
|
||||
<NavLink icon={<Database size={16} />} href="/storage" route={route} navigate={navigate}>{"Storage"}</NavLink>
|
||||
<NavLink icon={<Gift size={16} />} href="/gifts" route={route} navigate={navigate}>{"Star Gifts"}</NavLink>
|
||||
<NavLink icon={<Send size={16} />} href="/give-gifts" route={route} navigate={navigate}>{"Give Gifts"}</NavLink>
|
||||
<NavLink icon={<Sticker size={16} />} href="/stickers" route={route} navigate={navigate}>{"Stickers"}</NavLink>
|
||||
|
|
|
|||
|
|
@ -154,6 +154,26 @@ export function toSmallestUnits(value: string, currency: string): string | null
|
|||
return digits === "" ? "0" : digits;
|
||||
}
|
||||
|
||||
// formatBytes renders a byte count (as the JSON-string int64 the API sends)
|
||||
// in the largest unit that keeps it readable. Parses the decimal string
|
||||
// directly rather than through toNumeric first so precision past
|
||||
// Number.MAX_SAFE_INTEGER isn't silently lost before the division.
|
||||
export function formatBytes(value: string): string {
|
||||
const raw = (value ?? "").trim();
|
||||
if (!raw || !/^\d+$/.test(raw)) return "0 B";
|
||||
const bytes = Number(raw);
|
||||
if (!Number.isFinite(bytes)) return `${raw} B`;
|
||||
const units = ["B", "KB", "MB", "GB", "TB", "PB"];
|
||||
let size = bytes;
|
||||
let unit = 0;
|
||||
while (size >= 1024 && unit < units.length - 1) {
|
||||
size /= 1024;
|
||||
unit++;
|
||||
}
|
||||
const precision = unit === 0 ? 0 : 1;
|
||||
return `${size.toFixed(precision)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
export function formatSigned(value: string): string {
|
||||
const raw = (value ?? "").trim();
|
||||
if (!raw) return "0";
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import { StickerSetsPage } from "./StickerSetsPage";
|
|||
import { GiveGiftsPage } from "./GiveGiftsPage";
|
||||
import { ModerationCaseDetailPage } from "./ModerationCaseDetailPage";
|
||||
import { ModerationCasesPage } from "./ModerationCasesPage";
|
||||
import { StoragePage } from "./StoragePage";
|
||||
import { BotVerificationPage } from "./BotVerificationPage";
|
||||
import { BotVerificationRequestPage } from "./BotVerificationRequestPage";
|
||||
import { VerificationDetailPage } from "./VerificationDetailPage";
|
||||
|
|
@ -84,6 +85,9 @@ export function Routes({ route, navigate }: { route: RouteState; navigate: Navig
|
|||
if (route.path === "/account-ratings") {
|
||||
return <AccountRatingsPage navigate={navigate} />;
|
||||
}
|
||||
if (route.path === "/storage") {
|
||||
return <StoragePage navigate={navigate} />;
|
||||
}
|
||||
if (accountID) {
|
||||
return <AccountDetailPage id={Number(accountID)} navigate={navigate} />;
|
||||
}
|
||||
|
|
|
|||
118
cmd/telesrv-admin/web/src/pages/StoragePage.tsx
Normal file
118
cmd/telesrv-admin/web/src/pages/StoragePage.tsx
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
import { ChevronDown, Loader2, RefreshCw } from "lucide-react";
|
||||
import { useEffect, useState } from "react";
|
||||
import { api, errorMessage } from "../api";
|
||||
import { Alert, EmptyRow, Metric, PageFrame } from "../components/ui";
|
||||
import { displayUsername, formatBytes, formatQuantity } from "../lib/format";
|
||||
import type { Navigate } from "../routing";
|
||||
import type { AccountStorageRow, StorageStatsResponse } from "../types";
|
||||
|
||||
export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
|
||||
const [stats, setStats] = useState<StorageStatsResponse | null>(null);
|
||||
const [rows, setRows] = useState<AccountStorageRow[]>([]);
|
||||
const [hasMore, setHasMore] = useState(false);
|
||||
const [offset, setOffset] = useState(0);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState("");
|
||||
|
||||
async function loadStats() {
|
||||
try {
|
||||
setStats(await api.storageStats());
|
||||
} catch {
|
||||
// Stats are a header nicety; a failure here shouldn't block the list.
|
||||
}
|
||||
}
|
||||
|
||||
async function loadAccounts(next = false) {
|
||||
setBusy(true);
|
||||
setError("");
|
||||
const at = next ? offset : 0;
|
||||
const params = new URLSearchParams({ limit: "50", offset: String(at) });
|
||||
try {
|
||||
const result = await api.storageAccounts(params);
|
||||
const page = result.rows ?? [];
|
||||
setRows((current) => (next ? [...current, ...page] : page));
|
||||
setOffset(result.next_offset);
|
||||
setHasMore(Boolean(result.has_more));
|
||||
} catch (err) {
|
||||
setError(errorMessage(err));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
function refresh() {
|
||||
void loadStats();
|
||||
void loadAccounts(false);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
refresh();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []);
|
||||
|
||||
// Physical is what actually consumes disk/S3 (deduplicated); logical is the
|
||||
// sum of what the per-account table below adds up to. They legitimately
|
||||
// differ when the same content is shared by more than one document/photo.
|
||||
const dedupBytes = stats ? Math.max(0, Number(stats.LogicalBytes) - Number(stats.PhysicalBytes)) : 0;
|
||||
|
||||
return (
|
||||
<PageFrame
|
||||
title={"Storage"}
|
||||
eyebrow={"Media / Storage usage"}
|
||||
actions={
|
||||
<button className="btn icon-text" type="button" onClick={refresh} disabled={busy}>
|
||||
<RefreshCw size={15} className={busy ? "spin" : ""} /> {"Refresh"}
|
||||
</button>
|
||||
}
|
||||
>
|
||||
{error && <Alert>{error}</Alert>}
|
||||
<div className="metric-row">
|
||||
<Metric label={"Physical usage (on disk / S3)"} value={stats ? formatBytes(stats.PhysicalBytes) : "-"} />
|
||||
<Metric label={"Logical usage (sum per account)"} value={stats ? formatBytes(stats.LogicalBytes) : "-"} />
|
||||
<Metric label={"Saved by dedup"} value={formatBytes(String(dedupBytes))} tone={dedupBytes > 0 ? "good" : "neutral"} />
|
||||
<Metric label={"Backend"} value={stats?.BackendKind ?? "-"} />
|
||||
</div>
|
||||
<div className="metric-row">
|
||||
<Metric label={"Documents"} value={stats ? formatQuantity(stats.DocumentCount) : "-"} />
|
||||
<Metric label={"Photos"} value={stats ? formatQuantity(stats.PhotoCount) : "-"} />
|
||||
<Metric label={"Accounts with media"} value={stats ? formatQuantity(stats.AccountCount) : "-"} />
|
||||
<Metric
|
||||
label={"Unattributed"}
|
||||
value={stats ? formatBytes(stats.UnattributedBytes) : "-"}
|
||||
tone={stats && Number(stats.UnattributedBytes) > 0 ? "warn" : "neutral"}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="table-wrap">
|
||||
<table className="data-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>{"User ID"}</th>
|
||||
<th>{"Account"}</th>
|
||||
<th>{"Storage used"}</th>
|
||||
<th>{"Files"}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((row) => (
|
||||
<tr key={row.UserID}>
|
||||
<td className="mono">{row.UserID}</td>
|
||||
<td>{displayUsername(row.Username) || row.FirstName || "-"}</td>
|
||||
<td className="mono">{formatBytes(row.Bytes)}</td>
|
||||
<td className="mono">{formatQuantity(row.FileCount)}</td>
|
||||
</tr>
|
||||
))}
|
||||
{rows.length === 0 && <EmptyRow colSpan={4} />}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{hasMore && (
|
||||
<div className="toolbar">
|
||||
<button className="btn icon-text" type="button" onClick={() => loadAccounts(true)} disabled={busy}>
|
||||
{busy ? <Loader2 size={15} className="spin" /> : <ChevronDown size={15} />} {"Load more"}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</PageFrame>
|
||||
);
|
||||
}
|
||||
|
|
@ -21,6 +21,7 @@ export function routeTitle(pathname: string): string {
|
|||
if (pathname.startsWith("/verification")) return "Official Verification";
|
||||
if (pathname.startsWith("/collectible-usernames")) return "Collectible Usernames";
|
||||
if (pathname.startsWith("/account-ratings")) return "Account Rating";
|
||||
if (pathname.startsWith("/storage")) return "Storage";
|
||||
if (pathname.startsWith("/accounts")) return "Accounts";
|
||||
if (pathname.startsWith("/channels")) return "Supergroups and Channels";
|
||||
if (pathname.startsWith("/bots")) return "Bots";
|
||||
|
|
@ -39,6 +40,7 @@ export function routeSubtitle(pathname: string): string {
|
|||
if (pathname.startsWith("/verification")) return "Console / Verification";
|
||||
if (pathname.startsWith("/collectible-usernames")) return "Console / Collectible usernames";
|
||||
if (pathname.startsWith("/account-ratings")) return "Console / Account rating";
|
||||
if (pathname.startsWith("/storage")) return "Console / Storage";
|
||||
if (pathname.startsWith("/accounts")) return "Console / Accounts";
|
||||
if (pathname.startsWith("/channels")) return "Console / Channels";
|
||||
if (pathname.startsWith("/bots")) return "Console / Bots";
|
||||
|
|
|
|||
|
|
@ -789,6 +789,30 @@ export type AccountStatsResponse = {
|
|||
online: number;
|
||||
};
|
||||
|
||||
export type StorageStatsResponse = {
|
||||
PhysicalBytes: string;
|
||||
LogicalBytes: string;
|
||||
UnattributedBytes: string;
|
||||
DocumentCount: string;
|
||||
PhotoCount: string;
|
||||
AccountCount: string;
|
||||
BackendKind: string;
|
||||
};
|
||||
|
||||
export type AccountStorageRow = {
|
||||
UserID: string;
|
||||
Username: string;
|
||||
FirstName: string;
|
||||
Bytes: string;
|
||||
FileCount: string;
|
||||
};
|
||||
|
||||
export type AccountStorageListResponse = {
|
||||
rows: AccountStorageRow[] | null;
|
||||
has_more: boolean;
|
||||
next_offset: number;
|
||||
};
|
||||
|
||||
export type ChannelListResponse = {
|
||||
query: string;
|
||||
limit: number;
|
||||
|
|
|
|||
|
|
@ -710,14 +710,36 @@ func run(logger *zap.Logger) error {
|
|||
cachedPhotos := userprojection.NewCachedPhotoProvider(mediaStore, userprojection.DefaultPhotoCacheTTL)
|
||||
privacyStore := privacyapp.NewCachedPrivacyStore(postgres.NewPrivacyStore(pool), 0)
|
||||
storyStore := postgres.NewStoryStore(pool)
|
||||
blobBackend, err := filesapp.NewLocalFS(cfg.BlobDir)
|
||||
// Transient upload-part scratch storage always stays on local disk
|
||||
// (TELESRV_BLOB_DIR) regardless of the permanent blob backend below --
|
||||
// one S3 round trip per ~512KB chunk isn't worth it for data deleted
|
||||
// within minutes of assembly.
|
||||
localBlobFS, err := filesapp.NewLocalFS(cfg.BlobDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init blob backend: %w", err)
|
||||
return fmt.Errorf("init local blob dir: %w", err)
|
||||
}
|
||||
var blobBackend filesapp.BlobBackend = localBlobFS
|
||||
var spaceGuard filesapp.SpaceGuard = filesapp.NoopSpaceGuard{}
|
||||
if cfg.BlobBackendKind == "s3" {
|
||||
s3Backend, err := filesapp.NewS3FS(ctx, cfg.S3Endpoint, cfg.S3AccessKeyID, cfg.S3SecretAccessKey, cfg.S3Bucket, cfg.S3Region, cfg.S3UseSSL, cfg.S3PathStyle)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init s3 blob backend: %w", err)
|
||||
}
|
||||
blobBackend = s3Backend
|
||||
logger.Info("blob backend ready", zap.String("backend", "s3"), zap.String("bucket", cfg.S3Bucket), zap.String("endpoint", cfg.S3Endpoint))
|
||||
if cfg.StorageLowSpaceGuardEnable && cfg.StorageMaxTotalBytes > 0 {
|
||||
s3Guard := filesapp.NewS3BudgetSpaceGuard(cfg.StorageMaxTotalBytes)
|
||||
spaceGuard = s3Guard
|
||||
go filesapp.NewS3DiskUsageWorker(s3Guard, mediaStore, cfg.StorageUsageRefreshInterval, logger.Named("files").Named("diskusage")).Run(ctx)
|
||||
}
|
||||
} else {
|
||||
logger.Info("blob backend ready", zap.String("backend", "localfs"), zap.String("dir", cfg.BlobDir))
|
||||
if cfg.StorageLowSpaceGuardEnable && cfg.StorageMinFreeBytes > 0 {
|
||||
localGuard := filesapp.NewLocalDiskSpaceGuard(cfg.StorageMinFreeBytes)
|
||||
spaceGuard = localGuard
|
||||
go filesapp.NewLocalDiskUsageWorker(localGuard, cfg.BlobDir, cfg.StorageUsageRefreshInterval, logger.Named("files").Named("diskusage")).Run(ctx)
|
||||
}
|
||||
}
|
||||
logger.Info("blob backend ready",
|
||||
zap.String("backend", "localfs"),
|
||||
zap.String("dir", cfg.BlobDir),
|
||||
)
|
||||
filesService := filesapp.NewService(mediaStore, blobBackend, cfg.DC,
|
||||
filesapp.WithLogger(logger),
|
||||
filesapp.WithUploadPartQuota(domain.UploadPartQuota{
|
||||
|
|
@ -725,6 +747,8 @@ func run(logger *zap.Logger) error {
|
|||
MaxParts: cfg.UploadInFlightMaxParts,
|
||||
MaxFiles: cfg.UploadInFlightMaxFiles,
|
||||
}),
|
||||
filesapp.WithUploadPartBackend(localBlobFS),
|
||||
filesapp.WithSpaceGuard(spaceGuard),
|
||||
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
|
||||
externalMediaOption(cfg),
|
||||
webPagePreviewOption(cfg),
|
||||
|
|
@ -822,6 +846,10 @@ func run(logger *zap.Logger) error {
|
|||
Restrictions: adminStore,
|
||||
OfficialGifts: officialgifts.New(cfg.OfficialGiftsDir),
|
||||
})
|
||||
storageRetentionMaxAge := cfg.StorageRetentionMaxAge
|
||||
if !cfg.StorageRetentionEnable {
|
||||
storageRetentionMaxAge = 0
|
||||
}
|
||||
go maintenance.NewRetentionWorker(dispatchOutboxStore, tempAuthKeyStore, logger.Named("maintenance").Named("retention"),
|
||||
cfg.UpdateEventRetention,
|
||||
cfg.RetentionInterval,
|
||||
|
|
@ -836,6 +864,7 @@ func run(logger *zap.Logger) error {
|
|||
WithUserUpdateRetention(updateEventStore).
|
||||
WithChannelUpdateRetention(channelStore).
|
||||
WithOrphanAuthKeyRetention(authKeyStore, activeSessions, cfg.OrphanAuthKeyRetention).
|
||||
WithOrphanedMediaRetention(filesService, storageRetentionMaxAge).
|
||||
Run(ctx)
|
||||
go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"),
|
||||
cfg.UploadPartTTL,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue