s3 support
This commit is contained in:
parent
fa5cfaf14d
commit
03f10b66ee
53 changed files with 2796 additions and 102 deletions
44
.env.example
44
.env.example
|
|
@ -239,6 +239,33 @@ TELESRV_BOT_VERIFICATION_ENABLED=true
|
|||
# weights and recompute timing live in the Advanced section below.
|
||||
TELESRV_RATING_ENABLED=true
|
||||
|
||||
## Storage & Media -- Where uploaded media is stored, and how the server reacts to running low on space.
|
||||
|
||||
# Where uploaded media (photos, documents, stickers) is physically stored.
|
||||
# "localfs" writes to TELESRV_BLOB_DIR on this machine's disk. "s3" writes to
|
||||
# an S3-compatible object store -- self-hosted MinIO or AWS S3 -- configured
|
||||
# below. Switching only affects new uploads; existing files stay wherever
|
||||
# they were written and remain readable.
|
||||
TELESRV_BLOB_BACKEND=localfs
|
||||
# Only used when TELESRV_BLOB_BACKEND=s3.
|
||||
TELESRV_S3_ENDPOINT=
|
||||
TELESRV_S3_REGION=us-east-1
|
||||
TELESRV_S3_BUCKET=
|
||||
TELESRV_S3_ACCESS_KEY_ID=
|
||||
TELESRV_S3_SECRET_ACCESS_KEY=
|
||||
TELESRV_S3_USE_SSL=true
|
||||
# MinIO typically needs this on (bucket in the URL path); AWS S3 does not.
|
||||
TELESRV_S3_PATH_STYLE=false
|
||||
# Reject new uploads once storage is nearly full, instead of letting the disk
|
||||
# fill up. Thresholds live in the Advanced section below.
|
||||
TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE=true
|
||||
# Automatically delete old media once it's no longer referenced by any
|
||||
# message, profile photo, or sticker set (never deletes media still visible
|
||||
# in a conversation). Off by default -- storage usage is tracked and shown
|
||||
# in the admin panel either way; enable this once you're comfortable with
|
||||
# what it will reclaim. Retention age lives in the Advanced section below.
|
||||
TELESRV_STORAGE_RETENTION_ENABLE=false
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Advanced / internal tuning
|
||||
|
|
@ -367,6 +394,23 @@ TELESRV_STICKER_SEED_MAX_SETS=300
|
|||
# Sticker set auto-installed for every newly registered account; <=0 disables this.
|
||||
TELESRV_DEFAULT_STICKER_SET_ID=0
|
||||
|
||||
# Storage low-space guard thresholds (master toggle is TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE above).
|
||||
# localfs: reject new uploads once real free disk bytes fall below this; <=0 disables.
|
||||
TELESRV_STORAGE_MIN_FREE_BYTES=1073741824
|
||||
# Reject new uploads once total tracked blob bytes would exceed this. The only
|
||||
# meaningful "low space" signal on the s3 backend (no OS free-space concept);
|
||||
# optional soft cap on localfs too. <=0 disables.
|
||||
TELESRV_STORAGE_MAX_TOTAL_BYTES=0
|
||||
# How often the cached free-space/usage gauge behind the guard above refreshes.
|
||||
TELESRV_STORAGE_USAGE_REFRESH_INTERVAL=1m
|
||||
# Storage retention sweep tuning (master toggle is TELESRV_STORAGE_RETENTION_ENABLE above).
|
||||
# How long a document/photo must have had zero references before the sweep
|
||||
# deletes it -- not how old the media itself is, and it never touches media
|
||||
# still referenced by a live message/profile-photo/sticker-set. The sweep
|
||||
# itself runs alongside every other retention check on the shared
|
||||
# TELESRV_RETENTION_INTERVAL/TELESRV_RETENTION_BATCH cadence above.
|
||||
TELESRV_STORAGE_RETENTION_MAX_AGE=720h
|
||||
|
||||
# New-account perks: free Telegram Premium months and starting Stars balance.
|
||||
TELESRV_PREMIUM_GRANT_MONTHS=3
|
||||
TELESRV_STARS_STARTING_GRANT=1000
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -2541,3 +2542,133 @@ func clampBotVerificationLimit(limit int) int {
|
|||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
// storageUsageListDefaultLimit/storageUsageListMaxLimit bound the per-account
|
||||
// storage breakdown page, same convention as the account rating list above.
|
||||
const (
|
||||
storageUsageListDefaultLimit = 50
|
||||
storageUsageListMaxLimit = 200
|
||||
)
|
||||
|
||||
// perOwnerMediaSizeSQL is shared between StorageStats and
|
||||
// ListAccountStorageUsage: each document row's size is its stored column;
|
||||
// each photo row has no single size column (JSONB sizes holds one entry per
|
||||
// rendition), so its "attributed size" is the largest rendition -- the
|
||||
// dominant cost, thumbnails are comparatively tiny. This is an
|
||||
// approximation (not the exact sum of every rendition's blob bytes, which
|
||||
// would require joining file_blobs by location_key prefix) chosen for admin
|
||||
// visibility, not billing precision.
|
||||
const perOwnerMediaSizeSQL = `
|
||||
SELECT owner_user_id, size FROM documents
|
||||
UNION ALL
|
||||
SELECT p.owner_user_id, COALESCE((
|
||||
SELECT MAX((elem->>'size')::bigint) FROM jsonb_array_elements(p.sizes) elem
|
||||
), 0) AS size
|
||||
FROM photos p
|
||||
`
|
||||
|
||||
// StorageStatsRow is the admin panel's storage overview: physical bytes
|
||||
// (from file_blobs, backend-dedup-aware -- what's actually consuming disk
|
||||
// or S3) versus logical bytes (sum of the same approximate per-row
|
||||
// attribution the per-account breakdown uses, which can legitimately be
|
||||
// higher than physical when identical content is shared by more than one
|
||||
// document/photo).
|
||||
type StorageStatsRow struct {
|
||||
PhysicalBytes int64 `json:"PhysicalBytes,string"`
|
||||
LogicalBytes int64 `json:"LogicalBytes,string"`
|
||||
UnattributedBytes int64 `json:"UnattributedBytes,string"`
|
||||
DocumentCount int64 `json:"DocumentCount,string"`
|
||||
PhotoCount int64 `json:"PhotoCount,string"`
|
||||
AccountCount int64 `json:"AccountCount,string"`
|
||||
BackendKind string
|
||||
}
|
||||
|
||||
// StorageStats returns the admin panel's storage overview.
|
||||
func (s *readStore) StorageStats(ctx context.Context) (StorageStatsRow, error) {
|
||||
var stats StorageStatsRow
|
||||
if err := s.pool.QueryRow(ctx, `SELECT COALESCE(SUM(size), 0)::bigint FROM file_blobs`).Scan(&stats.PhysicalBytes); err != nil {
|
||||
return StorageStatsRow{}, fmt.Errorf("sum physical blob bytes: %w", err)
|
||||
}
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x`).Scan(&stats.LogicalBytes); err != nil {
|
||||
return StorageStatsRow{}, fmt.Errorf("sum logical media bytes: %w", err)
|
||||
}
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(SUM(size), 0)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id = 0`).Scan(&stats.UnattributedBytes); err != nil {
|
||||
return StorageStatsRow{}, fmt.Errorf("sum unattributed media bytes: %w", err)
|
||||
}
|
||||
if err := s.pool.QueryRow(ctx, `SELECT count(*)::bigint FROM documents`).Scan(&stats.DocumentCount); err != nil {
|
||||
return StorageStatsRow{}, fmt.Errorf("count documents: %w", err)
|
||||
}
|
||||
if err := s.pool.QueryRow(ctx, `SELECT count(*)::bigint FROM photos`).Scan(&stats.PhotoCount); err != nil {
|
||||
return StorageStatsRow{}, fmt.Errorf("count photos: %w", err)
|
||||
}
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
SELECT count(DISTINCT owner_user_id)::bigint FROM (`+perOwnerMediaSizeSQL+`) x WHERE owner_user_id <> 0`).Scan(&stats.AccountCount); err != nil {
|
||||
return StorageStatsRow{}, fmt.Errorf("count storage accounts: %w", err)
|
||||
}
|
||||
stats.BackendKind = strings.ToLower(strings.TrimSpace(os.Getenv("TELESRV_BLOB_BACKEND")))
|
||||
if stats.BackendKind == "" {
|
||||
stats.BackendKind = "localfs"
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// AccountStorageRow is one account's row in the per-account storage
|
||||
// breakdown table.
|
||||
type AccountStorageRow struct {
|
||||
UserID int64 `json:"UserID,string"`
|
||||
Username string
|
||||
FirstName string
|
||||
Bytes int64 `json:"Bytes,string"`
|
||||
FileCount int64 `json:"FileCount,string"`
|
||||
}
|
||||
|
||||
// ListAccountStorageUsage pages the per-account storage breakdown, largest
|
||||
// user first. Offset-based (not keyset): storage administration on a
|
||||
// self-hosted deployment doesn't need to support arbitrarily deep pages
|
||||
// efficiently the way an infinite-scroll feed does.
|
||||
func (s *readStore) ListAccountStorageUsage(ctx context.Context, offset, limit int) ([]AccountStorageRow, bool, error) {
|
||||
if limit <= 0 {
|
||||
limit = storageUsageListDefaultLimit
|
||||
}
|
||||
if limit > storageUsageListMaxLimit {
|
||||
limit = storageUsageListMaxLimit
|
||||
}
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
WITH totals AS (
|
||||
SELECT owner_user_id, SUM(size)::bigint AS bytes, COUNT(*)::bigint AS file_count
|
||||
FROM (`+perOwnerMediaSizeSQL+`) x
|
||||
WHERE owner_user_id <> 0
|
||||
GROUP BY owner_user_id
|
||||
)
|
||||
SELECT t.owner_user_id, COALESCE(u.username, ''), COALESCE(u.first_name, ''), t.bytes, t.file_count
|
||||
FROM totals t
|
||||
LEFT JOIN users u ON u.id = t.owner_user_id
|
||||
ORDER BY t.bytes DESC, t.owner_user_id
|
||||
OFFSET $1
|
||||
LIMIT $2`, offset, limit+1)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("list account storage usage: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]AccountStorageRow, 0, limit+1)
|
||||
for rows.Next() {
|
||||
var item AccountStorageRow
|
||||
if err := rows.Scan(&item.UserID, &item.Username, &item.FirstName, &item.Bytes, &item.FileCount); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
hasMore := len(out) > limit
|
||||
if hasMore {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out, hasMore, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,8 @@ func (s *server) routes() http.Handler {
|
|||
mux.Handle("GET /api/collectible-usernames/{id}", s.requireAuthAPI(http.HandlerFunc(s.handleCollectibleUsernameDetailAPI)))
|
||||
mux.Handle("GET /api/account-ratings", s.requireAuthAPI(http.HandlerFunc(s.handleAccountRatingsAPI)))
|
||||
mux.Handle("GET /api/account-ratings/{user_id}", s.requireAuthAPI(http.HandlerFunc(s.handleAccountRatingDetailAPI)))
|
||||
mux.Handle("GET /api/storage/stats", s.requireAuthAPI(http.HandlerFunc(s.handleStorageStatsAPI)))
|
||||
mux.Handle("GET /api/storage/accounts", s.requireAuthAPI(http.HandlerFunc(s.handleStorageAccountsAPI)))
|
||||
mux.Handle("GET /api/moderation/cases", s.requireAuthAPI(http.HandlerFunc(s.handleModerationCasesAPI)))
|
||||
mux.Handle("GET /api/moderation/cases/{id}", s.requireAuthAPI(http.HandlerFunc(s.handleModerationCaseAPI)))
|
||||
mux.Handle("GET /api/moderation/reports/{id}", s.requireAuthAPI(http.HandlerFunc(s.handleModerationReportAPI)))
|
||||
|
|
@ -2342,6 +2344,53 @@ func (s *server) handleAccountRatingsAPI(w http.ResponseWriter, r *http.Request)
|
|||
})
|
||||
}
|
||||
|
||||
func (s *server) handleStorageStatsAPI(w http.ResponseWriter, r *http.Request) {
|
||||
if s.read == nil {
|
||||
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")
|
||||
return
|
||||
}
|
||||
stats, err := s.read.StorageStats(r.Context())
|
||||
if err != nil {
|
||||
writeAPIError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, stats)
|
||||
}
|
||||
|
||||
// handleStorageAccountsAPI pages the per-account storage breakdown,
|
||||
// largest first.
|
||||
func (s *server) handleStorageAccountsAPI(w http.ResponseWriter, r *http.Request) {
|
||||
if s.read == nil {
|
||||
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")
|
||||
return
|
||||
}
|
||||
query := r.URL.Query()
|
||||
offset, err := parseInt(query.Get("offset"))
|
||||
if err != nil || offset < 0 {
|
||||
writeAPIError(w, http.StatusBadRequest, "invalid offset")
|
||||
return
|
||||
}
|
||||
limit, err := parseInt(query.Get("limit"))
|
||||
if err != nil || limit < 0 {
|
||||
writeAPIError(w, http.StatusBadRequest, "invalid limit")
|
||||
return
|
||||
}
|
||||
rows, hasMore, err := s.read.ListAccountStorageUsage(r.Context(), offset, limit)
|
||||
if err != nil {
|
||||
writeAPIError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
nextOffset := 0
|
||||
if hasMore {
|
||||
nextOffset = offset + len(rows)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"rows": rows,
|
||||
"has_more": hasMore,
|
||||
"next_offset": nextOffset,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *server) handleAccountRatingDetailAPI(w http.ResponseWriter, r *http.Request) {
|
||||
if s.read == nil {
|
||||
writeAPIError(w, http.StatusServiceUnavailable, "read store is not configured")
|
||||
|
|
|
|||
10
cmd/telesrv-admin/web/dist/assets/index-CmjlUwWc.js
vendored
Normal file
10
cmd/telesrv-admin/web/dist/assets/index-CmjlUwWc.js
vendored
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
2
cmd/telesrv-admin/web/dist/index.html
vendored
2
cmd/telesrv-admin/web/dist/index.html
vendored
|
|
@ -23,7 +23,7 @@
|
|||
})();
|
||||
</script>
|
||||
|
||||
<script type="module" crossorigin src="/assets/index-D0qBFBU-.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index-CmjlUwWc.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-EkAGiEK9.css">
|
||||
</head>
|
||||
<body>
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ import type {
|
|||
AccountRatingDetail,
|
||||
AccountRatingListResponse,
|
||||
AccountStatsResponse,
|
||||
AccountStorageListResponse,
|
||||
StorageStatsResponse,
|
||||
AdminLoginResult,
|
||||
AdminSession,
|
||||
BotDetail,
|
||||
|
|
@ -165,6 +167,9 @@ export const api = {
|
|||
request<AccountRatingListResponse>(`/api/account-ratings?${params.toString()}`),
|
||||
accountRating: (userID: string) =>
|
||||
request<AccountRatingDetail>(`/api/account-ratings/${encodeURIComponent(userID)}`),
|
||||
storageStats: () => request<StorageStatsResponse>("/api/storage/stats"),
|
||||
storageAccounts: (params: URLSearchParams) =>
|
||||
request<AccountStorageListResponse>(`/api/storage/accounts?${params.toString()}`),
|
||||
verificationApplications: (params: URLSearchParams) =>
|
||||
request<VerificationApplicationListResponse>(`/api/verification/applications?${params.toString()}`),
|
||||
// The application id is an int64 decimal string end to end, so it is never
|
||||
|
|
|
|||
|
|
@ -99,6 +99,7 @@ export function Shell({
|
|||
)}
|
||||
<NavLink icon={<AtSign size={16} />} href="/collectible-usernames" route={route} navigate={navigate}>{"NFT Usernames"}</NavLink>
|
||||
<NavLink icon={<Trophy size={16} />} href="/account-ratings" route={route} navigate={navigate}>{"Account Rating"}</NavLink>
|
||||
<NavLink icon={<Database size={16} />} href="/storage" route={route} navigate={navigate}>{"Storage"}</NavLink>
|
||||
<NavLink icon={<Gift size={16} />} href="/gifts" route={route} navigate={navigate}>{"Star Gifts"}</NavLink>
|
||||
<NavLink icon={<Send size={16} />} href="/give-gifts" route={route} navigate={navigate}>{"Give Gifts"}</NavLink>
|
||||
<NavLink icon={<Sticker size={16} />} href="/stickers" route={route} navigate={navigate}>{"Stickers"}</NavLink>
|
||||
|
|
|
|||
|
|
@ -154,6 +154,26 @@ export function toSmallestUnits(value: string, currency: string): string | null
|
|||
return digits === "" ? "0" : digits;
|
||||
}
|
||||
|
||||
// formatBytes renders a byte count (as the JSON-string int64 the API sends)
|
||||
// in the largest unit that keeps it readable. Parses the decimal string
|
||||
// directly rather than through toNumeric first so precision past
|
||||
// Number.MAX_SAFE_INTEGER isn't silently lost before the division.
|
||||
export function formatBytes(value: string): string {
|
||||
const raw = (value ?? "").trim();
|
||||
if (!raw || !/^\d+$/.test(raw)) return "0 B";
|
||||
const bytes = Number(raw);
|
||||
if (!Number.isFinite(bytes)) return `${raw} B`;
|
||||
const units = ["B", "KB", "MB", "GB", "TB", "PB"];
|
||||
let size = bytes;
|
||||
let unit = 0;
|
||||
while (size >= 1024 && unit < units.length - 1) {
|
||||
size /= 1024;
|
||||
unit++;
|
||||
}
|
||||
const precision = unit === 0 ? 0 : 1;
|
||||
return `${size.toFixed(precision)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
export function formatSigned(value: string): string {
|
||||
const raw = (value ?? "").trim();
|
||||
if (!raw) return "0";
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import { StickerSetsPage } from "./StickerSetsPage";
|
|||
import { GiveGiftsPage } from "./GiveGiftsPage";
|
||||
import { ModerationCaseDetailPage } from "./ModerationCaseDetailPage";
|
||||
import { ModerationCasesPage } from "./ModerationCasesPage";
|
||||
import { StoragePage } from "./StoragePage";
|
||||
import { BotVerificationPage } from "./BotVerificationPage";
|
||||
import { BotVerificationRequestPage } from "./BotVerificationRequestPage";
|
||||
import { VerificationDetailPage } from "./VerificationDetailPage";
|
||||
|
|
@ -84,6 +85,9 @@ export function Routes({ route, navigate }: { route: RouteState; navigate: Navig
|
|||
if (route.path === "/account-ratings") {
|
||||
return <AccountRatingsPage navigate={navigate} />;
|
||||
}
|
||||
if (route.path === "/storage") {
|
||||
return <StoragePage navigate={navigate} />;
|
||||
}
|
||||
if (accountID) {
|
||||
return <AccountDetailPage id={Number(accountID)} navigate={navigate} />;
|
||||
}
|
||||
|
|
|
|||
118
cmd/telesrv-admin/web/src/pages/StoragePage.tsx
Normal file
118
cmd/telesrv-admin/web/src/pages/StoragePage.tsx
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
import { ChevronDown, Loader2, RefreshCw } from "lucide-react";
|
||||
import { useEffect, useState } from "react";
|
||||
import { api, errorMessage } from "../api";
|
||||
import { Alert, EmptyRow, Metric, PageFrame } from "../components/ui";
|
||||
import { displayUsername, formatBytes, formatQuantity } from "../lib/format";
|
||||
import type { Navigate } from "../routing";
|
||||
import type { AccountStorageRow, StorageStatsResponse } from "../types";
|
||||
|
||||
export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
|
||||
const [stats, setStats] = useState<StorageStatsResponse | null>(null);
|
||||
const [rows, setRows] = useState<AccountStorageRow[]>([]);
|
||||
const [hasMore, setHasMore] = useState(false);
|
||||
const [offset, setOffset] = useState(0);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState("");
|
||||
|
||||
async function loadStats() {
|
||||
try {
|
||||
setStats(await api.storageStats());
|
||||
} catch {
|
||||
// Stats are a header nicety; a failure here shouldn't block the list.
|
||||
}
|
||||
}
|
||||
|
||||
async function loadAccounts(next = false) {
|
||||
setBusy(true);
|
||||
setError("");
|
||||
const at = next ? offset : 0;
|
||||
const params = new URLSearchParams({ limit: "50", offset: String(at) });
|
||||
try {
|
||||
const result = await api.storageAccounts(params);
|
||||
const page = result.rows ?? [];
|
||||
setRows((current) => (next ? [...current, ...page] : page));
|
||||
setOffset(result.next_offset);
|
||||
setHasMore(Boolean(result.has_more));
|
||||
} catch (err) {
|
||||
setError(errorMessage(err));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
function refresh() {
|
||||
void loadStats();
|
||||
void loadAccounts(false);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
refresh();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []);
|
||||
|
||||
// Physical is what actually consumes disk/S3 (deduplicated); logical is the
|
||||
// sum of what the per-account table below adds up to. They legitimately
|
||||
// differ when the same content is shared by more than one document/photo.
|
||||
const dedupBytes = stats ? Math.max(0, Number(stats.LogicalBytes) - Number(stats.PhysicalBytes)) : 0;
|
||||
|
||||
return (
|
||||
<PageFrame
|
||||
title={"Storage"}
|
||||
eyebrow={"Media / Storage usage"}
|
||||
actions={
|
||||
<button className="btn icon-text" type="button" onClick={refresh} disabled={busy}>
|
||||
<RefreshCw size={15} className={busy ? "spin" : ""} /> {"Refresh"}
|
||||
</button>
|
||||
}
|
||||
>
|
||||
{error && <Alert>{error}</Alert>}
|
||||
<div className="metric-row">
|
||||
<Metric label={"Physical usage (on disk / S3)"} value={stats ? formatBytes(stats.PhysicalBytes) : "-"} />
|
||||
<Metric label={"Logical usage (sum per account)"} value={stats ? formatBytes(stats.LogicalBytes) : "-"} />
|
||||
<Metric label={"Saved by dedup"} value={formatBytes(String(dedupBytes))} tone={dedupBytes > 0 ? "good" : "neutral"} />
|
||||
<Metric label={"Backend"} value={stats?.BackendKind ?? "-"} />
|
||||
</div>
|
||||
<div className="metric-row">
|
||||
<Metric label={"Documents"} value={stats ? formatQuantity(stats.DocumentCount) : "-"} />
|
||||
<Metric label={"Photos"} value={stats ? formatQuantity(stats.PhotoCount) : "-"} />
|
||||
<Metric label={"Accounts with media"} value={stats ? formatQuantity(stats.AccountCount) : "-"} />
|
||||
<Metric
|
||||
label={"Unattributed"}
|
||||
value={stats ? formatBytes(stats.UnattributedBytes) : "-"}
|
||||
tone={stats && Number(stats.UnattributedBytes) > 0 ? "warn" : "neutral"}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="table-wrap">
|
||||
<table className="data-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>{"User ID"}</th>
|
||||
<th>{"Account"}</th>
|
||||
<th>{"Storage used"}</th>
|
||||
<th>{"Files"}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((row) => (
|
||||
<tr key={row.UserID}>
|
||||
<td className="mono">{row.UserID}</td>
|
||||
<td>{displayUsername(row.Username) || row.FirstName || "-"}</td>
|
||||
<td className="mono">{formatBytes(row.Bytes)}</td>
|
||||
<td className="mono">{formatQuantity(row.FileCount)}</td>
|
||||
</tr>
|
||||
))}
|
||||
{rows.length === 0 && <EmptyRow colSpan={4} />}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{hasMore && (
|
||||
<div className="toolbar">
|
||||
<button className="btn icon-text" type="button" onClick={() => loadAccounts(true)} disabled={busy}>
|
||||
{busy ? <Loader2 size={15} className="spin" /> : <ChevronDown size={15} />} {"Load more"}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</PageFrame>
|
||||
);
|
||||
}
|
||||
|
|
@ -21,6 +21,7 @@ export function routeTitle(pathname: string): string {
|
|||
if (pathname.startsWith("/verification")) return "Official Verification";
|
||||
if (pathname.startsWith("/collectible-usernames")) return "Collectible Usernames";
|
||||
if (pathname.startsWith("/account-ratings")) return "Account Rating";
|
||||
if (pathname.startsWith("/storage")) return "Storage";
|
||||
if (pathname.startsWith("/accounts")) return "Accounts";
|
||||
if (pathname.startsWith("/channels")) return "Supergroups and Channels";
|
||||
if (pathname.startsWith("/bots")) return "Bots";
|
||||
|
|
@ -39,6 +40,7 @@ export function routeSubtitle(pathname: string): string {
|
|||
if (pathname.startsWith("/verification")) return "Console / Verification";
|
||||
if (pathname.startsWith("/collectible-usernames")) return "Console / Collectible usernames";
|
||||
if (pathname.startsWith("/account-ratings")) return "Console / Account rating";
|
||||
if (pathname.startsWith("/storage")) return "Console / Storage";
|
||||
if (pathname.startsWith("/accounts")) return "Console / Accounts";
|
||||
if (pathname.startsWith("/channels")) return "Console / Channels";
|
||||
if (pathname.startsWith("/bots")) return "Console / Bots";
|
||||
|
|
|
|||
|
|
@ -789,6 +789,30 @@ export type AccountStatsResponse = {
|
|||
online: number;
|
||||
};
|
||||
|
||||
export type StorageStatsResponse = {
|
||||
PhysicalBytes: string;
|
||||
LogicalBytes: string;
|
||||
UnattributedBytes: string;
|
||||
DocumentCount: string;
|
||||
PhotoCount: string;
|
||||
AccountCount: string;
|
||||
BackendKind: string;
|
||||
};
|
||||
|
||||
export type AccountStorageRow = {
|
||||
UserID: string;
|
||||
Username: string;
|
||||
FirstName: string;
|
||||
Bytes: string;
|
||||
FileCount: string;
|
||||
};
|
||||
|
||||
export type AccountStorageListResponse = {
|
||||
rows: AccountStorageRow[] | null;
|
||||
has_more: boolean;
|
||||
next_offset: number;
|
||||
};
|
||||
|
||||
export type ChannelListResponse = {
|
||||
query: string;
|
||||
limit: number;
|
||||
|
|
|
|||
|
|
@ -710,14 +710,36 @@ func run(logger *zap.Logger) error {
|
|||
cachedPhotos := userprojection.NewCachedPhotoProvider(mediaStore, userprojection.DefaultPhotoCacheTTL)
|
||||
privacyStore := privacyapp.NewCachedPrivacyStore(postgres.NewPrivacyStore(pool), 0)
|
||||
storyStore := postgres.NewStoryStore(pool)
|
||||
blobBackend, err := filesapp.NewLocalFS(cfg.BlobDir)
|
||||
// Transient upload-part scratch storage always stays on local disk
|
||||
// (TELESRV_BLOB_DIR) regardless of the permanent blob backend below --
|
||||
// one S3 round trip per ~512KB chunk isn't worth it for data deleted
|
||||
// within minutes of assembly.
|
||||
localBlobFS, err := filesapp.NewLocalFS(cfg.BlobDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init blob backend: %w", err)
|
||||
return fmt.Errorf("init local blob dir: %w", err)
|
||||
}
|
||||
var blobBackend filesapp.BlobBackend = localBlobFS
|
||||
var spaceGuard filesapp.SpaceGuard = filesapp.NoopSpaceGuard{}
|
||||
if cfg.BlobBackendKind == "s3" {
|
||||
s3Backend, err := filesapp.NewS3FS(ctx, cfg.S3Endpoint, cfg.S3AccessKeyID, cfg.S3SecretAccessKey, cfg.S3Bucket, cfg.S3Region, cfg.S3UseSSL, cfg.S3PathStyle)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init s3 blob backend: %w", err)
|
||||
}
|
||||
blobBackend = s3Backend
|
||||
logger.Info("blob backend ready", zap.String("backend", "s3"), zap.String("bucket", cfg.S3Bucket), zap.String("endpoint", cfg.S3Endpoint))
|
||||
if cfg.StorageLowSpaceGuardEnable && cfg.StorageMaxTotalBytes > 0 {
|
||||
s3Guard := filesapp.NewS3BudgetSpaceGuard(cfg.StorageMaxTotalBytes)
|
||||
spaceGuard = s3Guard
|
||||
go filesapp.NewS3DiskUsageWorker(s3Guard, mediaStore, cfg.StorageUsageRefreshInterval, logger.Named("files").Named("diskusage")).Run(ctx)
|
||||
}
|
||||
} else {
|
||||
logger.Info("blob backend ready", zap.String("backend", "localfs"), zap.String("dir", cfg.BlobDir))
|
||||
if cfg.StorageLowSpaceGuardEnable && cfg.StorageMinFreeBytes > 0 {
|
||||
localGuard := filesapp.NewLocalDiskSpaceGuard(cfg.StorageMinFreeBytes)
|
||||
spaceGuard = localGuard
|
||||
go filesapp.NewLocalDiskUsageWorker(localGuard, cfg.BlobDir, cfg.StorageUsageRefreshInterval, logger.Named("files").Named("diskusage")).Run(ctx)
|
||||
}
|
||||
}
|
||||
logger.Info("blob backend ready",
|
||||
zap.String("backend", "localfs"),
|
||||
zap.String("dir", cfg.BlobDir),
|
||||
)
|
||||
filesService := filesapp.NewService(mediaStore, blobBackend, cfg.DC,
|
||||
filesapp.WithLogger(logger),
|
||||
filesapp.WithUploadPartQuota(domain.UploadPartQuota{
|
||||
|
|
@ -725,6 +747,8 @@ func run(logger *zap.Logger) error {
|
|||
MaxParts: cfg.UploadInFlightMaxParts,
|
||||
MaxFiles: cfg.UploadInFlightMaxFiles,
|
||||
}),
|
||||
filesapp.WithUploadPartBackend(localBlobFS),
|
||||
filesapp.WithSpaceGuard(spaceGuard),
|
||||
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
|
||||
externalMediaOption(cfg),
|
||||
webPagePreviewOption(cfg),
|
||||
|
|
@ -822,6 +846,10 @@ func run(logger *zap.Logger) error {
|
|||
Restrictions: adminStore,
|
||||
OfficialGifts: officialgifts.New(cfg.OfficialGiftsDir),
|
||||
})
|
||||
storageRetentionMaxAge := cfg.StorageRetentionMaxAge
|
||||
if !cfg.StorageRetentionEnable {
|
||||
storageRetentionMaxAge = 0
|
||||
}
|
||||
go maintenance.NewRetentionWorker(dispatchOutboxStore, tempAuthKeyStore, logger.Named("maintenance").Named("retention"),
|
||||
cfg.UpdateEventRetention,
|
||||
cfg.RetentionInterval,
|
||||
|
|
@ -836,6 +864,7 @@ func run(logger *zap.Logger) error {
|
|||
WithUserUpdateRetention(updateEventStore).
|
||||
WithChannelUpdateRetention(channelStore).
|
||||
WithOrphanAuthKeyRetention(authKeyStore, activeSessions, cfg.OrphanAuthKeyRetention).
|
||||
WithOrphanedMediaRetention(filesService, storageRetentionMaxAge).
|
||||
Run(ctx)
|
||||
go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"),
|
||||
cfg.UploadPartTTL,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
DROP INDEX IF EXISTS idx_photos_orphaned_at;
|
||||
DROP INDEX IF EXISTS idx_documents_orphaned_at;
|
||||
ALTER TABLE public.photos DROP COLUMN IF EXISTS orphaned_at;
|
||||
ALTER TABLE public.documents DROP COLUMN IF EXISTS orphaned_at;
|
||||
|
||||
DROP TABLE IF EXISTS public.media_references;
|
||||
|
||||
DROP INDEX IF EXISTS idx_photos_owner_user_id;
|
||||
DROP INDEX IF EXISTS idx_documents_owner_user_id;
|
||||
ALTER TABLE public.photos DROP COLUMN IF EXISTS owner_user_id;
|
||||
ALTER TABLE public.documents DROP COLUMN IF EXISTS owner_user_id;
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
-- Storage management: track who uploaded each document/photo, and track
|
||||
-- every live reference to it (message boxes, channel messages, profile
|
||||
-- photos, sticker material, gifts) so orphaned media (nothing left pointing
|
||||
-- at it) can be identified safely for later cleanup. A media row's
|
||||
-- orphaned_at is set the instant its last reference is removed, and cleared
|
||||
-- if a new reference appears -- retention sweeps only ever act on rows where
|
||||
-- orphaned_at is set, so media still visible in a live conversation is never
|
||||
-- touched regardless of age.
|
||||
|
||||
ALTER TABLE public.documents ADD COLUMN owner_user_id bigint DEFAULT 0 NOT NULL;
|
||||
ALTER TABLE public.photos ADD COLUMN owner_user_id bigint DEFAULT 0 NOT NULL;
|
||||
-- Pre-existing rows predate ownership tracking and have no recorded
|
||||
-- uploader; they stay at 0 ("unattributed") rather than being backfilled
|
||||
-- with a guess, and are surfaced as their own bucket in the admin UI.
|
||||
|
||||
CREATE INDEX idx_documents_owner_user_id ON public.documents (owner_user_id) WHERE owner_user_id <> 0;
|
||||
CREATE INDEX idx_photos_owner_user_id ON public.photos (owner_user_id) WHERE owner_user_id <> 0;
|
||||
|
||||
CREATE TABLE public.media_references (
|
||||
media_kind text NOT NULL,
|
||||
media_id bigint NOT NULL,
|
||||
ref_kind text NOT NULL,
|
||||
ref_key text NOT NULL,
|
||||
created_at timestamp with time zone DEFAULT now() NOT NULL,
|
||||
PRIMARY KEY (media_kind, media_id, ref_kind, ref_key)
|
||||
);
|
||||
CREATE INDEX idx_media_references_media ON public.media_references (media_kind, media_id);
|
||||
|
||||
ALTER TABLE public.documents ADD COLUMN orphaned_at timestamp with time zone;
|
||||
ALTER TABLE public.photos ADD COLUMN orphaned_at timestamp with time zone;
|
||||
CREATE INDEX idx_documents_orphaned_at ON public.documents (orphaned_at) WHERE orphaned_at IS NOT NULL;
|
||||
CREATE INDEX idx_photos_orphaned_at ON public.photos (orphaned_at) WHERE orphaned_at IS NOT NULL;
|
||||
|
|
@ -0,0 +1 @@
|
|||
DROP INDEX IF EXISTS idx_file_blobs_object_key_backend;
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
-- Supports the reference-count check ("is any other row still using this
|
||||
-- object_key on this backend?") that runs before a blob is physically
|
||||
-- deleted from its backend during storage retention sweeps.
|
||||
|
||||
CREATE INDEX idx_file_blobs_object_key_backend ON public.file_blobs (backend, object_key);
|
||||
18
go.mod
18
go.mod
|
|
@ -13,6 +13,7 @@ require (
|
|||
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa
|
||||
github.com/jackc/pgx/v5 v5.9.2
|
||||
github.com/lestrrat-go/jwx/v3 v3.1.1
|
||||
github.com/minio/minio-go/v7 v7.2.1
|
||||
github.com/pion/datachannel v1.6.2
|
||||
github.com/pion/dtls/v3 v3.1.5
|
||||
github.com/pion/ice/v4 v4.3.0
|
||||
|
|
@ -31,6 +32,7 @@ require (
|
|||
golang.org/x/image v0.31.0
|
||||
golang.org/x/net v0.57.0
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/sys v0.47.0
|
||||
golang.org/x/text v0.40.0
|
||||
)
|
||||
|
||||
|
|
@ -40,6 +42,7 @@ require (
|
|||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/coder/websocket v1.8.15 // indirect
|
||||
github.com/dlclark/regexp2 v1.12.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/fatih/color v1.19.0 // indirect
|
||||
github.com/ghodss/yaml v1.0.0 // indirect
|
||||
github.com/go-faster/jx v1.2.0 // indirect
|
||||
|
|
@ -55,6 +58,8 @@ require (
|
|||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.19.1 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/lestrrat-go/blackmagic v1.0.4 // indirect
|
||||
github.com/lestrrat-go/dsig v1.2.1 // indirect
|
||||
github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect
|
||||
|
|
@ -63,30 +68,37 @@ require (
|
|||
github.com/lestrrat-go/option/v2 v2.0.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.15 // indirect
|
||||
github.com/mattn/go-isatty v0.0.22 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.1 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/ogen-go/ogen v1.23.0 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pion/mdns/v2 v2.1.0 // indirect
|
||||
github.com/pion/randutil v0.1.0 // indirect
|
||||
github.com/pion/stun/v3 v3.1.6 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/refraction-networking/utls v1.8.2 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/segmentio/asm v1.2.1 // indirect
|
||||
github.com/shopspring/decimal v1.4.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
github.com/valyala/fastjson v1.6.10 // indirect
|
||||
github.com/wlynxg/anet v0.0.5 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
github.com/yuin/goldmark v1.8.4 // indirect
|
||||
github.com/yutopp/go-amf0 v0.1.0 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
|
||||
golang.org/x/mod v0.38.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
golang.org/x/tools v0.48.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.2 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
rsc.io/qr v0.2.0 // indirect
|
||||
)
|
||||
|
|
|
|||
35
go.sum
35
go.sum
|
|
@ -36,6 +36,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj
|
|||
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
|
||||
github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
|
|
@ -95,8 +97,11 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
|
|||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
|
||||
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
|
|
@ -121,8 +126,15 @@ github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy
|
|||
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
|
||||
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.2.1 h1:PfBfwvKB/MmqyN8Vb1G9voWisaM9OrLv+WwOvMwS9Dw=
|
||||
github.com/minio/minio-go/v7 v7.2.1/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0=
|
||||
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
|
||||
|
|
@ -135,6 +147,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
|
|||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
|
||||
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pion/datachannel v1.6.2 h1:7EXQ8TH3vTouBUdRWYbcX2edSx9Yj6k5zl5P+qyxEPc=
|
||||
github.com/pion/datachannel v1.6.2/go.mod h1:pzbdAZvyGtXbcHM1hBbsFaOTf40lZizU/dNlvVOak6E=
|
||||
github.com/pion/dtls/v3 v3.1.5 h1:9xJtVsHwMYeSjPp5Hh1FTis4DchnQWtnOa5o+6ygqfc=
|
||||
|
|
@ -174,24 +188,30 @@ github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEv
|
|||
github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM=
|
||||
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
|
||||
github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0=
|
||||
github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
|
||||
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
|
||||
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADTh4=
|
||||
github.com/valyala/fastjson v1.6.10/go.mod h1:e6FubmQouUNP73jtMLmcbxS6ydWIpOfhz34TSfO3JaE=
|
||||
github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=
|
||||
|
|
@ -207,6 +227,8 @@ github.com/yutopp/go-amf0 v0.1.0/go.mod h1:QzDOBr9RV6sQh6E5GFEJROZbU0iQKijORBmpr
|
|||
github.com/yutopp/go-flv v0.3.1/go.mod h1:pAlHPSVRMv5aCUKmGOS/dZn/ooTgnc09qOPmiUNMubs=
|
||||
github.com/yutopp/go-rtmp v0.0.7 h1:sKKm1MVV3ANbJHZlf3Kq8ecq99y5U7XnDUDxSjuK7KU=
|
||||
github.com/yutopp/go-rtmp v0.0.7/go.mod h1:KSwrC9Xj5Kf18EUlk1g7CScecjXfIqc0J5q+S0u6Irc=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
|
|
@ -243,7 +265,6 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
|
|
@ -256,6 +277,8 @@ golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk
|
|||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss=
|
||||
gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
|
|||
|
|
@ -27,6 +27,12 @@ type BlobBackend interface {
|
|||
// GetRange 只读 [offset, offset+limit) 段并返回该段字节与文件总大小(limit<=0 读到末尾),
|
||||
// 避免大文件每个 chunk 都整文件读入内存(getFile 按 chunk 多次请求 ⇒ 否则 O(N²) 放大)。
|
||||
GetRange(ctx context.Context, objectKey string, offset, limit int64) (data []byte, total int64, err error)
|
||||
// Delete removes the object at objectKey. Callers (storage retention GC)
|
||||
// must first confirm no remaining file_blobs row on this backend still
|
||||
// references objectKey -- content-addressed storage means the same
|
||||
// object can be shared by multiple documents/photos. Deleting an
|
||||
// already-absent object is not an error.
|
||||
Delete(ctx context.Context, objectKey string) error
|
||||
}
|
||||
|
||||
// UploadPartBackend 保存 upload.saveFilePart/saveBigFilePart 的临时分片字节。
|
||||
|
|
@ -205,6 +211,16 @@ func (l *LocalFS) GetRange(_ context.Context, objectKey string, offset, limit in
|
|||
return buf[:read], total, nil
|
||||
}
|
||||
|
||||
// Delete removes the on-disk object for objectKey. Missing objects are not
|
||||
// an error (idempotent, safe to retry). Callers must have already confirmed
|
||||
// no other file_blobs row on this backend still references objectKey.
|
||||
func (l *LocalFS) Delete(_ context.Context, objectKey string) error {
|
||||
if err := os.Remove(l.pathFor(objectKey)); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("delete blob: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *LocalFS) openBlobFile(objectKey string) (*sharedBlobFile, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
|
|
|||
|
|
@ -44,6 +44,32 @@ func TestLocalFSPutGetRoundTrip(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestLocalFSDelete(t *testing.T) {
|
||||
fs, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("new local fs: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
key, err := fs.Put(ctx, []byte("delete me"))
|
||||
if err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
if _, err := fs.Get(ctx, key); err != nil {
|
||||
t.Fatalf("get before delete: %v", err)
|
||||
}
|
||||
if err := fs.Delete(ctx, key); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
if _, err := fs.Get(ctx, key); err == nil {
|
||||
t.Fatal("expected get after delete to fail")
|
||||
}
|
||||
// Deleting an already-absent object must be idempotent, not an error --
|
||||
// the retention sweep can legitimately retry after a partial failure.
|
||||
if err := fs.Delete(ctx, key); err != nil {
|
||||
t.Fatalf("delete already-missing object: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalFSPutReaderRoundTrip(t *testing.T) {
|
||||
fs, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
|
|
|
|||
179
internal/app/files/blobs3.go
Normal file
179
internal/app/files/blobs3.go
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// S3FS stores blob bytes in an S3-compatible object store (self-hosted MinIO
|
||||
// or AWS S3). It implements BlobBackend using the same content-addressed
|
||||
// sha256-hex object key scheme as LocalFS, so file_blobs.object_key stays
|
||||
// comparable regardless of which backend wrote a given row and a deployment
|
||||
// can run with rows split across both backends (see TELESRV_BLOB_BACKEND).
|
||||
//
|
||||
// UploadPartBackend is intentionally NOT implemented here: transient upload
|
||||
// parts stay on local disk (see cmd/telesrv/main.go) even when the
|
||||
// permanent blob backend is s3 -- one S3 round trip per ~512KB chunk isn't
|
||||
// worth it for scratch data that's deleted within minutes of assembly.
|
||||
type S3FS struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
// NewS3FS creates an S3-compatible blob backend. endpoint is host[:port]
|
||||
// without a scheme (e.g. "minio.internal:9000" or "s3.amazonaws.com");
|
||||
// useSSL selects http vs https. pathStyle forces path-style addressing
|
||||
// (bucket in the URL path rather than as a subdomain), which self-hosted
|
||||
// MinIO typically requires and AWS S3 does not.
|
||||
func NewS3FS(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucket, region string, useSSL, pathStyle bool) (*S3FS, error) {
|
||||
if endpoint == "" || bucket == "" {
|
||||
return nil, fmt.Errorf("s3 blob backend: endpoint and bucket are required")
|
||||
}
|
||||
lookup := minio.BucketLookupAuto
|
||||
if pathStyle {
|
||||
lookup = minio.BucketLookupPath
|
||||
}
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: useSSL,
|
||||
Region: region,
|
||||
BucketLookup: lookup,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create s3 client: %w", err)
|
||||
}
|
||||
exists, err := client.BucketExists(ctx, bucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("check s3 bucket %q: %w", bucket, err)
|
||||
}
|
||||
if !exists {
|
||||
if err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: region}); err != nil {
|
||||
return nil, fmt.Errorf("create s3 bucket %q: %w", bucket, err)
|
||||
}
|
||||
}
|
||||
return &S3FS{client: client, bucket: bucket}, nil
|
||||
}
|
||||
|
||||
// Name 返回后端标识,与 file_blobs.backend 一致。
|
||||
func (s *S3FS) Name() string { return "s3" }
|
||||
|
||||
func (s *S3FS) Put(ctx context.Context, data []byte) (string, error) {
|
||||
key, _, _, err := s.PutReader(ctx, bytes.NewReader(data))
|
||||
return key, err
|
||||
}
|
||||
|
||||
// PutReader hashes the stream to a local temp file first (so the sha256 key
|
||||
// and exact size are known before the S3 PUT, matching LocalFS's
|
||||
// content-addressed dedup semantics -- an unknown-length streaming PUT would
|
||||
// need a second copy operation to rename-by-hash after the fact, which S3
|
||||
// has no equivalent of), then uploads it and checks for an existing object
|
||||
// with that key first to skip a redundant PUT.
|
||||
func (s *S3FS) PutReader(ctx context.Context, r io.Reader) (string, int64, []byte, error) {
|
||||
tmp, err := os.CreateTemp("", "blob-s3-*.tmp")
|
||||
if err != nil {
|
||||
return "", 0, nil, fmt.Errorf("create s3 blob staging file: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
h := sha256.New()
|
||||
size, err := copyWithContext(ctx, io.MultiWriter(tmp, h), r)
|
||||
closeErr := tmp.Close()
|
||||
if err != nil {
|
||||
return "", 0, nil, fmt.Errorf("stage s3 blob: %w", err)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return "", 0, nil, fmt.Errorf("close s3 blob staging file: %w", closeErr)
|
||||
}
|
||||
sum := h.Sum(nil)
|
||||
key := hex.EncodeToString(sum)
|
||||
|
||||
if _, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}); err == nil {
|
||||
return key, size, append([]byte(nil), sum...), nil
|
||||
} else if !isS3NotFound(err) {
|
||||
return "", 0, nil, fmt.Errorf("stat s3 blob: %w", err)
|
||||
}
|
||||
|
||||
f, err := os.Open(tmpPath)
|
||||
if err != nil {
|
||||
return "", 0, nil, fmt.Errorf("reopen s3 blob staging file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := s.client.PutObject(ctx, s.bucket, key, f, size, minio.PutObjectOptions{}); err != nil {
|
||||
return "", 0, nil, fmt.Errorf("put s3 blob: %w", err)
|
||||
}
|
||||
return key, size, append([]byte(nil), sum...), nil
|
||||
}
|
||||
|
||||
func (s *S3FS) Get(ctx context.Context, objectKey string) ([]byte, error) {
|
||||
obj, err := s.client.GetObject(ctx, s.bucket, objectKey, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get s3 blob: %w", err)
|
||||
}
|
||||
defer obj.Close()
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read s3 blob: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// GetRange 语义与 LocalFS.GetRange 一致:只读 [offset, offset+limit) 段,limit<=0 读到末尾,
|
||||
// total 取自对象实际大小。
|
||||
func (s *S3FS) GetRange(ctx context.Context, objectKey string, offset, limit int64) ([]byte, int64, error) {
|
||||
info, err := s.client.StatObject(ctx, s.bucket, objectKey, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("stat s3 blob: %w", err)
|
||||
}
|
||||
total := info.Size
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
if offset >= total {
|
||||
return []byte{}, total, nil
|
||||
}
|
||||
opts := minio.GetObjectOptions{}
|
||||
end := total - 1
|
||||
if limit > 0 && offset+limit-1 < end {
|
||||
end = offset + limit - 1
|
||||
}
|
||||
if err := opts.SetRange(offset, end); err != nil {
|
||||
return nil, 0, fmt.Errorf("set s3 range: %w", err)
|
||||
}
|
||||
obj, err := s.client.GetObject(ctx, s.bucket, objectKey, opts)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("get s3 blob range: %w", err)
|
||||
}
|
||||
defer obj.Close()
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("read s3 blob range: %w", err)
|
||||
}
|
||||
return data, total, nil
|
||||
}
|
||||
|
||||
// Delete removes the object at objectKey. A missing object is not an error
|
||||
// (idempotent, safe to retry). Callers must have already confirmed no other
|
||||
// file_blobs row on this backend still references objectKey.
|
||||
func (s *S3FS) Delete(ctx context.Context, objectKey string) error {
|
||||
if err := s.client.RemoveObject(ctx, s.bucket, objectKey, minio.RemoveObjectOptions{}); err != nil {
|
||||
if isS3NotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("delete s3 blob: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isS3NotFound(err error) bool {
|
||||
resp := minio.ToErrorResponse(err)
|
||||
return resp.Code == "NoSuchKey" || resp.Code == "NotFound" || resp.StatusCode == 404
|
||||
}
|
||||
99
internal/app/files/diskspace.go
Normal file
99
internal/app/files/diskspace.go
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
package files
|
||||
|
||||
import "sync/atomic"
|
||||
|
||||
// SpaceGuard bounds how much more may be written to the permanent blob
|
||||
// backend. LocalDiskSpaceGuard checks real OS free disk bytes;
|
||||
// S3BudgetSpaceGuard compares a cached tracked-bytes total against a
|
||||
// configured budget (S3 has no OS-level "free space" concept). Both are
|
||||
// refreshed periodically by DiskUsageWorker rather than recomputed on every
|
||||
// upload chunk.
|
||||
type SpaceGuard interface {
|
||||
// Allow reports whether writing `additional` more bytes is currently
|
||||
// permitted. false should surface to the client as domain.ErrStorageFull.
|
||||
Allow(additional int64) (bool, error)
|
||||
// Usage returns the last-refreshed (used, total) byte snapshot for the
|
||||
// admin panel. ok is false if no successful refresh has happened yet.
|
||||
Usage() (used, total int64, ok bool)
|
||||
}
|
||||
|
||||
// NoopSpaceGuard always allows writes; the default when the low-space guard
|
||||
// is disabled by configuration.
|
||||
type NoopSpaceGuard struct{}
|
||||
|
||||
func (NoopSpaceGuard) Allow(int64) (bool, error) { return true, nil }
|
||||
func (NoopSpaceGuard) Usage() (int64, int64, bool) { return 0, 0, false }
|
||||
|
||||
// LocalDiskSpaceGuard rejects writes once cached free disk bytes fall below
|
||||
// minFreeBytes (<=0 disables the check). The free-bytes figure is
|
||||
// refreshed by DiskUsageWorker, not recomputed per call, to avoid a statfs
|
||||
// syscall on every upload chunk -- reads are lock-free.
|
||||
type LocalDiskSpaceGuard struct {
|
||||
minFreeBytes int64
|
||||
free atomic.Int64
|
||||
total atomic.Int64
|
||||
ready atomic.Bool
|
||||
}
|
||||
|
||||
func NewLocalDiskSpaceGuard(minFreeBytes int64) *LocalDiskSpaceGuard {
|
||||
return &LocalDiskSpaceGuard{minFreeBytes: minFreeBytes}
|
||||
}
|
||||
|
||||
func (g *LocalDiskSpaceGuard) Allow(additional int64) (bool, error) {
|
||||
// Before the first refresh completes, allow rather than reject: a
|
||||
// startup race shouldn't turn into spurious upload failures.
|
||||
if g.minFreeBytes <= 0 || !g.ready.Load() {
|
||||
return true, nil
|
||||
}
|
||||
return g.free.Load()-additional >= g.minFreeBytes, nil
|
||||
}
|
||||
|
||||
func (g *LocalDiskSpaceGuard) Usage() (used, total int64, ok bool) {
|
||||
if !g.ready.Load() {
|
||||
return 0, 0, false
|
||||
}
|
||||
total = g.total.Load()
|
||||
used = total - g.free.Load()
|
||||
if used < 0 {
|
||||
used = 0
|
||||
}
|
||||
return used, total, true
|
||||
}
|
||||
|
||||
func (g *LocalDiskSpaceGuard) setFree(free, total int64) {
|
||||
g.free.Store(free)
|
||||
g.total.Store(total)
|
||||
g.ready.Store(true)
|
||||
}
|
||||
|
||||
// S3BudgetSpaceGuard rejects writes once a cached tracked-bytes total
|
||||
// (refreshed periodically from file_blobs) would exceed maxTotalBytes
|
||||
// (<=0 disables the check).
|
||||
type S3BudgetSpaceGuard struct {
|
||||
maxTotalBytes int64
|
||||
used atomic.Int64
|
||||
ready atomic.Bool
|
||||
}
|
||||
|
||||
func NewS3BudgetSpaceGuard(maxTotalBytes int64) *S3BudgetSpaceGuard {
|
||||
return &S3BudgetSpaceGuard{maxTotalBytes: maxTotalBytes}
|
||||
}
|
||||
|
||||
func (g *S3BudgetSpaceGuard) Allow(additional int64) (bool, error) {
|
||||
if g.maxTotalBytes <= 0 || !g.ready.Load() {
|
||||
return true, nil
|
||||
}
|
||||
return g.used.Load()+additional <= g.maxTotalBytes, nil
|
||||
}
|
||||
|
||||
func (g *S3BudgetSpaceGuard) Usage() (used, total int64, ok bool) {
|
||||
if !g.ready.Load() {
|
||||
return 0, 0, false
|
||||
}
|
||||
return g.used.Load(), g.maxTotalBytes, true
|
||||
}
|
||||
|
||||
func (g *S3BudgetSpaceGuard) setUsed(used int64) {
|
||||
g.used.Store(used)
|
||||
g.ready.Store(true)
|
||||
}
|
||||
76
internal/app/files/diskspace_test.go
Normal file
76
internal/app/files/diskspace_test.go
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
package files
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNoopSpaceGuardAlwaysAllows(t *testing.T) {
|
||||
g := NoopSpaceGuard{}
|
||||
if allowed, err := g.Allow(1 << 40); err != nil || !allowed {
|
||||
t.Fatalf("expected noop guard to always allow, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
if _, _, ok := g.Usage(); ok {
|
||||
t.Fatal("expected noop guard to report no usage snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalDiskSpaceGuardBeforeFirstRefresh(t *testing.T) {
|
||||
g := NewLocalDiskSpaceGuard(1 << 30)
|
||||
// Before setFree has ever run, the guard must not reject -- a startup
|
||||
// race shouldn't turn into spurious upload failures.
|
||||
if allowed, err := g.Allow(1 << 40); err != nil || !allowed {
|
||||
t.Fatalf("expected pre-refresh guard to allow, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
if _, _, ok := g.Usage(); ok {
|
||||
t.Fatal("expected no usage snapshot before first refresh")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalDiskSpaceGuardThreshold(t *testing.T) {
|
||||
const minFree = int64(1000)
|
||||
g := NewLocalDiskSpaceGuard(minFree)
|
||||
g.setFree(1500, 10000)
|
||||
|
||||
if allowed, err := g.Allow(400); err != nil || !allowed {
|
||||
t.Fatalf("writing 400 bytes leaves 1100 free (>= 1000 min): want allow, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
if allowed, err := g.Allow(600); err != nil || allowed {
|
||||
t.Fatalf("writing 600 bytes leaves 900 free (< 1000 min): want reject, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
|
||||
used, total, ok := g.Usage()
|
||||
if !ok || total != 10000 || used != 8500 {
|
||||
t.Fatalf("usage snapshot = used=%d total=%d ok=%v, want used=8500 total=10000 ok=true", used, total, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalDiskSpaceGuardDisabled(t *testing.T) {
|
||||
g := NewLocalDiskSpaceGuard(0)
|
||||
g.setFree(10, 1000)
|
||||
if allowed, err := g.Allow(1 << 40); err != nil || !allowed {
|
||||
t.Fatalf("minFreeBytes<=0 must disable the check, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BudgetSpaceGuardThreshold(t *testing.T) {
|
||||
const maxTotal = int64(10000)
|
||||
g := NewS3BudgetSpaceGuard(maxTotal)
|
||||
g.setUsed(9000)
|
||||
|
||||
if allowed, err := g.Allow(1000); err != nil || !allowed {
|
||||
t.Fatalf("9000+1000 == budget: want allow, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
if allowed, err := g.Allow(1001); err != nil || allowed {
|
||||
t.Fatalf("9000+1001 exceeds budget: want reject, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
|
||||
used, total, ok := g.Usage()
|
||||
if !ok || total != maxTotal || used != 9000 {
|
||||
t.Fatalf("usage snapshot = used=%d total=%d ok=%v, want used=9000 total=%d ok=true", used, total, ok, maxTotal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BudgetSpaceGuardBeforeFirstRefresh(t *testing.T) {
|
||||
g := NewS3BudgetSpaceGuard(100)
|
||||
if allowed, err := g.Allow(1 << 40); err != nil || !allowed {
|
||||
t.Fatalf("expected pre-refresh guard to allow, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
}
|
||||
15
internal/app/files/diskspace_unix.go
Normal file
15
internal/app/files/diskspace_unix.go
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
//go:build !windows
|
||||
|
||||
package files
|
||||
|
||||
import "golang.org/x/sys/unix"
|
||||
|
||||
// localDiskFreeBytes returns free (available to an unprivileged writer, not
|
||||
// counting reserved blocks) and total bytes for the filesystem containing path.
|
||||
func localDiskFreeBytes(path string) (free, total int64, err error) {
|
||||
var st unix.Statfs_t
|
||||
if err := unix.Statfs(path, &st); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return int64(st.Bavail) * int64(st.Bsize), int64(st.Blocks) * int64(st.Bsize), nil
|
||||
}
|
||||
19
internal/app/files/diskspace_windows.go
Normal file
19
internal/app/files/diskspace_windows.go
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
//go:build windows
|
||||
|
||||
package files
|
||||
|
||||
import "golang.org/x/sys/windows"
|
||||
|
||||
// localDiskFreeBytes returns free (available to the calling user) and total
|
||||
// bytes for the volume containing path.
|
||||
func localDiskFreeBytes(path string) (free, total int64, err error) {
|
||||
ptr, err := windows.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
var freeAvail, totalBytes, totalFree uint64
|
||||
if err := windows.GetDiskFreeSpaceEx(ptr, &freeAvail, &totalBytes, &totalFree); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return int64(freeAvail), int64(totalBytes), nil
|
||||
}
|
||||
81
internal/app/files/diskusage_worker.go
Normal file
81
internal/app/files/diskusage_worker.go
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// SumFileBlobBytesStore is the minimal store dependency DiskUsageWorker
|
||||
// needs to refresh an S3BudgetSpaceGuard.
|
||||
type SumFileBlobBytesStore interface {
|
||||
SumFileBlobBytes(ctx context.Context) (int64, error)
|
||||
}
|
||||
|
||||
// DiskUsageWorker periodically refreshes one SpaceGuard's cached usage
|
||||
// snapshot, so the upload path never pays a statfs syscall / SUM(size)
|
||||
// query per chunk.
|
||||
type DiskUsageWorker struct {
|
||||
interval time.Duration
|
||||
log *zap.Logger
|
||||
refresh func(ctx context.Context) error
|
||||
}
|
||||
|
||||
// NewLocalDiskUsageWorker refreshes a LocalDiskSpaceGuard from real OS free
|
||||
// disk bytes under root (the blob backend's storage directory).
|
||||
func NewLocalDiskUsageWorker(guard *LocalDiskSpaceGuard, root string, interval time.Duration, log *zap.Logger) *DiskUsageWorker {
|
||||
return newDiskUsageWorker(interval, log, func(context.Context) error {
|
||||
free, total, err := localDiskFreeBytes(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
guard.setFree(free, total)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// NewS3DiskUsageWorker refreshes an S3BudgetSpaceGuard from the tracked
|
||||
// file_blobs byte total.
|
||||
func NewS3DiskUsageWorker(guard *S3BudgetSpaceGuard, media SumFileBlobBytesStore, interval time.Duration, log *zap.Logger) *DiskUsageWorker {
|
||||
return newDiskUsageWorker(interval, log, func(ctx context.Context) error {
|
||||
used, err := media.SumFileBlobBytes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
guard.setUsed(used)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func newDiskUsageWorker(interval time.Duration, log *zap.Logger, refresh func(context.Context) error) *DiskUsageWorker {
|
||||
if interval <= 0 {
|
||||
interval = time.Minute
|
||||
}
|
||||
if log == nil {
|
||||
log = zap.NewNop()
|
||||
}
|
||||
return &DiskUsageWorker{interval: interval, log: log, refresh: refresh}
|
||||
}
|
||||
|
||||
// Run refreshes once immediately (so the guard isn't stuck "not ready" for
|
||||
// a full interval after startup), then on every tick until ctx is done.
|
||||
func (w *DiskUsageWorker) Run(ctx context.Context) {
|
||||
w.refreshOnce(ctx)
|
||||
ticker := time.NewTicker(w.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
w.refreshOnce(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *DiskUsageWorker) refreshOnce(ctx context.Context) {
|
||||
if err := w.refresh(ctx); err != nil {
|
||||
w.log.Warn("refresh storage usage snapshot failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
|
@ -63,7 +63,7 @@ func (s *Service) CreatePhotoFromUpload(ctx context.Context, file domain.Uploade
|
|||
if len(data) == 0 {
|
||||
return domain.Photo{}, domain.ErrPhotoInvalid
|
||||
}
|
||||
photo, err := s.createPhoto(ctx, data, photoSizeSpecsForMessage(data))
|
||||
photo, err := s.createPhoto(ctx, data, photoSizeSpecsForMessage(data), file.OwnerUserID)
|
||||
if err != nil {
|
||||
return domain.Photo{}, err
|
||||
}
|
||||
|
|
@ -86,11 +86,13 @@ func (s *Service) CreatePhotoFromUpload(ctx context.Context, file domain.Uploade
|
|||
}
|
||||
|
||||
// CreatePhotoFromBytes stores already-fetched image bytes as a message Photo.
|
||||
// There is no uploader (e.g. a server-fetched webpage preview image), so the
|
||||
// photo is attributed to the system (owner_user_id 0) for storage accounting.
|
||||
func (s *Service) CreatePhotoFromBytes(ctx context.Context, data []byte) (domain.Photo, error) {
|
||||
if len(data) == 0 {
|
||||
return domain.Photo{}, domain.ErrPhotoInvalid
|
||||
}
|
||||
return s.createPhoto(ctx, data, photoSizeSpecsForMessage(data))
|
||||
return s.createPhoto(ctx, data, photoSizeSpecsForMessage(data), 0)
|
||||
}
|
||||
|
||||
// GetPhoto 按 id 返回已存储照片。
|
||||
|
|
@ -148,7 +150,7 @@ func (s *Service) CreateAvatarFromUpload(ctx context.Context, file domain.Upload
|
|||
if len(data) == 0 {
|
||||
return domain.Photo{}, domain.ErrPhotoInvalid
|
||||
}
|
||||
return s.createAvatarPhoto(ctx, data)
|
||||
return s.createAvatarPhoto(ctx, data, file.OwnerUserID)
|
||||
}
|
||||
|
||||
// CreateAvatarVideoFromUpload stores an animated profile video as photo.video_sizes.
|
||||
|
|
@ -206,6 +208,7 @@ func (s *Service) createAvatarVideoFromUpload(ctx context.Context, file domain.U
|
|||
Date: int(time.Now().Unix()),
|
||||
DCID: s.dc,
|
||||
Sizes: sizes,
|
||||
OwnerUserID: file.OwnerUserID,
|
||||
}
|
||||
if err := s.media.PutPhoto(ctx, photo); err != nil {
|
||||
return domain.Photo{}, err
|
||||
|
|
@ -306,6 +309,7 @@ func (s *Service) CreateDocumentFromUpload(ctx context.Context, file domain.Uplo
|
|||
Size: body.Size,
|
||||
DCID: s.dc,
|
||||
Attributes: spec.Attributes,
|
||||
OwnerUserID: file.OwnerUserID,
|
||||
}
|
||||
thumbMaterialized := false
|
||||
if spec.Thumb != nil {
|
||||
|
|
@ -620,7 +624,7 @@ func (s *Service) DeleteProfilePhotosKind(ctx context.Context, ownerType domain.
|
|||
}
|
||||
|
||||
// createPhoto 把字节落 blob(每个尺寸一个 location_key,指向同一内容)并写 photos 表。
|
||||
func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSizeSpec) (domain.Photo, error) {
|
||||
func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSizeSpec, ownerUserID int64) (domain.Photo, error) {
|
||||
photoID := randomID()
|
||||
sizes, err := s.putPhotoStaticSizes(ctx, photoID, data, specs)
|
||||
if err != nil {
|
||||
|
|
@ -633,6 +637,7 @@ func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSiz
|
|||
Date: int(time.Now().Unix()),
|
||||
DCID: s.dc,
|
||||
Sizes: sizes,
|
||||
OwnerUserID: ownerUserID,
|
||||
}
|
||||
if err := s.media.PutPhoto(ctx, photo); err != nil {
|
||||
return domain.Photo{}, err
|
||||
|
|
@ -640,7 +645,7 @@ func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSiz
|
|||
return photo, nil
|
||||
}
|
||||
|
||||
func (s *Service) createAvatarPhoto(ctx context.Context, data []byte) (domain.Photo, error) {
|
||||
func (s *Service) createAvatarPhoto(ctx context.Context, data []byte, ownerUserID int64) (domain.Photo, error) {
|
||||
photoID := randomID()
|
||||
sizes, err := s.putAvatarStaticSizes(ctx, photoID, data, photoSizeSpecsForAvatar(data))
|
||||
if err != nil {
|
||||
|
|
@ -653,6 +658,7 @@ func (s *Service) createAvatarPhoto(ctx context.Context, data []byte) (domain.Ph
|
|||
Date: int(time.Now().Unix()),
|
||||
DCID: s.dc,
|
||||
Sizes: sizes,
|
||||
OwnerUserID: ownerUserID,
|
||||
}
|
||||
if err := s.media.PutPhoto(ctx, photo); err != nil {
|
||||
return domain.Photo{}, err
|
||||
|
|
|
|||
94
internal/app/files/retention.go
Normal file
94
internal/app/files/retention.go
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// mediaRetentionStore is implemented by store.MediaStore backends that
|
||||
// support the storage retention sweep (currently only the Postgres store).
|
||||
// A type assertion, not a MediaStore interface method, keeps these
|
||||
// admin/maintenance-only queries out of the hot RPC-facing interface --
|
||||
// same convention as photoBatchStore above.
|
||||
type mediaRetentionStore interface {
|
||||
ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||
ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||
CountFileBlobRefs(ctx context.Context, backend, objectKey string) (int, error)
|
||||
DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||
DeletePhotoAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||
}
|
||||
|
||||
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
|
||||
// permanently deletes documents/photos that have had no live reference
|
||||
// (message/profile-photo/sticker-set, see media_references) since at least
|
||||
// cutoff, along with their blob(s) -- but only physically removes bytes
|
||||
// from the backend once confirming no other file_blobs row still needs the
|
||||
// object, since content-addressed storage means the same bytes can be
|
||||
// shared across documents/photos.
|
||||
func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
|
||||
store, ok := s.media.(mediaRetentionStore)
|
||||
if !ok || limit <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
deleted := 0
|
||||
docIDs, err := store.ListOrphanedDocumentIDsOlderThan(ctx, cutoff, limit)
|
||||
if err != nil {
|
||||
return deleted, fmt.Errorf("list orphaned documents: %w", err)
|
||||
}
|
||||
for _, id := range docIDs {
|
||||
blobs, err := store.DeleteDocumentAndBlobs(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("delete orphaned document failed", zap.Int64("document_id", id), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
deleted++
|
||||
}
|
||||
photoIDs, err := store.ListOrphanedPhotoIDsOlderThan(ctx, cutoff, limit)
|
||||
if err != nil {
|
||||
return deleted, fmt.Errorf("list orphaned photos: %w", err)
|
||||
}
|
||||
for _, id := range photoIDs {
|
||||
blobs, err := store.DeletePhotoAndBlobs(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("delete orphaned photo failed", zap.Int64("photo_id", id), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
deleted++
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// deleteOrphanedBlobs removes each blob from its backend once confirming
|
||||
// (via CountFileBlobRefs) no other file_blobs row still references
|
||||
// (backend, object_key). Only blobs on the currently active backend are
|
||||
// physically removed -- a blob left over from a previously active backend
|
||||
// (deployment switched TELESRV_BLOB_BACKEND at some point; switching back
|
||||
// isn't supported) is logged and skipped rather than silently dropped,
|
||||
// since there's no configured client to reach it right now anyway.
|
||||
func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionStore, blobs []domain.FileBlob) {
|
||||
for _, b := range blobs {
|
||||
refs, err := store.CountFileBlobRefs(ctx, string(b.Backend), b.ObjectKey)
|
||||
if err != nil {
|
||||
s.log.Warn("count file blob refs failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if refs > 0 {
|
||||
continue
|
||||
}
|
||||
if string(b.Backend) != s.blobs.Name() {
|
||||
s.log.Warn("orphaned blob is on an inactive backend, skipping physical delete",
|
||||
zap.String("backend", string(b.Backend)), zap.String("object_key", b.ObjectKey))
|
||||
continue
|
||||
}
|
||||
if err := s.blobs.Delete(ctx, b.ObjectKey); err != nil {
|
||||
s.log.Warn("delete orphaned blob failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -168,6 +168,16 @@ func (f *fakeMediaStore) GetFileBlobs(_ context.Context, keys []string) (map[str
|
|||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeMediaStore) SumFileBlobBytes(_ context.Context) (int64, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var total int64
|
||||
for _, b := range f.blobs {
|
||||
total += b.Size
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (f *fakeMediaStore) GetSeedState(_ context.Context, key string) (string, bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ type Service struct {
|
|||
stickerSetCache *stickerSetFullCache
|
||||
stickerSetNegCache *stickerSetNegativeCache
|
||||
uploadQuota domain.UploadPartQuota
|
||||
spaceGuard SpaceGuard
|
||||
mapTiles *mapTileProxy
|
||||
externalMedia *externalMediaFetcher
|
||||
webpage *webpageFetcher
|
||||
|
|
@ -116,6 +117,30 @@ func WithUploadPartQuota(quota domain.UploadPartQuota) Option {
|
|||
}
|
||||
}
|
||||
|
||||
// WithSpaceGuard installs the low-disk-space upload guard. Not calling this
|
||||
// (or passing nil) leaves the default NoopSpaceGuard, which never rejects.
|
||||
func WithSpaceGuard(guard SpaceGuard) Option {
|
||||
return func(s *Service) {
|
||||
if guard != nil {
|
||||
s.spaceGuard = guard
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithUploadPartBackend overrides where transient upload-part chunks are
|
||||
// staged before assembly, independent of the permanent blob backend passed
|
||||
// to NewService. Needed when the permanent backend is s3 (S3FS doesn't
|
||||
// implement UploadPartBackend -- chunk-per-request S3 round trips aren't
|
||||
// worth it for scratch data deleted within minutes): pass a LocalFS here so
|
||||
// uploads keep working, while permanent blobs still land in s3.
|
||||
func WithUploadPartBackend(backend UploadPartBackend) Option {
|
||||
return func(s *Service) {
|
||||
if backend != nil {
|
||||
s.uploadParts = backend
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NewService 创建 files 服务。dc 是本 server 的 DC id,写入新建 document/photo 的 dc_id。
|
||||
func NewService(media store.MediaStore, blobs BlobBackend, dc int, opts ...Option) *Service {
|
||||
s := &Service{
|
||||
|
|
@ -132,6 +157,7 @@ func NewService(media store.MediaStore, blobs BlobBackend, dc int, opts ...Optio
|
|||
MaxParts: DefaultUploadInFlightMaxParts,
|
||||
MaxFiles: DefaultUploadInFlightMaxFiles,
|
||||
},
|
||||
spaceGuard: NoopSpaceGuard{},
|
||||
}
|
||||
if partBackend, ok := blobs.(UploadPartBackend); ok {
|
||||
s.uploadParts = partBackend
|
||||
|
|
@ -205,6 +231,16 @@ func (s *Service) saveFilePart(ctx context.Context, part domain.UploadPart, byte
|
|||
if s.uploadParts == nil {
|
||||
return fmt.Errorf("upload part backend not configured")
|
||||
}
|
||||
// Cheapest possible rejection point: reject before any disk write once
|
||||
// the permanent blob backend is low on space. Upload parts themselves
|
||||
// always land on local scratch disk (see UploadPartBackend), but a
|
||||
// low-space condition on the permanent backend means assembly will
|
||||
// fail anyway, so there's no point accepting more chunks toward it.
|
||||
if allowed, err := s.spaceGuard.Allow(int64(len(bytes))); err != nil {
|
||||
return err
|
||||
} else if !allowed {
|
||||
return domain.ErrStorageFull
|
||||
}
|
||||
slot, err := s.checkUploadPartQuota(ctx, part)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -584,10 +620,19 @@ type assembledUploadBlob struct {
|
|||
// assembleUploadBlob 把上传分片流式写入正式 blob。调用方应在 durable media 元数据
|
||||
// 成功提交后调用 cleanupUploadParts,避免 metadata 写失败时丢失可重试的上传分片。
|
||||
func (s *Service) assembleUploadBlob(ctx context.Context, ownerUserID, fileID int64, expectedParts int) (assembledUploadBlob, error) {
|
||||
parts, _, err := s.loadAndValidateUploadParts(ctx, ownerUserID, fileID, expectedParts)
|
||||
parts, total, err := s.loadAndValidateUploadParts(ctx, ownerUserID, fileID, expectedParts)
|
||||
if err != nil {
|
||||
return assembledUploadBlob{}, err
|
||||
}
|
||||
// Re-check free space against the full assembled size right before
|
||||
// committing to the permanent blob backend: SaveFilePart already
|
||||
// checked each chunk, but free space may have dropped since then over
|
||||
// the lifetime of a large multi-part upload.
|
||||
if allowed, err := s.spaceGuard.Allow(total); err != nil {
|
||||
return assembledUploadBlob{}, err
|
||||
} else if !allowed {
|
||||
return assembledUploadBlob{}, domain.ErrStorageFull
|
||||
}
|
||||
if s.uploadParts == nil {
|
||||
return assembledUploadBlob{}, fmt.Errorf("upload part backend not configured")
|
||||
}
|
||||
|
|
|
|||
27
internal/app/maintenance/media_retention.go
Normal file
27
internal/app/maintenance/media_retention.go
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
package maintenance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// OrphanedMediaRetentionStore deletes documents/photos that have been
|
||||
// orphaned (no live message/profile-photo/sticker-set reference remains,
|
||||
// tracked via media_references + orphaned_at) for at least the configured
|
||||
// age, along with their underlying blob once no other file_blobs row on
|
||||
// its backend still needs it. Never touches media that still has a live
|
||||
// reference, regardless of age.
|
||||
type OrphanedMediaRetentionStore interface {
|
||||
DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error)
|
||||
}
|
||||
|
||||
// WithOrphanedMediaRetention enables the storage retention sweep. maxAge is
|
||||
// how long a document/photo must have been orphaned before it's actually
|
||||
// deleted (not how old the media itself is) -- <=0 leaves the sweep
|
||||
// disabled even if a store is provided, matching
|
||||
// TELESRV_STORAGE_RETENTION_ENABLE=false being the safe default.
|
||||
func (w *RetentionWorker) WithOrphanedMediaRetention(store OrphanedMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
|
||||
w.orphanedMedia = store
|
||||
w.orphanedMediaMaxAge = maxAge
|
||||
return w
|
||||
}
|
||||
|
|
@ -118,6 +118,7 @@ type RetentionWorker struct {
|
|||
orphanAuthKeys OrphanAuthKeyRetentionStore
|
||||
activeAuthKeys ActiveRawAuthKeyProvider
|
||||
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
|
||||
orphanedMedia OrphanedMediaRetentionStore
|
||||
logger *zap.Logger
|
||||
retention time.Duration
|
||||
botAPIRetention time.Duration
|
||||
|
|
@ -126,6 +127,7 @@ type RetentionWorker struct {
|
|||
authDeliveryReportRetention time.Duration
|
||||
outboxPoisonRetention time.Duration
|
||||
outboxPoisonInterval time.Duration
|
||||
orphanedMediaMaxAge time.Duration
|
||||
interval time.Duration
|
||||
batch int
|
||||
}
|
||||
|
|
@ -412,6 +414,17 @@ func (w *RetentionWorker) runRetentionOnce(ctx context.Context) {
|
|||
w.logger.Info("expired channel_update_events contiguous-prefix cleanup complete", zap.Int("deleted", channelDeleted))
|
||||
}
|
||||
}
|
||||
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
|
||||
// Only ever touches documents/photos already marked orphaned (no live
|
||||
// message/profile-photo/sticker-set reference remains) -- media still
|
||||
// visible in a conversation is never a candidate, regardless of age.
|
||||
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
|
||||
if err != nil {
|
||||
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
|
||||
} else if mediaDeleted > 0 {
|
||||
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) orphanHeartbeatInterval() time.Duration {
|
||||
|
|
|
|||
|
|
@ -253,6 +253,45 @@ type Config struct {
|
|||
StarGiftTONStartingGrant int64
|
||||
// BlobDir 是本地磁盘 blob backend 根目录(媒体文件字节内容)。
|
||||
BlobDir string
|
||||
// BlobBackendKind selects the blob storage backend: "localfs" (default)
|
||||
// or "s3". Transient upload parts always stay on local disk (BlobDir)
|
||||
// regardless of this setting -- only the permanent blob store moves.
|
||||
BlobBackendKind string
|
||||
// S3Endpoint/S3Region/S3Bucket/S3AccessKeyID/S3SecretAccessKey/S3UseSSL/
|
||||
// S3PathStyle configure the s3 blob backend; only used when
|
||||
// BlobBackendKind == "s3". Works against self-hosted MinIO or AWS S3.
|
||||
S3Endpoint string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3AccessKeyID string
|
||||
S3SecretAccessKey string
|
||||
S3UseSSL bool
|
||||
S3PathStyle bool
|
||||
// StorageLowSpaceGuardEnable turns on the pre-upload free-space check.
|
||||
StorageLowSpaceGuardEnable bool
|
||||
// StorageMinFreeBytes: for the localfs backend, reject new uploads once
|
||||
// real free disk bytes fall below this. <=0 disables this check.
|
||||
StorageMinFreeBytes int64
|
||||
// StorageMaxTotalBytes: reject new uploads once total tracked blob
|
||||
// bytes would exceed this -- the only meaningful "low space" signal for
|
||||
// the s3 backend (no OS-level free space concept), and usable as an
|
||||
// optional soft budget cap on localfs too. <=0 disables this check.
|
||||
StorageMaxTotalBytes int64
|
||||
// StorageUsageRefreshInterval is how often the cached free-space/budget
|
||||
// usage gauge refreshes; <=0 uses a 1 minute default.
|
||||
StorageUsageRefreshInterval time.Duration
|
||||
// StorageRetentionEnable turns on the orphaned-media age sweep. Off by
|
||||
// default: orphaned media (no live message/profile-photo/sticker-set
|
||||
// reference) is tracked and visible in the admin panel, but nothing is
|
||||
// auto-deleted until the operator explicitly opts in.
|
||||
StorageRetentionEnable bool
|
||||
// StorageRetentionMaxAge is how long a document/photo must have been
|
||||
// orphaned (not how old the media itself is) before the sweep deletes
|
||||
// it. Never deletes media that still has a live reference, regardless
|
||||
// of age. <=0 uses a 30 day default. The sweep itself runs on the
|
||||
// shared RetentionInterval/RetentionBatch cadence alongside every other
|
||||
// retention check (see maintenance.RetentionWorker).
|
||||
StorageRetentionMaxAge time.Duration
|
||||
// StickerSeedDir 是 reaction / sticker 资源种子目录(导入到 documents/sticker_sets + blob)。
|
||||
StickerSeedDir string
|
||||
// StickerSeedMaxSets 限制导入的常规贴纸集数量(避免启动时导入过多包),<=0 表示不限。
|
||||
|
|
@ -756,6 +795,20 @@ func Load() (Config, error) {
|
|||
OfficialGiftsDir: envOr("TELESRV_OFFICIAL_GIFTS_DIR", "data/official-gifts"),
|
||||
StarGiftTONStartingGrant: envInt64Or("TELESRV_STARGIFT_TON_STARTING_GRANT", 10_000_000_000),
|
||||
BlobDir: envOr("TELESRV_BLOB_DIR", "data/blobs"),
|
||||
BlobBackendKind: strings.ToLower(strings.TrimSpace(envOr("TELESRV_BLOB_BACKEND", "localfs"))),
|
||||
S3Endpoint: envOr("TELESRV_S3_ENDPOINT", ""),
|
||||
S3Region: envOr("TELESRV_S3_REGION", "us-east-1"),
|
||||
S3Bucket: envOr("TELESRV_S3_BUCKET", ""),
|
||||
S3AccessKeyID: envOr("TELESRV_S3_ACCESS_KEY_ID", ""),
|
||||
S3SecretAccessKey: envOr("TELESRV_S3_SECRET_ACCESS_KEY", ""),
|
||||
S3UseSSL: envBoolOr("TELESRV_S3_USE_SSL", true),
|
||||
S3PathStyle: envBoolOr("TELESRV_S3_PATH_STYLE", false),
|
||||
StorageLowSpaceGuardEnable: envBoolOr("TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE", true),
|
||||
StorageMinFreeBytes: envInt64Or("TELESRV_STORAGE_MIN_FREE_BYTES", 1<<30),
|
||||
StorageMaxTotalBytes: envInt64Or("TELESRV_STORAGE_MAX_TOTAL_BYTES", 0),
|
||||
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
|
||||
StorageRetentionEnable: envBoolOr("TELESRV_STORAGE_RETENTION_ENABLE", false),
|
||||
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
|
||||
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
|
||||
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
|
||||
PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"),
|
||||
|
|
@ -930,6 +983,9 @@ func Load() (Config, error) {
|
|||
if err := validateTelegramLoginConfig(cfg); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if err := validateStorageConfig(cfg); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
|
|
@ -1132,6 +1188,40 @@ func validateVerificationConfig(cfg Config) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// validateStorageConfig checks the blob backend selection and storage
|
||||
// management (low-space guard, retention sweep) settings.
|
||||
func validateStorageConfig(cfg Config) error {
|
||||
switch cfg.BlobBackendKind {
|
||||
case "localfs":
|
||||
// no extra requirements
|
||||
case "s3":
|
||||
if strings.TrimSpace(cfg.S3Endpoint) == "" {
|
||||
return fmt.Errorf("TELESRV_S3_ENDPOINT is required when TELESRV_BLOB_BACKEND=s3")
|
||||
}
|
||||
if strings.TrimSpace(cfg.S3Bucket) == "" {
|
||||
return fmt.Errorf("TELESRV_S3_BUCKET is required when TELESRV_BLOB_BACKEND=s3")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("TELESRV_BLOB_BACKEND must be \"localfs\" or \"s3\", got %q", cfg.BlobBackendKind)
|
||||
}
|
||||
if cfg.StorageMinFreeBytes < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_MIN_FREE_BYTES must be non-negative")
|
||||
}
|
||||
if cfg.StorageMaxTotalBytes < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
|
||||
}
|
||||
if cfg.StorageUsageRefreshInterval < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be non-negative")
|
||||
}
|
||||
if cfg.StorageRetentionMaxAge < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
|
||||
}
|
||||
if cfg.StorageRetentionEnable && cfg.StorageRetentionMaxAge <= 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_ENABLE is true")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateAdminRBACConfig checks the panel/adminapi permission configuration.
|
||||
// An unparsable permission name is refused rather than ignored: a silently
|
||||
// dropped permission is either a lockout or an unintended grant.
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ type MediaBackend string
|
|||
const (
|
||||
// MediaBackendLocalFS 表示 blob 字节存在本地磁盘(object_key 为相对路径)。
|
||||
MediaBackendLocalFS MediaBackend = "localfs"
|
||||
// MediaBackendS3 表示 blob 字节存在 S3 兼容对象存储(MinIO 或 AWS S3)。
|
||||
MediaBackendS3 MediaBackend = "s3"
|
||||
)
|
||||
|
||||
// FileBlob 是一个可下载的二进制对象的索引项:location_key → 后端/对象键/大小/mime。
|
||||
|
|
@ -233,6 +235,10 @@ type Document struct {
|
|||
DCID int `json:"dc_id,omitempty"`
|
||||
Attributes []DocumentAttribute `json:"attributes,omitempty"`
|
||||
Thumbs []PhotoSize `json:"thumbs,omitempty"`
|
||||
// OwnerUserID is who uploaded this document; 0 for rows predating storage
|
||||
// accounting or for system-originated documents. Used for per-account
|
||||
// storage usage reporting, not part of the tg wire protocol.
|
||||
OwnerUserID int64 `json:"owner_user_id,omitempty"`
|
||||
}
|
||||
|
||||
// StickerSetRef 返回该文档归属的贴纸集引用(若有 sticker/custom_emoji 属性)。
|
||||
|
|
@ -384,6 +390,9 @@ type Photo struct {
|
|||
DCID int `json:"dc_id,omitempty"`
|
||||
HasStickers bool `json:"has_stickers,omitempty"`
|
||||
Sizes []PhotoSize `json:"sizes,omitempty"`
|
||||
// OwnerUserID is who uploaded this photo; 0 for rows predating storage
|
||||
// accounting or for system-originated photos (e.g. webpage previews).
|
||||
OwnerUserID int64 `json:"owner_user_id,omitempty"`
|
||||
}
|
||||
|
||||
func ClonePhotoPtr(photo *Photo) *Photo {
|
||||
|
|
|
|||
|
|
@ -11,4 +11,8 @@ var (
|
|||
ErrUploadQuotaExceeded = errors.New("upload quota exceeded")
|
||||
ErrPhotoInvalid = errors.New("photo invalid")
|
||||
ErrDocumentInvalid = errors.New("document invalid")
|
||||
// ErrStorageFull is returned when the configured low-space guard rejects a
|
||||
// write: local disk free bytes (or, on the s3 backend, the configured
|
||||
// total-bytes budget) has fallen below the configured threshold.
|
||||
ErrStorageFull = errors.New("storage full")
|
||||
)
|
||||
|
|
|
|||
90
internal/domain/media_refs.go
Normal file
90
internal/domain/media_refs.go
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
package domain
|
||||
|
||||
// MediaKind identifies which table a media_references row points into.
|
||||
type MediaKind string
|
||||
|
||||
const (
|
||||
MediaKindDocument MediaKind = "document"
|
||||
MediaKindPhoto MediaKind = "photo"
|
||||
)
|
||||
|
||||
// MediaRefKind identifies why a document/photo is still considered "live" --
|
||||
// each distinct kind of place that can hold a durable pointer to media gets
|
||||
// its own ref_kind, so removing one kind of reference (e.g. a deleted
|
||||
// message) doesn't accidentally drop a still-live reference of another kind
|
||||
// (e.g. the same document also set as someone's profile photo).
|
||||
type MediaRefKind string
|
||||
|
||||
const (
|
||||
MediaRefKindMessageBox MediaRefKind = "message_box"
|
||||
MediaRefKindChannelMessage MediaRefKind = "channel_message"
|
||||
MediaRefKindProfilePhoto MediaRefKind = "profile_photo"
|
||||
MediaRefKindStickerSet MediaRefKind = "sticker_set"
|
||||
MediaRefKindGift MediaRefKind = "gift"
|
||||
)
|
||||
|
||||
// MediaReference is one live pointer to a document/photo. GC (the storage
|
||||
// retention sweep) only considers a document/photo eligible for deletion
|
||||
// once every reference to it has been removed.
|
||||
type MediaReference struct {
|
||||
Kind MediaKind
|
||||
MediaID int64
|
||||
RefKind MediaRefKind
|
||||
RefKey string
|
||||
}
|
||||
|
||||
// OrphanCandidate is a document/photo whose last reference has been removed
|
||||
// (orphaned_at is set) and is old enough to be considered for the storage
|
||||
// retention sweep.
|
||||
type OrphanCandidate struct {
|
||||
Kind MediaKind
|
||||
MediaID int64
|
||||
Backend MediaBackend
|
||||
ObjectKey string
|
||||
Size int64
|
||||
OrphanedAt int64 // unix seconds
|
||||
}
|
||||
|
||||
// MediaRefTarget identifies one document/photo embedded in a message's media
|
||||
// snapshot.
|
||||
type MediaRefTarget struct {
|
||||
Kind MediaKind
|
||||
ID int64
|
||||
}
|
||||
|
||||
// ExtractMediaRefTargets returns every document/photo id embedded in a
|
||||
// message's media snapshot (including nested ones -- a live photo's video
|
||||
// document, a webpage preview's photo), deduplicated. Used to register/drop
|
||||
// media_references rows when a message carrying this media is
|
||||
// created/edited/deleted.
|
||||
func ExtractMediaRefTargets(media *MessageMedia) []MediaRefTarget {
|
||||
if media == nil {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[MediaRefTarget]struct{}, 2)
|
||||
var out []MediaRefTarget
|
||||
add := func(kind MediaKind, id int64) {
|
||||
if id == 0 {
|
||||
return
|
||||
}
|
||||
t := MediaRefTarget{Kind: kind, ID: id}
|
||||
if _, ok := seen[t]; ok {
|
||||
return
|
||||
}
|
||||
seen[t] = struct{}{}
|
||||
out = append(out, t)
|
||||
}
|
||||
if media.Photo != nil {
|
||||
add(MediaKindPhoto, media.Photo.ID)
|
||||
}
|
||||
if media.Document != nil {
|
||||
add(MediaKindDocument, media.Document.ID)
|
||||
}
|
||||
if media.LivePhotoVideo != nil {
|
||||
add(MediaKindDocument, media.LivePhotoVideo.ID)
|
||||
}
|
||||
if media.WebPage != nil && media.WebPage.Photo != nil {
|
||||
add(MediaKindPhoto, media.WebPage.Photo.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
@ -107,6 +107,11 @@ func locationInvalidErr() error { return tgerr.New(400, "LOCATION_INVALID")
|
|||
func fileIDInvalidErr() error { return tgerr.New(400, "FILE_ID_INVALID") }
|
||||
func documentInvalidErr() error { return tgerr.New(400, "DOCUMENT_INVALID") }
|
||||
|
||||
// storageFullErr surfaces the low-disk-space upload guard. Deliberately not
|
||||
// a flood-wait: a full disk won't resolve itself in 60 seconds, and telling
|
||||
// the client to retry shortly would be misleading.
|
||||
func storageFullErr() error { return tgerr.New(400, "STORAGE_FULL") }
|
||||
|
||||
func mediaEmptyErr() error { return tgerr.New(400, "MEDIA_EMPTY") }
|
||||
|
||||
func frozenMethodInvalidErr() error { return tgerr.New(420, "FROZEN_METHOD_INVALID") }
|
||||
|
|
|
|||
|
|
@ -684,6 +684,8 @@ func photoUploadErr(err error) error {
|
|||
return filePartsInvalidErr()
|
||||
case errors.Is(err, domain.ErrPhotoInvalid):
|
||||
return photoInvalidErr()
|
||||
case errors.Is(err, domain.ErrStorageFull):
|
||||
return storageFullErr()
|
||||
default:
|
||||
return internalErr()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1144,6 +1144,8 @@ func mediaUploadErr(err error) error {
|
|||
return photoInvalidErr()
|
||||
case errors.Is(err, domain.ErrDocumentInvalid):
|
||||
return mediaInvalidErr()
|
||||
case errors.Is(err, domain.ErrStorageFull):
|
||||
return storageFullErr()
|
||||
default:
|
||||
return internalErr()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -327,6 +327,8 @@ func fileSaveErr(err error) error {
|
|||
return filePartTooBigErr()
|
||||
case errors.Is(err, domain.ErrUploadQuotaExceeded):
|
||||
return floodWaitErr(60)
|
||||
case errors.Is(err, domain.ErrStorageFull):
|
||||
return storageFullErr()
|
||||
default:
|
||||
return internalErr()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,10 @@ type MediaStore interface {
|
|||
// GetFileBlobs 批量按 location_key 取 FileBlob 元数据(缺失的 key 不出现在返回 map 中)。
|
||||
// 供启动预热等需一次性加载大量 blob 的路径,替代逐个 GetFileBlob 的 N+1 往返。
|
||||
GetFileBlobs(ctx context.Context, locationKeys []string) (map[string]domain.FileBlob, error)
|
||||
// SumFileBlobBytes 返回全部 blob 的物理字节总量(去重后,同内容只算一次)。
|
||||
// 供低磁盘空间守卫的周期性用量刷新使用(尤其是 s3 backend 的预算模式,没有
|
||||
// 操作系统级"剩余空间"概念,只能靠这个累计值和配置的预算比较)。
|
||||
SumFileBlobBytes(ctx context.Context) (int64, error)
|
||||
|
||||
// seed 状态。只记录静态资源 catalog 的内容 hash,用于启动时跳过未变化的重复导入;
|
||||
// 真实可服务性仍由 documents/file_blobs 校验保证,不能只相信这里的 hash。
|
||||
|
|
|
|||
|
|
@ -479,6 +479,11 @@ ORDER BY id`, channel.ID, id32)
|
|||
// 删除入口统一静默跳过(官方客户端对它禁用删除)。
|
||||
continue
|
||||
}
|
||||
// Drop this message's media_references (storage GC); orphans the
|
||||
// document/photo if this was its last live reference anywhere.
|
||||
if err := removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindChannelMessage, channelMessageRefKey(channel.ID, id)); err != nil {
|
||||
return nil, domain.ChannelUpdateEvent{}, channel, fmt.Errorf("remove deleted channel message media references: %w", err)
|
||||
}
|
||||
deleted = append(deleted, id)
|
||||
if err := s.insertChannelAdminLogTx(ctx, tx, domain.ChannelAdminLogEvent{
|
||||
ChannelID: channel.ID,
|
||||
|
|
|
|||
|
|
@ -272,6 +272,10 @@ WHERE location_key = ANY($1::text[])`, locationKeys)
|
|||
return out, nil
|
||||
}
|
||||
|
||||
func (s *MediaStore) SumFileBlobBytes(ctx context.Context) (int64, error) {
|
||||
return s.q.SumFileBlobBytes(ctx)
|
||||
}
|
||||
|
||||
func (s *MediaStore) GetSeedState(ctx context.Context, key string) (string, bool, error) {
|
||||
var hash string
|
||||
if err := s.db.QueryRow(ctx, `
|
||||
|
|
@ -332,6 +336,7 @@ func putDocumentParams(doc domain.Document) (sqlcgen.PutDocumentParams, error) {
|
|||
DcID: int32(doc.DCID),
|
||||
AttributesJson: attrs,
|
||||
ThumbsJson: thumbs,
|
||||
OwnerUserID: doc.OwnerUserID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -472,6 +477,20 @@ func (c *documentMetaCache) put(id int64, doc domain.Document) {
|
|||
}
|
||||
}
|
||||
|
||||
// remove evicts id, e.g. after the row is permanently deleted (storage
|
||||
// retention sweep) so a stale cache hit can't outlive the row.
|
||||
func (c *documentMetaCache) remove(id int64) {
|
||||
if c == nil || id == 0 {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if el, ok := c.m[id]; ok {
|
||||
c.ll.Remove(el)
|
||||
delete(c.m, id)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneDocument(doc domain.Document) domain.Document {
|
||||
doc.FileReference = append([]byte(nil), doc.FileReference...)
|
||||
if len(doc.Attributes) > 0 {
|
||||
|
|
@ -514,6 +533,7 @@ func documentFromRow(row sqlcgen.GetDocumentRow) (domain.Document, error) {
|
|||
DCID: int(row.DcID),
|
||||
Attributes: attrs,
|
||||
Thumbs: thumbs,
|
||||
OwnerUserID: row.OwnerUserID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -532,6 +552,7 @@ func (s *MediaStore) PutPhoto(ctx context.Context, photo domain.Photo) error {
|
|||
DcID: int32(photo.DCID),
|
||||
HasStickers: photo.HasStickers,
|
||||
SizesJson: sizes,
|
||||
OwnerUserID: photo.OwnerUserID,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -543,7 +564,7 @@ func (s *MediaStore) GetPhoto(ctx context.Context, id int64) (domain.Photo, bool
|
|||
}
|
||||
return domain.Photo{}, false, err
|
||||
}
|
||||
photo, err := photoFromFields(row.ID, row.AccessHash, row.FileReference, int(row.Date), int(row.DcID), row.HasStickers, row.SizesJson)
|
||||
photo, err := photoFromFields(row.ID, row.AccessHash, row.FileReference, int(row.Date), int(row.DcID), row.HasStickers, row.SizesJson, row.OwnerUserID)
|
||||
if err != nil {
|
||||
return domain.Photo{}, false, err
|
||||
}
|
||||
|
|
@ -572,7 +593,7 @@ func (s *MediaStore) GetPhotos(ctx context.Context, ids []int64) ([]domain.Photo
|
|||
return nil, nil
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, access_hash, file_reference, date, dc_id, has_stickers, sizes::text
|
||||
SELECT id, access_hash, file_reference, date, dc_id, has_stickers, sizes::text, owner_user_id
|
||||
FROM photos
|
||||
WHERE id = ANY($1::bigint[])
|
||||
`, unique)
|
||||
|
|
@ -613,14 +634,15 @@ func scanPhotoRow(row photoScanner) (domain.Photo, error) {
|
|||
dcID int32
|
||||
hasStickers bool
|
||||
sizesJSON string
|
||||
ownerUserID int64
|
||||
)
|
||||
if err := row.Scan(&id, &accessHash, &fileReference, &date, &dcID, &hasStickers, &sizesJSON); err != nil {
|
||||
if err := row.Scan(&id, &accessHash, &fileReference, &date, &dcID, &hasStickers, &sizesJSON, &ownerUserID); err != nil {
|
||||
return domain.Photo{}, err
|
||||
}
|
||||
return photoFromFields(id, accessHash, fileReference, int(date), int(dcID), hasStickers, sizesJSON)
|
||||
return photoFromFields(id, accessHash, fileReference, int(date), int(dcID), hasStickers, sizesJSON, ownerUserID)
|
||||
}
|
||||
|
||||
func photoFromFields(id, accessHash int64, fileReference []byte, date, dcID int, hasStickers bool, sizesJSON string) (domain.Photo, error) {
|
||||
func photoFromFields(id, accessHash int64, fileReference []byte, date, dcID int, hasStickers bool, sizesJSON string, ownerUserID int64) (domain.Photo, error) {
|
||||
sizes, err := decodePhotoSizes(sizesJSON)
|
||||
if err != nil {
|
||||
return domain.Photo{}, err
|
||||
|
|
@ -633,6 +655,7 @@ func photoFromFields(id, accessHash int64, fileReference []byte, date, dcID int,
|
|||
DCID: dcID,
|
||||
HasStickers: hasStickers,
|
||||
Sizes: sizes,
|
||||
OwnerUserID: ownerUserID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -697,7 +720,7 @@ func (s *MediaStore) CreateStickerSet(ctx context.Context, set domain.StickerSet
|
|||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return addStickerSetMediaReferencesTx(ctx, qtx, set.ID, docs)
|
||||
})
|
||||
if err != nil {
|
||||
if stickerSetShortNameConflict(err) {
|
||||
|
|
@ -726,7 +749,13 @@ func (s *MediaStore) UpdateStickerSet(ctx context.Context, set domain.StickerSet
|
|||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
// Re-register from scratch: drops references for any document no
|
||||
// longer in the set (candidate for storage GC once orphaned long
|
||||
// enough) and refreshes the rest.
|
||||
if err := removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindStickerSet, stickerSetRefKey(set.ID)); err != nil {
|
||||
return err
|
||||
}
|
||||
return addStickerSetMediaReferencesTx(ctx, qtx, set.ID, docs)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -751,8 +780,10 @@ WHERE id = $1
|
|||
if tag.RowsAffected() == 0 {
|
||||
return domain.ErrStickerSetInvalid
|
||||
}
|
||||
_, err = tx.Exec(ctx, `DELETE FROM user_sticker_sets WHERE sticker_set_id = $1`, setID)
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM user_sticker_sets WHERE sticker_set_id = $1`, setID); err != nil {
|
||||
return err
|
||||
}
|
||||
return removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindStickerSet, stickerSetRefKey(setID))
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -769,11 +800,40 @@ WHERE id = $1
|
|||
if tag.RowsAffected() == 0 {
|
||||
return domain.ErrStickerSetInvalid
|
||||
}
|
||||
_, err = tx.Exec(ctx, `DELETE FROM user_sticker_sets WHERE sticker_set_id = $1`, setID)
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM user_sticker_sets WHERE sticker_set_id = $1`, setID); err != nil {
|
||||
return err
|
||||
}
|
||||
return removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindStickerSet, stickerSetRefKey(setID))
|
||||
})
|
||||
}
|
||||
|
||||
func stickerSetRefKey(setID int64) string {
|
||||
return fmt.Sprintf("stickerset:%d", setID)
|
||||
}
|
||||
|
||||
// addStickerSetMediaReferencesTx registers every document belonging to a
|
||||
// sticker set as referenced (storage GC), clearing orphaned_at on each.
|
||||
func addStickerSetMediaReferencesTx(ctx context.Context, qtx *sqlcgen.Queries, setID int64, docs []domain.Document) error {
|
||||
refKey := stickerSetRefKey(setID)
|
||||
for _, doc := range docs {
|
||||
if doc.ID == 0 {
|
||||
continue
|
||||
}
|
||||
if err := qtx.InsertMediaReference(ctx, sqlcgen.InsertMediaReferenceParams{
|
||||
MediaKind: string(domain.MediaKindDocument),
|
||||
MediaID: doc.ID,
|
||||
RefKind: string(domain.MediaRefKindStickerSet),
|
||||
RefKey: refKey,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("register sticker set media reference: %w", err)
|
||||
}
|
||||
if err := qtx.ClearDocumentOrphan(ctx, doc.ID); err != nil {
|
||||
return fmt.Errorf("clear sticker document orphan: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func insertStickerSet(ctx context.Context, db sqlcgen.DBTX, set domain.StickerSet) error {
|
||||
thumbs, err := jsonArrayOrEmpty(set.Thumbs)
|
||||
if err != nil {
|
||||
|
|
@ -1197,16 +1257,30 @@ func (s *MediaStore) AddProfilePhotoKind(ctx context.Context, ownerType domain.P
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.Exec(ctx, `
|
||||
if _, err := s.db.Exec(ctx, `
|
||||
INSERT INTO profile_photos (owner_peer_type, owner_peer_id, kind, photo_id, date, active, sort_order)
|
||||
VALUES ($1, $2, $3, $4, $5, true, $6)
|
||||
ON CONFLICT (owner_peer_type, owner_peer_id, kind, photo_id) DO UPDATE SET
|
||||
date = EXCLUDED.date,
|
||||
active = true,
|
||||
sort_order = EXCLUDED.sort_order
|
||||
`, string(ownerType), ownerID, string(kind), photoID, date, next+1)
|
||||
`, string(ownerType), ownerID, string(kind), photoID, date, next+1); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.q.InsertMediaReference(ctx, sqlcgen.InsertMediaReferenceParams{
|
||||
MediaKind: string(domain.MediaKindPhoto),
|
||||
MediaID: photoID,
|
||||
RefKind: string(domain.MediaRefKindProfilePhoto),
|
||||
RefKey: profilePhotoRefKey(ownerType, ownerID, kind, photoID),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("register profile photo reference: %w", err)
|
||||
}
|
||||
return s.q.ClearPhotoOrphan(ctx, photoID)
|
||||
}
|
||||
|
||||
func profilePhotoRefKey(ownerType domain.PeerType, ownerID int64, kind domain.ProfilePhotoKind, photoID int64) string {
|
||||
return fmt.Sprintf("%s:%d:kind:%s:photo:%d", ownerType, ownerID, kind, photoID)
|
||||
}
|
||||
|
||||
func (s *MediaStore) CurrentProfilePhotoKind(ctx context.Context, ownerType domain.PeerType, ownerID int64, kind domain.ProfilePhotoKind) (int64, bool, error) {
|
||||
kind = normalizeProfilePhotoKind(kind)
|
||||
|
|
@ -1336,7 +1410,7 @@ func (s *MediaStore) ListProfilePhotoDetailsKind(ctx context.Context, ownerType
|
|||
var err error
|
||||
if offset < 0 && maxID > 0 {
|
||||
rows, err = s.db.Query(ctx, `
|
||||
SELECT ph.id, ph.access_hash, ph.file_reference, ph.date, ph.dc_id, ph.has_stickers, ph.sizes::text AS sizes_json
|
||||
SELECT ph.id, ph.access_hash, ph.file_reference, ph.date, ph.dc_id, ph.has_stickers, ph.sizes::text AS sizes_json, ph.owner_user_id
|
||||
FROM profile_photos pp
|
||||
JOIN photos ph ON ph.id = pp.photo_id
|
||||
WHERE pp.owner_peer_type = $1
|
||||
|
|
@ -1352,7 +1426,7 @@ LIMIT $5
|
|||
offset = 0
|
||||
}
|
||||
rows, err = s.db.Query(ctx, `
|
||||
SELECT ph.id, ph.access_hash, ph.file_reference, ph.date, ph.dc_id, ph.has_stickers, ph.sizes::text AS sizes_json
|
||||
SELECT ph.id, ph.access_hash, ph.file_reference, ph.date, ph.dc_id, ph.has_stickers, ph.sizes::text AS sizes_json, ph.owner_user_id
|
||||
FROM profile_photos pp
|
||||
JOIN photos ph ON ph.id = pp.photo_id
|
||||
WHERE pp.owner_peer_type = $1
|
||||
|
|
@ -1429,6 +1503,19 @@ RETURNING photo_id
|
|||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, id := range deleted {
|
||||
if err := s.q.RemoveMediaReference(ctx, sqlcgen.RemoveMediaReferenceParams{
|
||||
MediaKind: string(domain.MediaKindPhoto),
|
||||
MediaID: id,
|
||||
RefKind: string(domain.MediaRefKindProfilePhoto),
|
||||
RefKey: profilePhotoRefKey(ownerType, ownerID, kind, id),
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("remove profile photo reference: %w", err)
|
||||
}
|
||||
if err := s.q.OrphanPhotoIfUnreferenced(ctx, id); err != nil {
|
||||
return nil, fmt.Errorf("orphan check profile photo: %w", err)
|
||||
}
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ import (
|
|||
// 写路径只在「创建」和「编辑改媒体」两处维护,删除靠读查询 JOIN 过滤 deleted、不在此维护。
|
||||
|
||||
// insertChannelMediaIndexTx 为一条频道消息按其媒体类别写索引行(无类别则 no-op)。
|
||||
// 同时登记该消息内嵌 document/photo 的 media_references(存储回收用),与分类
|
||||
// 索引共用同一事务。
|
||||
func insertChannelMediaIndexTx(ctx context.Context, tx pgx.Tx, channelID int64, id, date int, media *domain.MessageMedia, entities []domain.MessageEntity) error {
|
||||
for _, c := range domain.ClassifyMediaCategories(media, entities) {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
|
|
@ -22,15 +24,18 @@ ON CONFLICT (channel_id, id, category) DO NOTHING`, channelID, id, int16(c), dat
|
|||
return fmt.Errorf("insert channel media index: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindChannelMessage, channelMessageRefKey(channelID, id))
|
||||
}
|
||||
|
||||
// deleteChannelMediaIndexTx 清掉一条频道消息的全部索引行(编辑改媒体前先清后插)。
|
||||
// 只在 replaceChannelMediaIndexTx 内被调用;真正的消息删除不清 *_media 分类索引
|
||||
// (读时靠 JOIN deleted 过滤,见文件头注释),但仍需在此清掉 media_references,
|
||||
// 否则 replace 场景下旧媒体永远不会被判定为孤儿。
|
||||
func deleteChannelMediaIndexTx(ctx context.Context, tx pgx.Tx, channelID int64, id int) error {
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM channel_message_media WHERE channel_id = $1 AND id = $2`, channelID, id); err != nil {
|
||||
return fmt.Errorf("delete channel media index: %w", err)
|
||||
}
|
||||
return nil
|
||||
return removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindChannelMessage, channelMessageRefKey(channelID, id))
|
||||
}
|
||||
|
||||
// replaceChannelMediaIndexTx 在编辑替换媒体后重建索引行(类别可能变化)。
|
||||
|
|
@ -41,7 +46,8 @@ func replaceChannelMediaIndexTx(ctx context.Context, tx pgx.Tx, channelID int64,
|
|||
return insertChannelMediaIndexTx(ctx, tx, channelID, id, date, media, entities)
|
||||
}
|
||||
|
||||
// insertMessageBoxMediaIndexTx 为一条私聊 owner box 按其媒体类别写索引行。
|
||||
// insertMessageBoxMediaIndexTx 为一条私聊 owner box 按其媒体类别写索引行。同时登记
|
||||
// media_references(存储回收用)。
|
||||
func insertMessageBoxMediaIndexTx(ctx context.Context, tx pgx.Tx, ownerUserID, peerID int64, boxID, date int, media *domain.MessageMedia, entities []domain.MessageEntity) error {
|
||||
for _, c := range domain.ClassifyMediaCategories(media, entities) {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
|
|
@ -51,15 +57,18 @@ ON CONFLICT (owner_user_id, box_id, category) DO NOTHING`, ownerUserID, boxID, p
|
|||
return fmt.Errorf("insert message box media index: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindMessageBox, messageBoxRefKey(ownerUserID, boxID))
|
||||
}
|
||||
|
||||
// deleteMessageBoxMediaIndexTx 清掉一条私聊 owner box 的全部索引行。
|
||||
// deleteMessageBoxMediaIndexTx 清掉一条私聊 owner box 的全部索引行。只在
|
||||
// replaceMessageBoxMediaIndexTx 内被调用(编辑改媒体前先清后插);真正的消息
|
||||
// 删除不清 message_box_media(读时靠 JOIN deleted 过滤),但仍需在此清掉
|
||||
// media_references,否则 replace 场景下旧媒体永远不会被判定为孤儿。
|
||||
func deleteMessageBoxMediaIndexTx(ctx context.Context, tx pgx.Tx, ownerUserID int64, boxID int) error {
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM message_box_media WHERE owner_user_id = $1 AND box_id = $2`, ownerUserID, boxID); err != nil {
|
||||
return fmt.Errorf("delete message box media index: %w", err)
|
||||
}
|
||||
return nil
|
||||
return removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindMessageBox, messageBoxRefKey(ownerUserID, boxID))
|
||||
}
|
||||
|
||||
// replaceMessageBoxMediaIndexTx 在编辑替换媒体后重建索引行。
|
||||
|
|
|
|||
212
internal/store/postgres/media_refs.go
Normal file
212
internal/store/postgres/media_refs.go
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
"telesrv/internal/store/postgres/sqlcgen"
|
||||
)
|
||||
|
||||
// messageBoxRefKey/channelMessageRefKey are the ref_key encodings used by
|
||||
// media_references rows registered from the private-mailbox and channel
|
||||
// message write paths, respectively. Kept as named helpers so the add and
|
||||
// remove sides can never drift apart on format.
|
||||
func messageBoxRefKey(ownerUserID int64, boxID int) string {
|
||||
return fmt.Sprintf("user:%d:box:%d", ownerUserID, boxID)
|
||||
}
|
||||
|
||||
func channelMessageRefKey(channelID int64, messageID int) string {
|
||||
return fmt.Sprintf("channel:%d:msg:%d", channelID, messageID)
|
||||
}
|
||||
|
||||
// addMediaReferencesTx registers every document/photo embedded in media as
|
||||
// referenced by refKind/refKey, clearing orphaned_at on each if it had been
|
||||
// set by an earlier removal. Must run in the same transaction as the write
|
||||
// that creates the reference (message send/edit).
|
||||
func addMediaReferencesTx(ctx context.Context, tx sqlcgen.DBTX, media *domain.MessageMedia, refKind domain.MediaRefKind, refKey string) error {
|
||||
targets := domain.ExtractMediaRefTargets(media)
|
||||
if len(targets) == 0 {
|
||||
return nil
|
||||
}
|
||||
q := sqlcgen.New(tx)
|
||||
for _, t := range targets {
|
||||
if err := q.InsertMediaReference(ctx, sqlcgen.InsertMediaReferenceParams{
|
||||
MediaKind: string(t.Kind),
|
||||
MediaID: t.ID,
|
||||
RefKind: string(refKind),
|
||||
RefKey: refKey,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("insert media reference: %w", err)
|
||||
}
|
||||
var clearErr error
|
||||
switch t.Kind {
|
||||
case domain.MediaKindDocument:
|
||||
clearErr = q.ClearDocumentOrphan(ctx, t.ID)
|
||||
case domain.MediaKindPhoto:
|
||||
clearErr = q.ClearPhotoOrphan(ctx, t.ID)
|
||||
}
|
||||
if clearErr != nil {
|
||||
return fmt.Errorf("clear media orphan: %w", clearErr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeMediaReferencesByKeyTx drops every media_references row registered
|
||||
// under refKind/refKey (no need to know which document/photo ids those were
|
||||
// -- the delete finds them) and, for each one that becomes fully
|
||||
// unreferenced as a result, marks it orphaned so the storage retention
|
||||
// sweep can consider it once old enough. Must run in the same transaction
|
||||
// as the write that removes the reference (a message being soft-deleted).
|
||||
func removeMediaReferencesByKeyTx(ctx context.Context, tx sqlcgen.DBTX, refKind domain.MediaRefKind, refKey string) error {
|
||||
rows, err := tx.Query(ctx, `
|
||||
DELETE FROM media_references
|
||||
WHERE ref_kind = $1 AND ref_key = $2
|
||||
RETURNING media_kind, media_id`, string(refKind), refKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("remove media references: %w", err)
|
||||
}
|
||||
type removedRef struct {
|
||||
kind string
|
||||
id int64
|
||||
}
|
||||
var removed []removedRef
|
||||
for rows.Next() {
|
||||
var r removedRef
|
||||
if err := rows.Scan(&r.kind, &r.id); err != nil {
|
||||
rows.Close()
|
||||
return fmt.Errorf("scan removed media reference: %w", err)
|
||||
}
|
||||
removed = append(removed, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("remove media references: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
q := sqlcgen.New(tx)
|
||||
for _, r := range removed {
|
||||
var orphanErr error
|
||||
switch domain.MediaKind(r.kind) {
|
||||
case domain.MediaKindDocument:
|
||||
orphanErr = q.OrphanDocumentIfUnreferenced(ctx, r.id)
|
||||
case domain.MediaKindPhoto:
|
||||
orphanErr = q.OrphanPhotoIfUnreferenced(ctx, r.id)
|
||||
}
|
||||
if orphanErr != nil {
|
||||
return fmt.Errorf("orphan check media: %w", orphanErr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---- storage retention sweep ----
|
||||
|
||||
// ListOrphanedDocumentIDsOlderThan returns document ids whose orphaned_at is
|
||||
// set and older than cutoff, oldest first, up to limit.
|
||||
func (s *MediaStore) ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListOrphanedDocumentIDsOlderThan(ctx, sqlcgen.ListOrphanedDocumentIDsOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// ListOrphanedPhotoIDsOlderThan returns photo ids whose orphaned_at is set
|
||||
// and older than cutoff, oldest first, up to limit.
|
||||
func (s *MediaStore) ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListOrphanedPhotoIDsOlderThan(ctx, sqlcgen.ListOrphanedPhotoIDsOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// CountFileBlobRefs reports how many file_blobs rows still point at
|
||||
// (backend, objectKey) -- the caller must not physically delete the object
|
||||
// from that backend while this is > 0 (content-addressed storage: the same
|
||||
// object can be shared by multiple documents/photos).
|
||||
func (s *MediaStore) CountFileBlobRefs(ctx context.Context, backend, objectKey string) (int, error) {
|
||||
n, err := s.q.CountFileBlobRefs(ctx, sqlcgen.CountFileBlobRefsParams{Backend: backend, ObjectKey: objectKey})
|
||||
return int(n), err
|
||||
}
|
||||
|
||||
// DeleteDocumentAndBlobs deletes a document row and every file_blobs row it
|
||||
// owns (main body + thumbnail variants), returning what was deleted so the
|
||||
// caller can physically remove each object from its backend once confirming
|
||||
// (via CountFileBlobRefs, after this call) no other row still needs it.
|
||||
// Assumes the document is already orphaned -- does not check references.
|
||||
func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error) {
|
||||
var blobs []domain.FileBlob
|
||||
err := withTx(ctx, s.db, "delete document and blobs", func(tx pgx.Tx) error {
|
||||
qtx := s.q.WithTx(tx)
|
||||
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
|
||||
ExactKey: fmt.Sprintf("doc:%d", id),
|
||||
PrefixPattern: fmt.Sprintf("doc:%d:%%", id),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list document blobs: %w", err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
blobs = append(blobs, domain.FileBlob{
|
||||
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
|
||||
})
|
||||
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
|
||||
return fmt.Errorf("delete file blob row: %w", err)
|
||||
}
|
||||
}
|
||||
if err := qtx.DeleteDocumentRow(ctx, id); err != nil {
|
||||
return fmt.Errorf("delete document row: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.documents.remove(id)
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
|
||||
// (one per rendition size), returning what was deleted so the caller can
|
||||
// physically remove each object from its backend once confirming (via
|
||||
// CountFileBlobRefs, after this call) no other row still needs it. Assumes
|
||||
// the photo is already orphaned -- does not check references.
|
||||
func (s *MediaStore) DeletePhotoAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error) {
|
||||
var blobs []domain.FileBlob
|
||||
err := withTx(ctx, s.db, "delete photo and blobs", func(tx pgx.Tx) error {
|
||||
qtx := s.q.WithTx(tx)
|
||||
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
|
||||
ExactKey: fmt.Sprintf("photo:%d", id),
|
||||
PrefixPattern: fmt.Sprintf("photo:%d:%%", id),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list photo blobs: %w", err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
blobs = append(blobs, domain.FileBlob{
|
||||
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
|
||||
})
|
||||
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
|
||||
return fmt.Errorf("delete file blob row: %w", err)
|
||||
}
|
||||
}
|
||||
if err := qtx.DeletePhotoRow(ctx, id); err != nil {
|
||||
return fmt.Errorf("delete photo row: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return blobs, nil
|
||||
}
|
||||
109
internal/store/postgres/media_refs_integration_test.go
Normal file
109
internal/store/postgres/media_refs_integration_test.go
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// TestMediaReferenceOrphanTransitions proves the core storage-retention
|
||||
// safety invariant: a document's orphaned_at is set only once every
|
||||
// reference to it is gone, and cleared the instant a new one appears --
|
||||
// so the retention sweep never targets media still visible in a
|
||||
// conversation, regardless of how many places reference it.
|
||||
func TestMediaReferenceOrphanTransitions(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
s := NewMediaStore(pool)
|
||||
|
||||
const docID = int64(9100000000000000101)
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM media_references WHERE media_kind = 'document' AND media_id = $1`, docID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM documents WHERE id = $1`, docID)
|
||||
})
|
||||
|
||||
if err := s.PutDocument(ctx, domain.Document{ID: docID, MimeType: "text/plain", Size: 10}); err != nil {
|
||||
t.Fatalf("put document: %v", err)
|
||||
}
|
||||
|
||||
media := &domain.MessageMedia{Kind: domain.MessageMediaKindDocument, Document: &domain.Document{ID: docID}}
|
||||
|
||||
// A freshly created document has no orphaned_at yet either way -- it's
|
||||
// simply unreferenced until a message send registers the first
|
||||
// reference, at which point normal tracking takes over.
|
||||
orphaned, err := documentOrphanedAt(ctx, pool, docID)
|
||||
if err != nil {
|
||||
t.Fatalf("query orphaned_at: %v", err)
|
||||
}
|
||||
if orphaned {
|
||||
t.Fatal("expected a freshly inserted document to not be marked orphaned yet")
|
||||
}
|
||||
|
||||
// Adding a reference (as if a message carrying it was sent) clears it.
|
||||
mustAddRef(t, pool, media, domain.MediaRefKindMessageBox, "user:1:box:1")
|
||||
if orphaned, err := documentOrphanedAt(ctx, pool, docID); err != nil || orphaned {
|
||||
t.Fatalf("expected referenced document to not be orphaned, orphaned=%v err=%v", orphaned, err)
|
||||
}
|
||||
|
||||
// A second, independent reference (e.g. forwarded to another box).
|
||||
mustAddRef(t, pool, media, domain.MediaRefKindMessageBox, "user:2:box:5")
|
||||
|
||||
// Removing only one of the two references must NOT orphan the document.
|
||||
mustRemoveRefsByKey(t, pool, domain.MediaRefKindMessageBox, "user:1:box:1")
|
||||
if orphaned, err := documentOrphanedAt(ctx, pool, docID); err != nil || orphaned {
|
||||
t.Fatalf("expected document with a remaining reference to survive, orphaned=%v err=%v", orphaned, err)
|
||||
}
|
||||
|
||||
// Removing the last reference orphans it.
|
||||
mustRemoveRefsByKey(t, pool, domain.MediaRefKindMessageBox, "user:2:box:5")
|
||||
if orphaned, err := documentOrphanedAt(ctx, pool, docID); err != nil || !orphaned {
|
||||
t.Fatalf("expected document with no remaining reference to be orphaned, orphaned=%v err=%v", orphaned, err)
|
||||
}
|
||||
|
||||
// A reference reappearing after orphaning (e.g. re-sent) clears it again.
|
||||
mustAddRef(t, pool, media, domain.MediaRefKindMessageBox, "user:3:box:9")
|
||||
if orphaned, err := documentOrphanedAt(ctx, pool, docID); err != nil || orphaned {
|
||||
t.Fatalf("expected re-referenced document to no longer be orphaned, orphaned=%v err=%v", orphaned, err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustAddRef(t *testing.T, pool *pgxpool.Pool, media *domain.MessageMedia, refKind domain.MediaRefKind, refKey string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("begin tx: %v", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if err := addMediaReferencesTx(ctx, tx, media, refKind, refKey); err != nil {
|
||||
t.Fatalf("add media reference: %v", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
t.Fatalf("commit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRemoveRefsByKey(t *testing.T, pool *pgxpool.Pool, refKind domain.MediaRefKind, refKey string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("begin tx: %v", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if err := removeMediaReferencesByKeyTx(ctx, tx, refKind, refKey); err != nil {
|
||||
t.Fatalf("remove media references: %v", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
t.Fatalf("commit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func documentOrphanedAt(ctx context.Context, pool *pgxpool.Pool, id int64) (bool, error) {
|
||||
var orphaned bool
|
||||
err := pool.QueryRow(ctx, `SELECT orphaned_at IS NOT NULL FROM documents WHERE id = $1`, id).Scan(&orphaned)
|
||||
return orphaned, err
|
||||
}
|
||||
|
|
@ -174,6 +174,11 @@ func (s *MessageStore) finishDeleteMessagesTx(ctx context.Context, db sqlcgen.DB
|
|||
if row.ownerUserID == 0 || row.boxID == 0 {
|
||||
continue
|
||||
}
|
||||
// Drop this box's media_references (storage GC); orphans the
|
||||
// document/photo if this was its last live reference anywhere.
|
||||
if err := removeMediaReferencesByKeyTx(ctx, db, domain.MediaRefKindMessageBox, messageBoxRefKey(row.ownerUserID, row.boxID)); err != nil {
|
||||
return res, fmt.Errorf("remove deleted message media references: %w", err)
|
||||
}
|
||||
idsByOwner[row.ownerUserID] = append(idsByOwner[row.ownerUserID], row.boxID)
|
||||
if row.peer.ID != 0 {
|
||||
if peersByOwner[row.ownerUserID] == nil {
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ WHERE location_key = sqlc.arg(location_key)::text;
|
|||
-- documents -------------------------------------------------------------------
|
||||
|
||||
-- name: PutDocument :exec
|
||||
INSERT INTO documents (id, access_hash, file_reference, date, mime_type, size, dc_id, attributes, thumbs)
|
||||
INSERT INTO documents (id, access_hash, file_reference, date, mime_type, size, dc_id, attributes, thumbs, owner_user_id)
|
||||
VALUES (
|
||||
sqlc.arg(id)::bigint,
|
||||
sqlc.arg(access_hash)::bigint,
|
||||
|
|
@ -118,8 +118,13 @@ VALUES (
|
|||
sqlc.arg(size)::bigint,
|
||||
sqlc.arg(dc_id)::int,
|
||||
sqlc.arg(attributes_json)::jsonb,
|
||||
sqlc.arg(thumbs_json)::jsonb
|
||||
sqlc.arg(thumbs_json)::jsonb,
|
||||
sqlc.arg(owner_user_id)::bigint
|
||||
)
|
||||
-- owner_user_id is intentionally NOT in the UPDATE SET list: a document id is
|
||||
-- only ever (re-)upserted by its original uploader's own request replay, and
|
||||
-- keeping the first-write owner sticky avoids any risk of a later call
|
||||
-- (e.g. a forward re-touching the row) reassigning ownership.
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
access_hash = EXCLUDED.access_hash,
|
||||
file_reference = EXCLUDED.file_reference,
|
||||
|
|
@ -133,21 +138,26 @@ ON CONFLICT (id) DO UPDATE SET
|
|||
-- name: GetDocument :one
|
||||
SELECT id, access_hash, file_reference, date, mime_type, size, dc_id,
|
||||
attributes::text AS attributes_json,
|
||||
thumbs::text AS thumbs_json
|
||||
thumbs::text AS thumbs_json,
|
||||
owner_user_id
|
||||
FROM documents
|
||||
WHERE id = sqlc.arg(id)::bigint;
|
||||
|
||||
-- name: GetDocuments :many
|
||||
SELECT id, access_hash, file_reference, date, mime_type, size, dc_id,
|
||||
attributes::text AS attributes_json,
|
||||
thumbs::text AS thumbs_json
|
||||
thumbs::text AS thumbs_json,
|
||||
owner_user_id
|
||||
FROM documents
|
||||
WHERE id = ANY(sqlc.arg(ids)::bigint[]);
|
||||
|
||||
-- name: DeleteDocumentRow :exec
|
||||
DELETE FROM documents WHERE id = sqlc.arg(id)::bigint;
|
||||
|
||||
-- photos ----------------------------------------------------------------------
|
||||
|
||||
-- name: PutPhoto :exec
|
||||
INSERT INTO photos (id, access_hash, file_reference, date, dc_id, has_stickers, sizes)
|
||||
INSERT INTO photos (id, access_hash, file_reference, date, dc_id, has_stickers, sizes, owner_user_id)
|
||||
VALUES (
|
||||
sqlc.arg(id)::bigint,
|
||||
sqlc.arg(access_hash)::bigint,
|
||||
|
|
@ -155,8 +165,10 @@ VALUES (
|
|||
sqlc.arg(date)::int,
|
||||
sqlc.arg(dc_id)::int,
|
||||
sqlc.arg(has_stickers)::boolean,
|
||||
sqlc.arg(sizes_json)::jsonb
|
||||
sqlc.arg(sizes_json)::jsonb,
|
||||
sqlc.arg(owner_user_id)::bigint
|
||||
)
|
||||
-- owner_user_id intentionally not updated on conflict, see PutDocument.
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
access_hash = EXCLUDED.access_hash,
|
||||
file_reference = EXCLUDED.file_reference,
|
||||
|
|
@ -167,10 +179,87 @@ ON CONFLICT (id) DO UPDATE SET
|
|||
|
||||
-- name: GetPhoto :one
|
||||
SELECT id, access_hash, file_reference, date, dc_id, has_stickers,
|
||||
sizes::text AS sizes_json
|
||||
sizes::text AS sizes_json,
|
||||
owner_user_id
|
||||
FROM photos
|
||||
WHERE id = sqlc.arg(id)::bigint;
|
||||
|
||||
-- name: DeletePhotoRow :exec
|
||||
DELETE FROM photos WHERE id = sqlc.arg(id)::bigint;
|
||||
|
||||
-- media_references / storage retention -----------------------------------------
|
||||
|
||||
-- name: InsertMediaReference :exec
|
||||
INSERT INTO media_references (media_kind, media_id, ref_kind, ref_key)
|
||||
VALUES (sqlc.arg(media_kind)::text, sqlc.arg(media_id)::bigint, sqlc.arg(ref_kind)::text, sqlc.arg(ref_key)::text)
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- name: ClearDocumentOrphan :exec
|
||||
UPDATE documents SET orphaned_at = NULL
|
||||
WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
|
||||
|
||||
-- name: ClearPhotoOrphan :exec
|
||||
UPDATE photos SET orphaned_at = NULL
|
||||
WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
|
||||
|
||||
-- name: RemoveMediaReference :exec
|
||||
DELETE FROM media_references
|
||||
WHERE media_kind = sqlc.arg(media_kind)::text
|
||||
AND media_id = sqlc.arg(media_id)::bigint
|
||||
AND ref_kind = sqlc.arg(ref_kind)::text
|
||||
AND ref_key = sqlc.arg(ref_key)::text;
|
||||
|
||||
-- name: OrphanDocumentIfUnreferenced :exec
|
||||
UPDATE documents SET orphaned_at = now()
|
||||
WHERE id = sqlc.arg(media_id)::bigint
|
||||
AND orphaned_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_references WHERE media_kind = 'document' AND media_id = sqlc.arg(media_id)::bigint
|
||||
);
|
||||
|
||||
-- name: OrphanPhotoIfUnreferenced :exec
|
||||
UPDATE photos SET orphaned_at = now()
|
||||
WHERE id = sqlc.arg(media_id)::bigint
|
||||
AND orphaned_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_references WHERE media_kind = 'photo' AND media_id = sqlc.arg(media_id)::bigint
|
||||
);
|
||||
|
||||
-- name: ListOrphanedDocumentIDsOlderThan :many
|
||||
SELECT id FROM documents
|
||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
|
||||
ORDER BY orphaned_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: ListOrphanedPhotoIDsOlderThan :many
|
||||
SELECT id FROM photos
|
||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
|
||||
ORDER BY orphaned_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: CountFileBlobRefs :one
|
||||
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;
|
||||
|
||||
-- name: DeleteFileBlobRow :exec
|
||||
DELETE FROM file_blobs WHERE location_key = sqlc.arg(location_key)::text;
|
||||
|
||||
-- name: ListFileBlobsByLocationPrefix :many
|
||||
-- Matches a media's main blob (exact_key, e.g. "doc:123") plus every
|
||||
-- variant keyed off it (prefix_pattern, e.g. "doc:123:%" for thumbnails /
|
||||
-- "photo:456:%" for each rendition size) -- a document/photo can own
|
||||
-- multiple file_blobs rows.
|
||||
SELECT location_key, backend, object_key, size
|
||||
FROM file_blobs
|
||||
WHERE location_key = sqlc.arg(exact_key)::text
|
||||
OR location_key LIKE sqlc.arg(prefix_pattern)::text;
|
||||
|
||||
-- name: SumFileBlobBytes :one
|
||||
-- Physical bytes actually held by the blob backend (dedup-aware: identical
|
||||
-- content uploaded by different users is one row here). Used by the
|
||||
-- low-space guard's cached usage gauge and the admin panel's "physical
|
||||
-- usage" stat.
|
||||
SELECT COALESCE(SUM(size), 0)::bigint FROM file_blobs;
|
||||
|
||||
-- sticker_sets ----------------------------------------------------------------
|
||||
|
||||
-- name: PutStickerSet :exec
|
||||
|
|
|
|||
|
|
@ -48,6 +48,26 @@ func (q *Queries) AddProfilePhoto(ctx context.Context, arg AddProfilePhotoParams
|
|||
return err
|
||||
}
|
||||
|
||||
const clearDocumentOrphan = `-- name: ClearDocumentOrphan :exec
|
||||
UPDATE documents SET orphaned_at = NULL
|
||||
WHERE id = $1::bigint AND orphaned_at IS NOT NULL
|
||||
`
|
||||
|
||||
func (q *Queries) ClearDocumentOrphan(ctx context.Context, mediaID int64) error {
|
||||
_, err := q.db.Exec(ctx, clearDocumentOrphan, mediaID)
|
||||
return err
|
||||
}
|
||||
|
||||
const clearPhotoOrphan = `-- name: ClearPhotoOrphan :exec
|
||||
UPDATE photos SET orphaned_at = NULL
|
||||
WHERE id = $1::bigint AND orphaned_at IS NOT NULL
|
||||
`
|
||||
|
||||
func (q *Queries) ClearPhotoOrphan(ctx context.Context, mediaID int64) error {
|
||||
_, err := q.db.Exec(ctx, clearPhotoOrphan, mediaID)
|
||||
return err
|
||||
}
|
||||
|
||||
const countAvailableReactions = `-- name: CountAvailableReactions :one
|
||||
SELECT count(*)::int AS total FROM available_reactions
|
||||
`
|
||||
|
|
@ -59,6 +79,22 @@ func (q *Queries) CountAvailableReactions(ctx context.Context) (int32, error) {
|
|||
return total, err
|
||||
}
|
||||
|
||||
const countFileBlobRefs = `-- name: CountFileBlobRefs :one
|
||||
SELECT COUNT(*)::int FROM file_blobs WHERE backend = $1::text AND object_key = $2::text
|
||||
`
|
||||
|
||||
type CountFileBlobRefsParams struct {
|
||||
Backend string
|
||||
ObjectKey string
|
||||
}
|
||||
|
||||
func (q *Queries) CountFileBlobRefs(ctx context.Context, arg CountFileBlobRefsParams) (int32, error) {
|
||||
row := q.db.QueryRow(ctx, countFileBlobRefs, arg.Backend, arg.ObjectKey)
|
||||
var column_1 int32
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const countProfilePhotos = `-- name: CountProfilePhotos :one
|
||||
SELECT count(*)::int AS total
|
||||
FROM profile_photos
|
||||
|
|
@ -199,6 +235,15 @@ func (q *Queries) DeactivateProfilePhotos(ctx context.Context, arg DeactivatePro
|
|||
return items, nil
|
||||
}
|
||||
|
||||
const deleteDocumentRow = `-- name: DeleteDocumentRow :exec
|
||||
DELETE FROM documents WHERE id = $1::bigint
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteDocumentRow(ctx context.Context, id int64) error {
|
||||
_, err := q.db.Exec(ctx, deleteDocumentRow, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteExpiredUploadParts = `-- name: DeleteExpiredUploadParts :many
|
||||
WITH doomed AS (
|
||||
SELECT owner_user_id, file_id, part
|
||||
|
|
@ -240,6 +285,24 @@ func (q *Queries) DeleteExpiredUploadParts(ctx context.Context, arg DeleteExpire
|
|||
return items, nil
|
||||
}
|
||||
|
||||
const deleteFileBlobRow = `-- name: DeleteFileBlobRow :exec
|
||||
DELETE FROM file_blobs WHERE location_key = $1::text
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteFileBlobRow(ctx context.Context, locationKey string) error {
|
||||
_, err := q.db.Exec(ctx, deleteFileBlobRow, locationKey)
|
||||
return err
|
||||
}
|
||||
|
||||
const deletePhotoRow = `-- name: DeletePhotoRow :exec
|
||||
DELETE FROM photos WHERE id = $1::bigint
|
||||
`
|
||||
|
||||
func (q *Queries) DeletePhotoRow(ctx context.Context, id int64) error {
|
||||
_, err := q.db.Exec(ctx, deletePhotoRow, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteUploadParts = `-- name: DeleteUploadParts :many
|
||||
DELETE FROM upload_parts
|
||||
WHERE owner_user_id = $1::bigint
|
||||
|
|
@ -275,7 +338,8 @@ func (q *Queries) DeleteUploadParts(ctx context.Context, arg DeleteUploadPartsPa
|
|||
const getDocument = `-- name: GetDocument :one
|
||||
SELECT id, access_hash, file_reference, date, mime_type, size, dc_id,
|
||||
attributes::text AS attributes_json,
|
||||
thumbs::text AS thumbs_json
|
||||
thumbs::text AS thumbs_json,
|
||||
owner_user_id
|
||||
FROM documents
|
||||
WHERE id = $1::bigint
|
||||
`
|
||||
|
|
@ -290,6 +354,7 @@ type GetDocumentRow struct {
|
|||
DcID int32
|
||||
AttributesJson string
|
||||
ThumbsJson string
|
||||
OwnerUserID int64
|
||||
}
|
||||
|
||||
func (q *Queries) GetDocument(ctx context.Context, id int64) (GetDocumentRow, error) {
|
||||
|
|
@ -305,6 +370,7 @@ func (q *Queries) GetDocument(ctx context.Context, id int64) (GetDocumentRow, er
|
|||
&i.DcID,
|
||||
&i.AttributesJson,
|
||||
&i.ThumbsJson,
|
||||
&i.OwnerUserID,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
|
@ -312,7 +378,8 @@ func (q *Queries) GetDocument(ctx context.Context, id int64) (GetDocumentRow, er
|
|||
const getDocuments = `-- name: GetDocuments :many
|
||||
SELECT id, access_hash, file_reference, date, mime_type, size, dc_id,
|
||||
attributes::text AS attributes_json,
|
||||
thumbs::text AS thumbs_json
|
||||
thumbs::text AS thumbs_json,
|
||||
owner_user_id
|
||||
FROM documents
|
||||
WHERE id = ANY($1::bigint[])
|
||||
`
|
||||
|
|
@ -327,6 +394,7 @@ type GetDocumentsRow struct {
|
|||
DcID int32
|
||||
AttributesJson string
|
||||
ThumbsJson string
|
||||
OwnerUserID int64
|
||||
}
|
||||
|
||||
func (q *Queries) GetDocuments(ctx context.Context, ids []int64) ([]GetDocumentsRow, error) {
|
||||
|
|
@ -348,6 +416,7 @@ func (q *Queries) GetDocuments(ctx context.Context, ids []int64) ([]GetDocuments
|
|||
&i.DcID,
|
||||
&i.AttributesJson,
|
||||
&i.ThumbsJson,
|
||||
&i.OwnerUserID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -390,7 +459,8 @@ func (q *Queries) GetFileBlob(ctx context.Context, locationKey string) (GetFileB
|
|||
|
||||
const getPhoto = `-- name: GetPhoto :one
|
||||
SELECT id, access_hash, file_reference, date, dc_id, has_stickers,
|
||||
sizes::text AS sizes_json
|
||||
sizes::text AS sizes_json,
|
||||
owner_user_id
|
||||
FROM photos
|
||||
WHERE id = $1::bigint
|
||||
`
|
||||
|
|
@ -403,6 +473,7 @@ type GetPhotoRow struct {
|
|||
DcID int32
|
||||
HasStickers bool
|
||||
SizesJson string
|
||||
OwnerUserID int64
|
||||
}
|
||||
|
||||
func (q *Queries) GetPhoto(ctx context.Context, id int64) (GetPhotoRow, error) {
|
||||
|
|
@ -416,6 +487,7 @@ func (q *Queries) GetPhoto(ctx context.Context, id int64) (GetPhotoRow, error) {
|
|||
&i.DcID,
|
||||
&i.HasStickers,
|
||||
&i.SizesJson,
|
||||
&i.OwnerUserID,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
|
@ -686,6 +758,31 @@ func (q *Queries) GetUploadPartUsage(ctx context.Context, ownerUserID int64) (Ge
|
|||
return i, err
|
||||
}
|
||||
|
||||
const insertMediaReference = `-- name: InsertMediaReference :exec
|
||||
|
||||
INSERT INTO media_references (media_kind, media_id, ref_kind, ref_key)
|
||||
VALUES ($1::text, $2::bigint, $3::text, $4::text)
|
||||
ON CONFLICT DO NOTHING
|
||||
`
|
||||
|
||||
type InsertMediaReferenceParams struct {
|
||||
MediaKind string
|
||||
MediaID int64
|
||||
RefKind string
|
||||
RefKey string
|
||||
}
|
||||
|
||||
// media_references / storage retention -----------------------------------------
|
||||
func (q *Queries) InsertMediaReference(ctx context.Context, arg InsertMediaReferenceParams) error {
|
||||
_, err := q.db.Exec(ctx, insertMediaReference,
|
||||
arg.MediaKind,
|
||||
arg.MediaID,
|
||||
arg.RefKind,
|
||||
arg.RefKey,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const listAvailableReactions = `-- name: ListAvailableReactions :many
|
||||
SELECT
|
||||
reaction, title, inactive, premium,
|
||||
|
|
@ -728,6 +825,118 @@ func (q *Queries) ListAvailableReactions(ctx context.Context) ([]AvailableReacti
|
|||
return items, nil
|
||||
}
|
||||
|
||||
const listFileBlobsByLocationPrefix = `-- name: ListFileBlobsByLocationPrefix :many
|
||||
SELECT location_key, backend, object_key, size
|
||||
FROM file_blobs
|
||||
WHERE location_key = $1::text
|
||||
OR location_key LIKE $2::text
|
||||
`
|
||||
|
||||
type ListFileBlobsByLocationPrefixParams struct {
|
||||
ExactKey string
|
||||
PrefixPattern string
|
||||
}
|
||||
|
||||
type ListFileBlobsByLocationPrefixRow struct {
|
||||
LocationKey string
|
||||
Backend string
|
||||
ObjectKey string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// Matches a media's main blob (exact_key, e.g. "doc:123") plus every
|
||||
// variant keyed off it (prefix_pattern, e.g. "doc:123:%" for thumbnails /
|
||||
// "photo:456:%" for each rendition size) -- a document/photo can own
|
||||
// multiple file_blobs rows.
|
||||
func (q *Queries) ListFileBlobsByLocationPrefix(ctx context.Context, arg ListFileBlobsByLocationPrefixParams) ([]ListFileBlobsByLocationPrefixRow, error) {
|
||||
rows, err := q.db.Query(ctx, listFileBlobsByLocationPrefix, arg.ExactKey, arg.PrefixPattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListFileBlobsByLocationPrefixRow
|
||||
for rows.Next() {
|
||||
var i ListFileBlobsByLocationPrefixRow
|
||||
if err := rows.Scan(
|
||||
&i.LocationKey,
|
||||
&i.Backend,
|
||||
&i.ObjectKey,
|
||||
&i.Size,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listOrphanedDocumentIDsOlderThan = `-- name: ListOrphanedDocumentIDsOlderThan :many
|
||||
SELECT id FROM documents
|
||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
|
||||
ORDER BY orphaned_at ASC
|
||||
LIMIT $2::int
|
||||
`
|
||||
|
||||
type ListOrphanedDocumentIDsOlderThanParams struct {
|
||||
Cutoff pgtype.Timestamptz
|
||||
BatchLimit int32
|
||||
}
|
||||
|
||||
func (q *Queries) ListOrphanedDocumentIDsOlderThan(ctx context.Context, arg ListOrphanedDocumentIDsOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listOrphanedDocumentIDsOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listOrphanedPhotoIDsOlderThan = `-- name: ListOrphanedPhotoIDsOlderThan :many
|
||||
SELECT id FROM photos
|
||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
|
||||
ORDER BY orphaned_at ASC
|
||||
LIMIT $2::int
|
||||
`
|
||||
|
||||
type ListOrphanedPhotoIDsOlderThanParams struct {
|
||||
Cutoff pgtype.Timestamptz
|
||||
BatchLimit int32
|
||||
}
|
||||
|
||||
func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrphanedPhotoIDsOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listOrphanedPhotoIDsOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listProfilePhotos = `-- name: ListProfilePhotos :many
|
||||
SELECT photo_id
|
||||
FROM profile_photos
|
||||
|
|
@ -925,6 +1134,34 @@ func (q *Queries) NextProfilePhotoOrder(ctx context.Context, arg NextProfilePhot
|
|||
return max_order, err
|
||||
}
|
||||
|
||||
const orphanDocumentIfUnreferenced = `-- name: OrphanDocumentIfUnreferenced :exec
|
||||
UPDATE documents SET orphaned_at = now()
|
||||
WHERE id = $1::bigint
|
||||
AND orphaned_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_references WHERE media_kind = 'document' AND media_id = $1::bigint
|
||||
)
|
||||
`
|
||||
|
||||
func (q *Queries) OrphanDocumentIfUnreferenced(ctx context.Context, mediaID int64) error {
|
||||
_, err := q.db.Exec(ctx, orphanDocumentIfUnreferenced, mediaID)
|
||||
return err
|
||||
}
|
||||
|
||||
const orphanPhotoIfUnreferenced = `-- name: OrphanPhotoIfUnreferenced :exec
|
||||
UPDATE photos SET orphaned_at = now()
|
||||
WHERE id = $1::bigint
|
||||
AND orphaned_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_references WHERE media_kind = 'photo' AND media_id = $1::bigint
|
||||
)
|
||||
`
|
||||
|
||||
func (q *Queries) OrphanPhotoIfUnreferenced(ctx context.Context, mediaID int64) error {
|
||||
_, err := q.db.Exec(ctx, orphanPhotoIfUnreferenced, mediaID)
|
||||
return err
|
||||
}
|
||||
|
||||
const putAvailableReaction = `-- name: PutAvailableReaction :exec
|
||||
|
||||
INSERT INTO available_reactions (
|
||||
|
|
@ -995,7 +1232,7 @@ func (q *Queries) PutAvailableReaction(ctx context.Context, arg PutAvailableReac
|
|||
|
||||
const putDocument = `-- name: PutDocument :exec
|
||||
|
||||
INSERT INTO documents (id, access_hash, file_reference, date, mime_type, size, dc_id, attributes, thumbs)
|
||||
INSERT INTO documents (id, access_hash, file_reference, date, mime_type, size, dc_id, attributes, thumbs, owner_user_id)
|
||||
VALUES (
|
||||
$1::bigint,
|
||||
$2::bigint,
|
||||
|
|
@ -1005,7 +1242,8 @@ VALUES (
|
|||
$6::bigint,
|
||||
$7::int,
|
||||
$8::jsonb,
|
||||
$9::jsonb
|
||||
$9::jsonb,
|
||||
$10::bigint
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
access_hash = EXCLUDED.access_hash,
|
||||
|
|
@ -1028,9 +1266,14 @@ type PutDocumentParams struct {
|
|||
DcID int32
|
||||
AttributesJson []byte
|
||||
ThumbsJson []byte
|
||||
OwnerUserID int64
|
||||
}
|
||||
|
||||
// documents -------------------------------------------------------------------
|
||||
// owner_user_id is intentionally NOT in the UPDATE SET list: a document id is
|
||||
// only ever (re-)upserted by its original uploader's own request replay, and
|
||||
// keeping the first-write owner sticky avoids any risk of a later call
|
||||
// (e.g. a forward re-touching the row) reassigning ownership.
|
||||
func (q *Queries) PutDocument(ctx context.Context, arg PutDocumentParams) error {
|
||||
_, err := q.db.Exec(ctx, putDocument,
|
||||
arg.ID,
|
||||
|
|
@ -1042,6 +1285,7 @@ func (q *Queries) PutDocument(ctx context.Context, arg PutDocumentParams) error
|
|||
arg.DcID,
|
||||
arg.AttributesJson,
|
||||
arg.ThumbsJson,
|
||||
arg.OwnerUserID,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
|
@ -1089,7 +1333,7 @@ func (q *Queries) PutFileBlob(ctx context.Context, arg PutFileBlobParams) error
|
|||
|
||||
const putPhoto = `-- name: PutPhoto :exec
|
||||
|
||||
INSERT INTO photos (id, access_hash, file_reference, date, dc_id, has_stickers, sizes)
|
||||
INSERT INTO photos (id, access_hash, file_reference, date, dc_id, has_stickers, sizes, owner_user_id)
|
||||
VALUES (
|
||||
$1::bigint,
|
||||
$2::bigint,
|
||||
|
|
@ -1097,7 +1341,8 @@ VALUES (
|
|||
$4::int,
|
||||
$5::int,
|
||||
$6::boolean,
|
||||
$7::jsonb
|
||||
$7::jsonb,
|
||||
$8::bigint
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
access_hash = EXCLUDED.access_hash,
|
||||
|
|
@ -1116,9 +1361,11 @@ type PutPhotoParams struct {
|
|||
DcID int32
|
||||
HasStickers bool
|
||||
SizesJson []byte
|
||||
OwnerUserID int64
|
||||
}
|
||||
|
||||
// photos ----------------------------------------------------------------------
|
||||
// owner_user_id intentionally not updated on conflict, see PutDocument.
|
||||
func (q *Queries) PutPhoto(ctx context.Context, arg PutPhotoParams) error {
|
||||
_, err := q.db.Exec(ctx, putPhoto,
|
||||
arg.ID,
|
||||
|
|
@ -1128,6 +1375,7 @@ func (q *Queries) PutPhoto(ctx context.Context, arg PutPhotoParams) error {
|
|||
arg.DcID,
|
||||
arg.HasStickers,
|
||||
arg.SizesJson,
|
||||
arg.OwnerUserID,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
|
@ -1244,6 +1492,31 @@ func (q *Queries) PutStickerSet(ctx context.Context, arg PutStickerSetParams) er
|
|||
return err
|
||||
}
|
||||
|
||||
const removeMediaReference = `-- name: RemoveMediaReference :exec
|
||||
DELETE FROM media_references
|
||||
WHERE media_kind = $1::text
|
||||
AND media_id = $2::bigint
|
||||
AND ref_kind = $3::text
|
||||
AND ref_key = $4::text
|
||||
`
|
||||
|
||||
type RemoveMediaReferenceParams struct {
|
||||
MediaKind string
|
||||
MediaID int64
|
||||
RefKind string
|
||||
RefKey string
|
||||
}
|
||||
|
||||
func (q *Queries) RemoveMediaReference(ctx context.Context, arg RemoveMediaReferenceParams) error {
|
||||
_, err := q.db.Exec(ctx, removeMediaReference,
|
||||
arg.MediaKind,
|
||||
arg.MediaID,
|
||||
arg.RefKind,
|
||||
arg.RefKey,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const saveUploadPart = `-- name: SaveUploadPart :exec
|
||||
|
||||
INSERT INTO upload_parts (owner_user_id, file_id, part, total_parts, is_big, backend, object_key, size, sha256)
|
||||
|
|
@ -1295,3 +1568,18 @@ func (q *Queries) SaveUploadPart(ctx context.Context, arg SaveUploadPartParams)
|
|||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const sumFileBlobBytes = `-- name: SumFileBlobBytes :one
|
||||
SELECT COALESCE(SUM(size), 0)::bigint FROM file_blobs
|
||||
`
|
||||
|
||||
// Physical bytes actually held by the blob backend (dedup-aware: identical
|
||||
// content uploaded by different users is one row here). Used by the
|
||||
// low-space guard's cached usage gauge and the admin panel's "physical
|
||||
// usage" stat.
|
||||
func (q *Queries) SumFileBlobBytes(ctx context.Context) (int64, error) {
|
||||
row := q.db.QueryRow(ctx, sumFileBlobBytes)
|
||||
var column_1 int64
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,6 +79,34 @@ type AccountPrivacyRule struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type AccountRating struct {
|
||||
UserID int64
|
||||
Level int32
|
||||
Stars int64
|
||||
CurrentLevelStars int64
|
||||
NextLevelStars *int64
|
||||
StarsComponent int64
|
||||
ActivityComponent int64
|
||||
PenaltyComponent int64
|
||||
ManualComponent int64
|
||||
PendingStars int64
|
||||
PendingDate pgtype.Timestamptz
|
||||
ComputedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
Version int64
|
||||
}
|
||||
|
||||
type AccountRatingEvent struct {
|
||||
ID int64
|
||||
UserID int64
|
||||
Kind string
|
||||
Amount int64
|
||||
Reason string
|
||||
Actor string
|
||||
CommandKey *string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type AccountReactionSetting struct {
|
||||
UserID int64
|
||||
MessagesNotifyFrom string
|
||||
|
|
@ -218,6 +246,21 @@ type AttachMenuUserState struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type AuthDeliveryReport struct {
|
||||
ID int64
|
||||
AuthKeyID []byte
|
||||
SessionID int64
|
||||
ClientType string
|
||||
PhoneHash []byte
|
||||
CodeHash []byte
|
||||
IssuedUserID int64
|
||||
DeliveryID string
|
||||
Channel string
|
||||
Mnc string
|
||||
Fingerprint []byte
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type AuthKey struct {
|
||||
AuthKeyID int64
|
||||
Body []byte
|
||||
|
|
@ -448,6 +491,20 @@ type BotUserPermission struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type BotVerifierSetting struct {
|
||||
BotID int64
|
||||
IconDocumentID int64
|
||||
CompanyName string
|
||||
DefaultDescription string
|
||||
CanModifyCustomDescription bool
|
||||
Enabled bool
|
||||
GrantedBy string
|
||||
GrantReason string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
Version int64
|
||||
}
|
||||
|
||||
type BusinessAutomationDelivery struct {
|
||||
OwnerUserID int64
|
||||
PeerUserID int64
|
||||
|
|
@ -577,6 +634,18 @@ type ChannelAdminLogEvent struct {
|
|||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ChannelAntispamDecision struct {
|
||||
ID int64
|
||||
ChannelID int64
|
||||
MessageID int32
|
||||
AuthorUserID int64
|
||||
EvidenceSchemaVersion int16
|
||||
Evidence []byte
|
||||
EvidenceHash []byte
|
||||
ReportID *int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ChannelBoostSlot struct {
|
||||
UserID int64
|
||||
Slot int32
|
||||
|
|
@ -708,6 +777,8 @@ type ChannelMember struct {
|
|||
SlowmodeLastSendDate int32
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
HistoryClearAnchorID int32
|
||||
HistoryClearAnchorDate int32
|
||||
}
|
||||
|
||||
type ChannelMessage struct {
|
||||
|
|
@ -910,6 +981,55 @@ type ChatlistMembership struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ClientTelemetryEvent struct {
|
||||
ID int64
|
||||
UserID int64
|
||||
Kind string
|
||||
PeerType string
|
||||
PeerID int64
|
||||
SubjectIds []int64
|
||||
Payload []byte
|
||||
Fingerprint []byte
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type CollectibleUsername struct {
|
||||
ID int64
|
||||
Username string
|
||||
UsernameLower string
|
||||
Status string
|
||||
OwnerPeerType string
|
||||
OwnerPeerID int64
|
||||
PurchaseDate pgtype.Timestamptz
|
||||
Currency string
|
||||
Amount int64
|
||||
CryptoCurrency string
|
||||
CryptoAmount int64
|
||||
Url string
|
||||
OriginalOwnerPeerType string
|
||||
OriginalOwnerPeerID int64
|
||||
TransferCount int32
|
||||
Version int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type CollectibleUsernameTransfer struct {
|
||||
ID int64
|
||||
CollectibleID int64
|
||||
Kind string
|
||||
FromPeerType string
|
||||
FromPeerID int64
|
||||
ToPeerType string
|
||||
ToPeerID int64
|
||||
Currency string
|
||||
Amount int64
|
||||
Actor string
|
||||
Reason string
|
||||
CommandKey *string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type Community struct {
|
||||
ID int64
|
||||
AccessHash int64
|
||||
|
|
@ -1008,6 +1128,41 @@ type CountryCode struct {
|
|||
OrderIndex int32
|
||||
}
|
||||
|
||||
type CustomVerification struct {
|
||||
ID int64
|
||||
VerifierBotID int64
|
||||
PeerType string
|
||||
PeerID int64
|
||||
IconDocumentID int64
|
||||
Description string
|
||||
GrantedByUserID int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
Version int64
|
||||
}
|
||||
|
||||
type CustomVerificationRequest struct {
|
||||
ID int64
|
||||
VerifierBotID int64
|
||||
ApplicantUserID int64
|
||||
PeerType string
|
||||
PeerID int64
|
||||
PeerTitle string
|
||||
PeerUsername string
|
||||
Reason string
|
||||
RequestedDescription string
|
||||
Status string
|
||||
DecidedBy string
|
||||
DecisionReason string
|
||||
InternalNote string
|
||||
CorrelationID string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
ApprovedAt pgtype.Timestamptz
|
||||
RejectedAt pgtype.Timestamptz
|
||||
Version int64
|
||||
}
|
||||
|
||||
type Dialog struct {
|
||||
UserID int64
|
||||
PeerType string
|
||||
|
|
@ -1095,6 +1250,8 @@ type Document struct {
|
|||
Attributes []byte
|
||||
Thumbs []byte
|
||||
CreatedAt pgtype.Timestamptz
|
||||
OwnerUserID int64
|
||||
OrphanedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type EncryptedFile struct {
|
||||
|
|
@ -1286,6 +1443,14 @@ type LoginCodeMessageDelivery struct {
|
|||
ExpiresAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type MediaReference struct {
|
||||
MediaKind string
|
||||
MediaID int64
|
||||
RefKind string
|
||||
RefKey string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type MessageBox struct {
|
||||
OwnerUserID int64
|
||||
BoxID int32
|
||||
|
|
@ -1343,6 +1508,126 @@ type MessageBoxMedium struct {
|
|||
MessageDate int32
|
||||
}
|
||||
|
||||
type ModerationAction struct {
|
||||
ID int64
|
||||
CaseID int64
|
||||
DecisionID int64
|
||||
Kind string
|
||||
Payload []byte
|
||||
Status string
|
||||
Attempts int32
|
||||
AvailableAt pgtype.Timestamptz
|
||||
LeaseUntil pgtype.Timestamptz
|
||||
LastError string
|
||||
CommandID string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationAppeal struct {
|
||||
ID int64
|
||||
CaseID int64
|
||||
AppellantUserID int64
|
||||
AppealText string
|
||||
TextHash []byte
|
||||
Fingerprint []byte
|
||||
Status string
|
||||
PreviousCaseStatus string
|
||||
Reviewer string
|
||||
ReviewReason string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
ReviewedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationAppealLink struct {
|
||||
ID int64
|
||||
CaseID int64
|
||||
AppellantUserID int64
|
||||
TokenHash []byte
|
||||
ExpiresAt pgtype.Timestamptz
|
||||
AppealID *int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
ConsumedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationCase struct {
|
||||
ID int64
|
||||
TargetPeerType string
|
||||
TargetPeerID int64
|
||||
Status string
|
||||
Severity int16
|
||||
AssignedTo string
|
||||
Version int64
|
||||
ReportCount int32
|
||||
DistinctReporterCount int32
|
||||
FirstReportAt pgtype.Timestamptz
|
||||
LastReportAt pgtype.Timestamptz
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationCaseReport struct {
|
||||
CaseID int64
|
||||
ReportID int64
|
||||
AttachedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationDecision struct {
|
||||
ID int64
|
||||
CaseID int64
|
||||
AppealID *int64
|
||||
Kind string
|
||||
Actor string
|
||||
Reason string
|
||||
CommandID string
|
||||
Fingerprint []byte
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationLegacyEphemeralMigration struct {
|
||||
LegacyReportID int64
|
||||
ModerationReportID int64
|
||||
MigratedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationMediaHold struct {
|
||||
ReportID int64
|
||||
ItemOrdinal int16
|
||||
MediaKind string
|
||||
StorageKey string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
ReleasedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationReport struct {
|
||||
ID int64
|
||||
ReporterUserID int64
|
||||
Source string
|
||||
TargetPeerType string
|
||||
TargetPeerID int64
|
||||
Reason string
|
||||
ReportOption string
|
||||
ReportComment string
|
||||
CommentHash []byte
|
||||
Fingerprint []byte
|
||||
TaxonomyVersion int16
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationReportItem struct {
|
||||
ReportID int64
|
||||
Ordinal int16
|
||||
ItemKind string
|
||||
PeerType string
|
||||
PeerID int64
|
||||
ItemID int64
|
||||
SecondaryID int64
|
||||
AuthorUserID int64
|
||||
EvidenceSchemaVersion int16
|
||||
Evidence []byte
|
||||
EvidenceHash []byte
|
||||
}
|
||||
|
||||
type NotifySetting struct {
|
||||
OwnerUserID int64
|
||||
ScopeKind string
|
||||
|
|
@ -1412,6 +1697,11 @@ type PeerUsername struct {
|
|||
PeerType string
|
||||
PeerID int64
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
Username string
|
||||
Active bool
|
||||
Editable bool
|
||||
SortOrder int32
|
||||
CollectibleID *int64
|
||||
}
|
||||
|
||||
type Photo struct {
|
||||
|
|
@ -1423,6 +1713,8 @@ type Photo struct {
|
|||
HasStickers bool
|
||||
Sizes []byte
|
||||
CreatedAt pgtype.Timestamptz
|
||||
OwnerUserID int64
|
||||
OrphanedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type Poll struct {
|
||||
|
|
@ -1518,6 +1810,24 @@ type PrivateMessageReaction struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type PrivateNoForwardsChat struct {
|
||||
UserLowID int64
|
||||
UserHighID int64
|
||||
EnabledByUserID *int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type PrivateNoForwardsRequest struct {
|
||||
PrivateMessageSenderUserID int64
|
||||
PrivateMessageID int64
|
||||
RequesterUserID int64
|
||||
ResponderUserID int64
|
||||
ExpiresAt int32
|
||||
HandledAt int32
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ProfilePhoto struct {
|
||||
OwnerPeerType string
|
||||
OwnerPeerID int64
|
||||
|
|
@ -1568,6 +1878,16 @@ type SavedDialogPin struct {
|
|||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type SavedMessageReactionTag struct {
|
||||
UserID int64
|
||||
MessageBoxID int32
|
||||
ReactionType string
|
||||
ReactionValue string
|
||||
ChosenOrder int32
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type SavedMusic struct {
|
||||
UserID int64
|
||||
DocumentID int64
|
||||
|
|
@ -1645,6 +1965,21 @@ type SeedState struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type SponsoredMessageImpression struct {
|
||||
ID int64
|
||||
UserID int64
|
||||
RandomIDHash []byte
|
||||
TargetPeerType string
|
||||
TargetPeerID int64
|
||||
AuthorUserID int64
|
||||
EvidenceSchemaVersion int16
|
||||
Evidence []byte
|
||||
EvidenceHash []byte
|
||||
ReportID *int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
ExpiresAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type StarGiftAdminGrantCommand struct {
|
||||
RecipientUserID int64
|
||||
CommandKey string
|
||||
|
|
@ -1781,6 +2116,22 @@ type StarGiftCatalogRevision struct {
|
|||
BackgroundTextColor *int32
|
||||
}
|
||||
|
||||
// Purchase-time snapshot of per-admin channel gift notification intents; delivery uses deterministic private-message replay.
|
||||
type StarGiftChannelNotificationJob struct {
|
||||
SavedGiftID int64
|
||||
TargetUserID int64
|
||||
GiftDate int32
|
||||
Action []byte
|
||||
Attempts int32
|
||||
NextAttemptAt int32
|
||||
LeaseUntil int32
|
||||
DeliveredAt int32
|
||||
MessageID int32
|
||||
LastError string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type StarGiftCollectibleBackdrop struct {
|
||||
ID int64
|
||||
CollectibleRevisionID int64
|
||||
|
|
@ -2072,7 +2423,7 @@ type StarGiftUpgradeCommand struct {
|
|||
SourceEditPts int32
|
||||
}
|
||||
|
||||
// Owner-local service-message aliases (unique outputs and separate prepaid-upgrade notifications) for one saved gift aggregate.
|
||||
// Viewer-local private service-message aliases to saved gift aggregates; the saved gift owner may be that user or an authorized channel.
|
||||
type StarGiftUserMessageRef struct {
|
||||
OwnerUserID int64
|
||||
MsgID int32
|
||||
|
|
@ -2115,6 +2466,55 @@ type StarsBalance struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type StarsGiveaway struct {
|
||||
ID int64
|
||||
BuyerUserID int64
|
||||
FormID int64
|
||||
ChannelID int64
|
||||
LaunchMessageID int32
|
||||
RandomID int64
|
||||
Stars int64
|
||||
Users int32
|
||||
PerUserStars int64
|
||||
YearlyBoosts int32
|
||||
UntilDate int32
|
||||
PurposeJson []byte
|
||||
State string
|
||||
CreatedAt int32
|
||||
}
|
||||
|
||||
type StarsPurchaseCommand struct {
|
||||
BuyerUserID int64
|
||||
FormID int64
|
||||
RequestFingerprint []byte
|
||||
RecipientUserID *int64
|
||||
Stars int64
|
||||
Currency string
|
||||
Amount int64
|
||||
BalanceAfter int64
|
||||
TransactionID string
|
||||
CreatedAt int32
|
||||
Kind string
|
||||
SpendPeerType *string
|
||||
SpendPeerID *int64
|
||||
PurposeJson []byte
|
||||
}
|
||||
|
||||
type StarsPurchaseForm struct {
|
||||
BuyerUserID int64
|
||||
FormID int64
|
||||
RecipientUserID *int64
|
||||
Stars int64
|
||||
Currency string
|
||||
Amount int64
|
||||
IssuedAt int32
|
||||
ExpiresAt int32
|
||||
Kind string
|
||||
SpendPeerType *string
|
||||
SpendPeerID *int64
|
||||
PurposeJson []byte
|
||||
}
|
||||
|
||||
type StarsTransaction struct {
|
||||
ID int64
|
||||
UserID int64
|
||||
|
|
@ -2514,6 +2914,9 @@ type UserChannelMemberIndex struct {
|
|||
Forum bool
|
||||
PublicUsername bool
|
||||
CanPinMessages bool
|
||||
AvailableMinID int32
|
||||
HistoryClearAnchorID int32
|
||||
HistoryClearUpdatedAt int32
|
||||
}
|
||||
|
||||
type UserRecentReaction struct {
|
||||
|
|
@ -2530,6 +2933,7 @@ type UserSavedReactionTag struct {
|
|||
ReactionType string
|
||||
ReactionValue string
|
||||
Title string
|
||||
// Legacy unused column; visible counts are aggregated from saved_message_reaction_tags.
|
||||
ReactionCount int32
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
|
|
@ -2607,6 +3011,67 @@ type UserUpdateWatermark struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type VerificationApplication struct {
|
||||
ID int64
|
||||
ApplicantUserID int64
|
||||
TargetType string
|
||||
TargetID int64
|
||||
TargetTitle string
|
||||
TargetUsername string
|
||||
TargetAccessHash int64
|
||||
Category string
|
||||
Description string
|
||||
OfficialWebsite string
|
||||
SocialLinks []string
|
||||
PressLinks []string
|
||||
AdditionalNote string
|
||||
Status string
|
||||
ReviewerAdminID string
|
||||
DecisionReason string
|
||||
InternalNote string
|
||||
CorrelationID string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
SubmittedAt pgtype.Timestamptz
|
||||
ReviewedAt pgtype.Timestamptz
|
||||
Version int64
|
||||
}
|
||||
|
||||
type VerificationApplicationEvent struct {
|
||||
ID int64
|
||||
ApplicationID int64
|
||||
Kind string
|
||||
FromStatus string
|
||||
ToStatus string
|
||||
Actor string
|
||||
Reason string
|
||||
Note string
|
||||
CorrelationID string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type VerificationIcon struct {
|
||||
ID int64
|
||||
DocumentID int64
|
||||
OwnerBotID int64
|
||||
Name string
|
||||
Active bool
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type VerificationNotificationOutbox struct {
|
||||
ID int64
|
||||
ApplicationID int64
|
||||
RecipientUserID int64
|
||||
Kind string
|
||||
Payload []byte
|
||||
Attempts int32
|
||||
DeliveredAt pgtype.Timestamptz
|
||||
LastError string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type WebAuthorization struct {
|
||||
Hash int64
|
||||
RequestID int64
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ func TestStarGiftLifecycleMigrationsApply(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("migrate star gift lifecycle schema: %v", err)
|
||||
}
|
||||
if status.Dirty || status.Empty || status.Version != 20260714003127 {
|
||||
t.Fatalf("migration status = %+v, want clean version 20260714003127", status)
|
||||
if status.Dirty || status.Empty || status.Version != 20260714003129 {
|
||||
t.Fatalf("migration status = %+v, want clean version 20260714003129", status)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue