s3 support
This commit is contained in:
parent
fa5cfaf14d
commit
03f10b66ee
53 changed files with 2796 additions and 102 deletions
|
|
@ -27,6 +27,12 @@ type BlobBackend interface {
|
|||
// GetRange 只读 [offset, offset+limit) 段并返回该段字节与文件总大小(limit<=0 读到末尾),
|
||||
// 避免大文件每个 chunk 都整文件读入内存(getFile 按 chunk 多次请求 ⇒ 否则 O(N²) 放大)。
|
||||
GetRange(ctx context.Context, objectKey string, offset, limit int64) (data []byte, total int64, err error)
|
||||
// Delete removes the object at objectKey. Callers (storage retention GC)
|
||||
// must first confirm no remaining file_blobs row on this backend still
|
||||
// references objectKey -- content-addressed storage means the same
|
||||
// object can be shared by multiple documents/photos. Deleting an
|
||||
// already-absent object is not an error.
|
||||
Delete(ctx context.Context, objectKey string) error
|
||||
}
|
||||
|
||||
// UploadPartBackend 保存 upload.saveFilePart/saveBigFilePart 的临时分片字节。
|
||||
|
|
@ -205,6 +211,16 @@ func (l *LocalFS) GetRange(_ context.Context, objectKey string, offset, limit in
|
|||
return buf[:read], total, nil
|
||||
}
|
||||
|
||||
// Delete removes the on-disk object for objectKey. Missing objects are not
|
||||
// an error (idempotent, safe to retry). Callers must have already confirmed
|
||||
// no other file_blobs row on this backend still references objectKey.
|
||||
func (l *LocalFS) Delete(_ context.Context, objectKey string) error {
|
||||
if err := os.Remove(l.pathFor(objectKey)); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("delete blob: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *LocalFS) openBlobFile(objectKey string) (*sharedBlobFile, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
|
|
|||
|
|
@ -44,6 +44,32 @@ func TestLocalFSPutGetRoundTrip(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestLocalFSDelete(t *testing.T) {
|
||||
fs, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("new local fs: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
key, err := fs.Put(ctx, []byte("delete me"))
|
||||
if err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
if _, err := fs.Get(ctx, key); err != nil {
|
||||
t.Fatalf("get before delete: %v", err)
|
||||
}
|
||||
if err := fs.Delete(ctx, key); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
if _, err := fs.Get(ctx, key); err == nil {
|
||||
t.Fatal("expected get after delete to fail")
|
||||
}
|
||||
// Deleting an already-absent object must be idempotent, not an error --
|
||||
// the retention sweep can legitimately retry after a partial failure.
|
||||
if err := fs.Delete(ctx, key); err != nil {
|
||||
t.Fatalf("delete already-missing object: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalFSPutReaderRoundTrip(t *testing.T) {
|
||||
fs, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
|
|
|
|||
179
internal/app/files/blobs3.go
Normal file
179
internal/app/files/blobs3.go
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// S3FS stores blob bytes in an S3-compatible object store (self-hosted MinIO
|
||||
// or AWS S3). It implements BlobBackend using the same content-addressed
|
||||
// sha256-hex object key scheme as LocalFS, so file_blobs.object_key stays
|
||||
// comparable regardless of which backend wrote a given row and a deployment
|
||||
// can run with rows split across both backends (see TELESRV_BLOB_BACKEND).
|
||||
//
|
||||
// UploadPartBackend is intentionally NOT implemented here: transient upload
|
||||
// parts stay on local disk (see cmd/telesrv/main.go) even when the
|
||||
// permanent blob backend is s3 -- one S3 round trip per ~512KB chunk isn't
|
||||
// worth it for scratch data that's deleted within minutes of assembly.
|
||||
type S3FS struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
// NewS3FS creates an S3-compatible blob backend. endpoint is host[:port]
|
||||
// without a scheme (e.g. "minio.internal:9000" or "s3.amazonaws.com");
|
||||
// useSSL selects http vs https. pathStyle forces path-style addressing
|
||||
// (bucket in the URL path rather than as a subdomain), which self-hosted
|
||||
// MinIO typically requires and AWS S3 does not.
|
||||
func NewS3FS(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucket, region string, useSSL, pathStyle bool) (*S3FS, error) {
|
||||
if endpoint == "" || bucket == "" {
|
||||
return nil, fmt.Errorf("s3 blob backend: endpoint and bucket are required")
|
||||
}
|
||||
lookup := minio.BucketLookupAuto
|
||||
if pathStyle {
|
||||
lookup = minio.BucketLookupPath
|
||||
}
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: useSSL,
|
||||
Region: region,
|
||||
BucketLookup: lookup,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create s3 client: %w", err)
|
||||
}
|
||||
exists, err := client.BucketExists(ctx, bucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("check s3 bucket %q: %w", bucket, err)
|
||||
}
|
||||
if !exists {
|
||||
if err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: region}); err != nil {
|
||||
return nil, fmt.Errorf("create s3 bucket %q: %w", bucket, err)
|
||||
}
|
||||
}
|
||||
return &S3FS{client: client, bucket: bucket}, nil
|
||||
}
|
||||
|
||||
// Name 返回后端标识,与 file_blobs.backend 一致。
|
||||
func (s *S3FS) Name() string { return "s3" }
|
||||
|
||||
func (s *S3FS) Put(ctx context.Context, data []byte) (string, error) {
|
||||
key, _, _, err := s.PutReader(ctx, bytes.NewReader(data))
|
||||
return key, err
|
||||
}
|
||||
|
||||
// PutReader hashes the stream to a local temp file first (so the sha256 key
|
||||
// and exact size are known before the S3 PUT, matching LocalFS's
|
||||
// content-addressed dedup semantics -- an unknown-length streaming PUT would
|
||||
// need a second copy operation to rename-by-hash after the fact, which S3
|
||||
// has no equivalent of), then uploads it and checks for an existing object
|
||||
// with that key first to skip a redundant PUT.
|
||||
func (s *S3FS) PutReader(ctx context.Context, r io.Reader) (string, int64, []byte, error) {
|
||||
tmp, err := os.CreateTemp("", "blob-s3-*.tmp")
|
||||
if err != nil {
|
||||
return "", 0, nil, fmt.Errorf("create s3 blob staging file: %w", err)
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
h := sha256.New()
|
||||
size, err := copyWithContext(ctx, io.MultiWriter(tmp, h), r)
|
||||
closeErr := tmp.Close()
|
||||
if err != nil {
|
||||
return "", 0, nil, fmt.Errorf("stage s3 blob: %w", err)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return "", 0, nil, fmt.Errorf("close s3 blob staging file: %w", closeErr)
|
||||
}
|
||||
sum := h.Sum(nil)
|
||||
key := hex.EncodeToString(sum)
|
||||
|
||||
if _, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}); err == nil {
|
||||
return key, size, append([]byte(nil), sum...), nil
|
||||
} else if !isS3NotFound(err) {
|
||||
return "", 0, nil, fmt.Errorf("stat s3 blob: %w", err)
|
||||
}
|
||||
|
||||
f, err := os.Open(tmpPath)
|
||||
if err != nil {
|
||||
return "", 0, nil, fmt.Errorf("reopen s3 blob staging file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := s.client.PutObject(ctx, s.bucket, key, f, size, minio.PutObjectOptions{}); err != nil {
|
||||
return "", 0, nil, fmt.Errorf("put s3 blob: %w", err)
|
||||
}
|
||||
return key, size, append([]byte(nil), sum...), nil
|
||||
}
|
||||
|
||||
func (s *S3FS) Get(ctx context.Context, objectKey string) ([]byte, error) {
|
||||
obj, err := s.client.GetObject(ctx, s.bucket, objectKey, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get s3 blob: %w", err)
|
||||
}
|
||||
defer obj.Close()
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read s3 blob: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// GetRange 语义与 LocalFS.GetRange 一致:只读 [offset, offset+limit) 段,limit<=0 读到末尾,
|
||||
// total 取自对象实际大小。
|
||||
func (s *S3FS) GetRange(ctx context.Context, objectKey string, offset, limit int64) ([]byte, int64, error) {
|
||||
info, err := s.client.StatObject(ctx, s.bucket, objectKey, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("stat s3 blob: %w", err)
|
||||
}
|
||||
total := info.Size
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
if offset >= total {
|
||||
return []byte{}, total, nil
|
||||
}
|
||||
opts := minio.GetObjectOptions{}
|
||||
end := total - 1
|
||||
if limit > 0 && offset+limit-1 < end {
|
||||
end = offset + limit - 1
|
||||
}
|
||||
if err := opts.SetRange(offset, end); err != nil {
|
||||
return nil, 0, fmt.Errorf("set s3 range: %w", err)
|
||||
}
|
||||
obj, err := s.client.GetObject(ctx, s.bucket, objectKey, opts)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("get s3 blob range: %w", err)
|
||||
}
|
||||
defer obj.Close()
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("read s3 blob range: %w", err)
|
||||
}
|
||||
return data, total, nil
|
||||
}
|
||||
|
||||
// Delete removes the object at objectKey. A missing object is not an error
|
||||
// (idempotent, safe to retry). Callers must have already confirmed no other
|
||||
// file_blobs row on this backend still references objectKey.
|
||||
func (s *S3FS) Delete(ctx context.Context, objectKey string) error {
|
||||
if err := s.client.RemoveObject(ctx, s.bucket, objectKey, minio.RemoveObjectOptions{}); err != nil {
|
||||
if isS3NotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("delete s3 blob: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isS3NotFound(err error) bool {
|
||||
resp := minio.ToErrorResponse(err)
|
||||
return resp.Code == "NoSuchKey" || resp.Code == "NotFound" || resp.StatusCode == 404
|
||||
}
|
||||
99
internal/app/files/diskspace.go
Normal file
99
internal/app/files/diskspace.go
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
package files
|
||||
|
||||
import "sync/atomic"
|
||||
|
||||
// SpaceGuard bounds how much more may be written to the permanent blob
|
||||
// backend. LocalDiskSpaceGuard checks real OS free disk bytes;
|
||||
// S3BudgetSpaceGuard compares a cached tracked-bytes total against a
|
||||
// configured budget (S3 has no OS-level "free space" concept). Both are
|
||||
// refreshed periodically by DiskUsageWorker rather than recomputed on every
|
||||
// upload chunk.
|
||||
type SpaceGuard interface {
|
||||
// Allow reports whether writing `additional` more bytes is currently
|
||||
// permitted. false should surface to the client as domain.ErrStorageFull.
|
||||
Allow(additional int64) (bool, error)
|
||||
// Usage returns the last-refreshed (used, total) byte snapshot for the
|
||||
// admin panel. ok is false if no successful refresh has happened yet.
|
||||
Usage() (used, total int64, ok bool)
|
||||
}
|
||||
|
||||
// NoopSpaceGuard always allows writes; the default when the low-space guard
|
||||
// is disabled by configuration.
|
||||
type NoopSpaceGuard struct{}
|
||||
|
||||
func (NoopSpaceGuard) Allow(int64) (bool, error) { return true, nil }
|
||||
func (NoopSpaceGuard) Usage() (int64, int64, bool) { return 0, 0, false }
|
||||
|
||||
// LocalDiskSpaceGuard rejects writes once cached free disk bytes fall below
|
||||
// minFreeBytes (<=0 disables the check). The free-bytes figure is
|
||||
// refreshed by DiskUsageWorker, not recomputed per call, to avoid a statfs
|
||||
// syscall on every upload chunk -- reads are lock-free.
|
||||
type LocalDiskSpaceGuard struct {
|
||||
minFreeBytes int64
|
||||
free atomic.Int64
|
||||
total atomic.Int64
|
||||
ready atomic.Bool
|
||||
}
|
||||
|
||||
func NewLocalDiskSpaceGuard(minFreeBytes int64) *LocalDiskSpaceGuard {
|
||||
return &LocalDiskSpaceGuard{minFreeBytes: minFreeBytes}
|
||||
}
|
||||
|
||||
func (g *LocalDiskSpaceGuard) Allow(additional int64) (bool, error) {
|
||||
// Before the first refresh completes, allow rather than reject: a
|
||||
// startup race shouldn't turn into spurious upload failures.
|
||||
if g.minFreeBytes <= 0 || !g.ready.Load() {
|
||||
return true, nil
|
||||
}
|
||||
return g.free.Load()-additional >= g.minFreeBytes, nil
|
||||
}
|
||||
|
||||
func (g *LocalDiskSpaceGuard) Usage() (used, total int64, ok bool) {
|
||||
if !g.ready.Load() {
|
||||
return 0, 0, false
|
||||
}
|
||||
total = g.total.Load()
|
||||
used = total - g.free.Load()
|
||||
if used < 0 {
|
||||
used = 0
|
||||
}
|
||||
return used, total, true
|
||||
}
|
||||
|
||||
func (g *LocalDiskSpaceGuard) setFree(free, total int64) {
|
||||
g.free.Store(free)
|
||||
g.total.Store(total)
|
||||
g.ready.Store(true)
|
||||
}
|
||||
|
||||
// S3BudgetSpaceGuard rejects writes once a cached tracked-bytes total
|
||||
// (refreshed periodically from file_blobs) would exceed maxTotalBytes
|
||||
// (<=0 disables the check).
|
||||
type S3BudgetSpaceGuard struct {
|
||||
maxTotalBytes int64
|
||||
used atomic.Int64
|
||||
ready atomic.Bool
|
||||
}
|
||||
|
||||
func NewS3BudgetSpaceGuard(maxTotalBytes int64) *S3BudgetSpaceGuard {
|
||||
return &S3BudgetSpaceGuard{maxTotalBytes: maxTotalBytes}
|
||||
}
|
||||
|
||||
func (g *S3BudgetSpaceGuard) Allow(additional int64) (bool, error) {
|
||||
if g.maxTotalBytes <= 0 || !g.ready.Load() {
|
||||
return true, nil
|
||||
}
|
||||
return g.used.Load()+additional <= g.maxTotalBytes, nil
|
||||
}
|
||||
|
||||
func (g *S3BudgetSpaceGuard) Usage() (used, total int64, ok bool) {
|
||||
if !g.ready.Load() {
|
||||
return 0, 0, false
|
||||
}
|
||||
return g.used.Load(), g.maxTotalBytes, true
|
||||
}
|
||||
|
||||
func (g *S3BudgetSpaceGuard) setUsed(used int64) {
|
||||
g.used.Store(used)
|
||||
g.ready.Store(true)
|
||||
}
|
||||
76
internal/app/files/diskspace_test.go
Normal file
76
internal/app/files/diskspace_test.go
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
package files
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNoopSpaceGuardAlwaysAllows(t *testing.T) {
|
||||
g := NoopSpaceGuard{}
|
||||
if allowed, err := g.Allow(1 << 40); err != nil || !allowed {
|
||||
t.Fatalf("expected noop guard to always allow, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
if _, _, ok := g.Usage(); ok {
|
||||
t.Fatal("expected noop guard to report no usage snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalDiskSpaceGuardBeforeFirstRefresh(t *testing.T) {
|
||||
g := NewLocalDiskSpaceGuard(1 << 30)
|
||||
// Before setFree has ever run, the guard must not reject -- a startup
|
||||
// race shouldn't turn into spurious upload failures.
|
||||
if allowed, err := g.Allow(1 << 40); err != nil || !allowed {
|
||||
t.Fatalf("expected pre-refresh guard to allow, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
if _, _, ok := g.Usage(); ok {
|
||||
t.Fatal("expected no usage snapshot before first refresh")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalDiskSpaceGuardThreshold(t *testing.T) {
|
||||
const minFree = int64(1000)
|
||||
g := NewLocalDiskSpaceGuard(minFree)
|
||||
g.setFree(1500, 10000)
|
||||
|
||||
if allowed, err := g.Allow(400); err != nil || !allowed {
|
||||
t.Fatalf("writing 400 bytes leaves 1100 free (>= 1000 min): want allow, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
if allowed, err := g.Allow(600); err != nil || allowed {
|
||||
t.Fatalf("writing 600 bytes leaves 900 free (< 1000 min): want reject, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
|
||||
used, total, ok := g.Usage()
|
||||
if !ok || total != 10000 || used != 8500 {
|
||||
t.Fatalf("usage snapshot = used=%d total=%d ok=%v, want used=8500 total=10000 ok=true", used, total, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalDiskSpaceGuardDisabled(t *testing.T) {
|
||||
g := NewLocalDiskSpaceGuard(0)
|
||||
g.setFree(10, 1000)
|
||||
if allowed, err := g.Allow(1 << 40); err != nil || !allowed {
|
||||
t.Fatalf("minFreeBytes<=0 must disable the check, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BudgetSpaceGuardThreshold(t *testing.T) {
|
||||
const maxTotal = int64(10000)
|
||||
g := NewS3BudgetSpaceGuard(maxTotal)
|
||||
g.setUsed(9000)
|
||||
|
||||
if allowed, err := g.Allow(1000); err != nil || !allowed {
|
||||
t.Fatalf("9000+1000 == budget: want allow, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
if allowed, err := g.Allow(1001); err != nil || allowed {
|
||||
t.Fatalf("9000+1001 exceeds budget: want reject, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
|
||||
used, total, ok := g.Usage()
|
||||
if !ok || total != maxTotal || used != 9000 {
|
||||
t.Fatalf("usage snapshot = used=%d total=%d ok=%v, want used=9000 total=%d ok=true", used, total, ok, maxTotal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3BudgetSpaceGuardBeforeFirstRefresh(t *testing.T) {
|
||||
g := NewS3BudgetSpaceGuard(100)
|
||||
if allowed, err := g.Allow(1 << 40); err != nil || !allowed {
|
||||
t.Fatalf("expected pre-refresh guard to allow, got allowed=%v err=%v", allowed, err)
|
||||
}
|
||||
}
|
||||
15
internal/app/files/diskspace_unix.go
Normal file
15
internal/app/files/diskspace_unix.go
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
//go:build !windows
|
||||
|
||||
package files
|
||||
|
||||
import "golang.org/x/sys/unix"
|
||||
|
||||
// localDiskFreeBytes returns free (available to an unprivileged writer, not
|
||||
// counting reserved blocks) and total bytes for the filesystem containing path.
|
||||
func localDiskFreeBytes(path string) (free, total int64, err error) {
|
||||
var st unix.Statfs_t
|
||||
if err := unix.Statfs(path, &st); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return int64(st.Bavail) * int64(st.Bsize), int64(st.Blocks) * int64(st.Bsize), nil
|
||||
}
|
||||
19
internal/app/files/diskspace_windows.go
Normal file
19
internal/app/files/diskspace_windows.go
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
//go:build windows
|
||||
|
||||
package files
|
||||
|
||||
import "golang.org/x/sys/windows"
|
||||
|
||||
// localDiskFreeBytes returns free (available to the calling user) and total
|
||||
// bytes for the volume containing path.
|
||||
func localDiskFreeBytes(path string) (free, total int64, err error) {
|
||||
ptr, err := windows.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
var freeAvail, totalBytes, totalFree uint64
|
||||
if err := windows.GetDiskFreeSpaceEx(ptr, &freeAvail, &totalBytes, &totalFree); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return int64(freeAvail), int64(totalBytes), nil
|
||||
}
|
||||
81
internal/app/files/diskusage_worker.go
Normal file
81
internal/app/files/diskusage_worker.go
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// SumFileBlobBytesStore is the minimal store dependency DiskUsageWorker
|
||||
// needs to refresh an S3BudgetSpaceGuard.
|
||||
type SumFileBlobBytesStore interface {
|
||||
SumFileBlobBytes(ctx context.Context) (int64, error)
|
||||
}
|
||||
|
||||
// DiskUsageWorker periodically refreshes one SpaceGuard's cached usage
|
||||
// snapshot, so the upload path never pays a statfs syscall / SUM(size)
|
||||
// query per chunk.
|
||||
type DiskUsageWorker struct {
|
||||
interval time.Duration
|
||||
log *zap.Logger
|
||||
refresh func(ctx context.Context) error
|
||||
}
|
||||
|
||||
// NewLocalDiskUsageWorker refreshes a LocalDiskSpaceGuard from real OS free
|
||||
// disk bytes under root (the blob backend's storage directory).
|
||||
func NewLocalDiskUsageWorker(guard *LocalDiskSpaceGuard, root string, interval time.Duration, log *zap.Logger) *DiskUsageWorker {
|
||||
return newDiskUsageWorker(interval, log, func(context.Context) error {
|
||||
free, total, err := localDiskFreeBytes(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
guard.setFree(free, total)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// NewS3DiskUsageWorker refreshes an S3BudgetSpaceGuard from the tracked
|
||||
// file_blobs byte total.
|
||||
func NewS3DiskUsageWorker(guard *S3BudgetSpaceGuard, media SumFileBlobBytesStore, interval time.Duration, log *zap.Logger) *DiskUsageWorker {
|
||||
return newDiskUsageWorker(interval, log, func(ctx context.Context) error {
|
||||
used, err := media.SumFileBlobBytes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
guard.setUsed(used)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func newDiskUsageWorker(interval time.Duration, log *zap.Logger, refresh func(context.Context) error) *DiskUsageWorker {
|
||||
if interval <= 0 {
|
||||
interval = time.Minute
|
||||
}
|
||||
if log == nil {
|
||||
log = zap.NewNop()
|
||||
}
|
||||
return &DiskUsageWorker{interval: interval, log: log, refresh: refresh}
|
||||
}
|
||||
|
||||
// Run refreshes once immediately (so the guard isn't stuck "not ready" for
|
||||
// a full interval after startup), then on every tick until ctx is done.
|
||||
func (w *DiskUsageWorker) Run(ctx context.Context) {
|
||||
w.refreshOnce(ctx)
|
||||
ticker := time.NewTicker(w.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
w.refreshOnce(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *DiskUsageWorker) refreshOnce(ctx context.Context) {
|
||||
if err := w.refresh(ctx); err != nil {
|
||||
w.log.Warn("refresh storage usage snapshot failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
|
@ -63,7 +63,7 @@ func (s *Service) CreatePhotoFromUpload(ctx context.Context, file domain.Uploade
|
|||
if len(data) == 0 {
|
||||
return domain.Photo{}, domain.ErrPhotoInvalid
|
||||
}
|
||||
photo, err := s.createPhoto(ctx, data, photoSizeSpecsForMessage(data))
|
||||
photo, err := s.createPhoto(ctx, data, photoSizeSpecsForMessage(data), file.OwnerUserID)
|
||||
if err != nil {
|
||||
return domain.Photo{}, err
|
||||
}
|
||||
|
|
@ -86,11 +86,13 @@ func (s *Service) CreatePhotoFromUpload(ctx context.Context, file domain.Uploade
|
|||
}
|
||||
|
||||
// CreatePhotoFromBytes stores already-fetched image bytes as a message Photo.
|
||||
// There is no uploader (e.g. a server-fetched webpage preview image), so the
|
||||
// photo is attributed to the system (owner_user_id 0) for storage accounting.
|
||||
func (s *Service) CreatePhotoFromBytes(ctx context.Context, data []byte) (domain.Photo, error) {
|
||||
if len(data) == 0 {
|
||||
return domain.Photo{}, domain.ErrPhotoInvalid
|
||||
}
|
||||
return s.createPhoto(ctx, data, photoSizeSpecsForMessage(data))
|
||||
return s.createPhoto(ctx, data, photoSizeSpecsForMessage(data), 0)
|
||||
}
|
||||
|
||||
// GetPhoto 按 id 返回已存储照片。
|
||||
|
|
@ -148,7 +150,7 @@ func (s *Service) CreateAvatarFromUpload(ctx context.Context, file domain.Upload
|
|||
if len(data) == 0 {
|
||||
return domain.Photo{}, domain.ErrPhotoInvalid
|
||||
}
|
||||
return s.createAvatarPhoto(ctx, data)
|
||||
return s.createAvatarPhoto(ctx, data, file.OwnerUserID)
|
||||
}
|
||||
|
||||
// CreateAvatarVideoFromUpload stores an animated profile video as photo.video_sizes.
|
||||
|
|
@ -206,6 +208,7 @@ func (s *Service) createAvatarVideoFromUpload(ctx context.Context, file domain.U
|
|||
Date: int(time.Now().Unix()),
|
||||
DCID: s.dc,
|
||||
Sizes: sizes,
|
||||
OwnerUserID: file.OwnerUserID,
|
||||
}
|
||||
if err := s.media.PutPhoto(ctx, photo); err != nil {
|
||||
return domain.Photo{}, err
|
||||
|
|
@ -306,6 +309,7 @@ func (s *Service) CreateDocumentFromUpload(ctx context.Context, file domain.Uplo
|
|||
Size: body.Size,
|
||||
DCID: s.dc,
|
||||
Attributes: spec.Attributes,
|
||||
OwnerUserID: file.OwnerUserID,
|
||||
}
|
||||
thumbMaterialized := false
|
||||
if spec.Thumb != nil {
|
||||
|
|
@ -620,7 +624,7 @@ func (s *Service) DeleteProfilePhotosKind(ctx context.Context, ownerType domain.
|
|||
}
|
||||
|
||||
// createPhoto 把字节落 blob(每个尺寸一个 location_key,指向同一内容)并写 photos 表。
|
||||
func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSizeSpec) (domain.Photo, error) {
|
||||
func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSizeSpec, ownerUserID int64) (domain.Photo, error) {
|
||||
photoID := randomID()
|
||||
sizes, err := s.putPhotoStaticSizes(ctx, photoID, data, specs)
|
||||
if err != nil {
|
||||
|
|
@ -633,6 +637,7 @@ func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSiz
|
|||
Date: int(time.Now().Unix()),
|
||||
DCID: s.dc,
|
||||
Sizes: sizes,
|
||||
OwnerUserID: ownerUserID,
|
||||
}
|
||||
if err := s.media.PutPhoto(ctx, photo); err != nil {
|
||||
return domain.Photo{}, err
|
||||
|
|
@ -640,7 +645,7 @@ func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSiz
|
|||
return photo, nil
|
||||
}
|
||||
|
||||
func (s *Service) createAvatarPhoto(ctx context.Context, data []byte) (domain.Photo, error) {
|
||||
func (s *Service) createAvatarPhoto(ctx context.Context, data []byte, ownerUserID int64) (domain.Photo, error) {
|
||||
photoID := randomID()
|
||||
sizes, err := s.putAvatarStaticSizes(ctx, photoID, data, photoSizeSpecsForAvatar(data))
|
||||
if err != nil {
|
||||
|
|
@ -653,6 +658,7 @@ func (s *Service) createAvatarPhoto(ctx context.Context, data []byte) (domain.Ph
|
|||
Date: int(time.Now().Unix()),
|
||||
DCID: s.dc,
|
||||
Sizes: sizes,
|
||||
OwnerUserID: ownerUserID,
|
||||
}
|
||||
if err := s.media.PutPhoto(ctx, photo); err != nil {
|
||||
return domain.Photo{}, err
|
||||
|
|
|
|||
94
internal/app/files/retention.go
Normal file
94
internal/app/files/retention.go
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// mediaRetentionStore is implemented by store.MediaStore backends that
|
||||
// support the storage retention sweep (currently only the Postgres store).
|
||||
// A type assertion, not a MediaStore interface method, keeps these
|
||||
// admin/maintenance-only queries out of the hot RPC-facing interface --
|
||||
// same convention as photoBatchStore above.
|
||||
type mediaRetentionStore interface {
|
||||
ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||
ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||
CountFileBlobRefs(ctx context.Context, backend, objectKey string) (int, error)
|
||||
DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||
DeletePhotoAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||
}
|
||||
|
||||
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
|
||||
// permanently deletes documents/photos that have had no live reference
|
||||
// (message/profile-photo/sticker-set, see media_references) since at least
|
||||
// cutoff, along with their blob(s) -- but only physically removes bytes
|
||||
// from the backend once confirming no other file_blobs row still needs the
|
||||
// object, since content-addressed storage means the same bytes can be
|
||||
// shared across documents/photos.
|
||||
func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
|
||||
store, ok := s.media.(mediaRetentionStore)
|
||||
if !ok || limit <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
deleted := 0
|
||||
docIDs, err := store.ListOrphanedDocumentIDsOlderThan(ctx, cutoff, limit)
|
||||
if err != nil {
|
||||
return deleted, fmt.Errorf("list orphaned documents: %w", err)
|
||||
}
|
||||
for _, id := range docIDs {
|
||||
blobs, err := store.DeleteDocumentAndBlobs(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("delete orphaned document failed", zap.Int64("document_id", id), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
deleted++
|
||||
}
|
||||
photoIDs, err := store.ListOrphanedPhotoIDsOlderThan(ctx, cutoff, limit)
|
||||
if err != nil {
|
||||
return deleted, fmt.Errorf("list orphaned photos: %w", err)
|
||||
}
|
||||
for _, id := range photoIDs {
|
||||
blobs, err := store.DeletePhotoAndBlobs(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("delete orphaned photo failed", zap.Int64("photo_id", id), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
deleted++
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// deleteOrphanedBlobs removes each blob from its backend once confirming
|
||||
// (via CountFileBlobRefs) no other file_blobs row still references
|
||||
// (backend, object_key). Only blobs on the currently active backend are
|
||||
// physically removed -- a blob left over from a previously active backend
|
||||
// (deployment switched TELESRV_BLOB_BACKEND at some point; switching back
|
||||
// isn't supported) is logged and skipped rather than silently dropped,
|
||||
// since there's no configured client to reach it right now anyway.
|
||||
func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionStore, blobs []domain.FileBlob) {
|
||||
for _, b := range blobs {
|
||||
refs, err := store.CountFileBlobRefs(ctx, string(b.Backend), b.ObjectKey)
|
||||
if err != nil {
|
||||
s.log.Warn("count file blob refs failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if refs > 0 {
|
||||
continue
|
||||
}
|
||||
if string(b.Backend) != s.blobs.Name() {
|
||||
s.log.Warn("orphaned blob is on an inactive backend, skipping physical delete",
|
||||
zap.String("backend", string(b.Backend)), zap.String("object_key", b.ObjectKey))
|
||||
continue
|
||||
}
|
||||
if err := s.blobs.Delete(ctx, b.ObjectKey); err != nil {
|
||||
s.log.Warn("delete orphaned blob failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -168,6 +168,16 @@ func (f *fakeMediaStore) GetFileBlobs(_ context.Context, keys []string) (map[str
|
|||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeMediaStore) SumFileBlobBytes(_ context.Context) (int64, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var total int64
|
||||
for _, b := range f.blobs {
|
||||
total += b.Size
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (f *fakeMediaStore) GetSeedState(_ context.Context, key string) (string, bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ type Service struct {
|
|||
stickerSetCache *stickerSetFullCache
|
||||
stickerSetNegCache *stickerSetNegativeCache
|
||||
uploadQuota domain.UploadPartQuota
|
||||
spaceGuard SpaceGuard
|
||||
mapTiles *mapTileProxy
|
||||
externalMedia *externalMediaFetcher
|
||||
webpage *webpageFetcher
|
||||
|
|
@ -116,6 +117,30 @@ func WithUploadPartQuota(quota domain.UploadPartQuota) Option {
|
|||
}
|
||||
}
|
||||
|
||||
// WithSpaceGuard installs the low-disk-space upload guard. Not calling this
|
||||
// (or passing nil) leaves the default NoopSpaceGuard, which never rejects.
|
||||
func WithSpaceGuard(guard SpaceGuard) Option {
|
||||
return func(s *Service) {
|
||||
if guard != nil {
|
||||
s.spaceGuard = guard
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithUploadPartBackend overrides where transient upload-part chunks are
|
||||
// staged before assembly, independent of the permanent blob backend passed
|
||||
// to NewService. Needed when the permanent backend is s3 (S3FS doesn't
|
||||
// implement UploadPartBackend -- chunk-per-request S3 round trips aren't
|
||||
// worth it for scratch data deleted within minutes): pass a LocalFS here so
|
||||
// uploads keep working, while permanent blobs still land in s3.
|
||||
func WithUploadPartBackend(backend UploadPartBackend) Option {
|
||||
return func(s *Service) {
|
||||
if backend != nil {
|
||||
s.uploadParts = backend
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NewService 创建 files 服务。dc 是本 server 的 DC id,写入新建 document/photo 的 dc_id。
|
||||
func NewService(media store.MediaStore, blobs BlobBackend, dc int, opts ...Option) *Service {
|
||||
s := &Service{
|
||||
|
|
@ -132,6 +157,7 @@ func NewService(media store.MediaStore, blobs BlobBackend, dc int, opts ...Optio
|
|||
MaxParts: DefaultUploadInFlightMaxParts,
|
||||
MaxFiles: DefaultUploadInFlightMaxFiles,
|
||||
},
|
||||
spaceGuard: NoopSpaceGuard{},
|
||||
}
|
||||
if partBackend, ok := blobs.(UploadPartBackend); ok {
|
||||
s.uploadParts = partBackend
|
||||
|
|
@ -205,6 +231,16 @@ func (s *Service) saveFilePart(ctx context.Context, part domain.UploadPart, byte
|
|||
if s.uploadParts == nil {
|
||||
return fmt.Errorf("upload part backend not configured")
|
||||
}
|
||||
// Cheapest possible rejection point: reject before any disk write once
|
||||
// the permanent blob backend is low on space. Upload parts themselves
|
||||
// always land on local scratch disk (see UploadPartBackend), but a
|
||||
// low-space condition on the permanent backend means assembly will
|
||||
// fail anyway, so there's no point accepting more chunks toward it.
|
||||
if allowed, err := s.spaceGuard.Allow(int64(len(bytes))); err != nil {
|
||||
return err
|
||||
} else if !allowed {
|
||||
return domain.ErrStorageFull
|
||||
}
|
||||
slot, err := s.checkUploadPartQuota(ctx, part)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -584,10 +620,19 @@ type assembledUploadBlob struct {
|
|||
// assembleUploadBlob 把上传分片流式写入正式 blob。调用方应在 durable media 元数据
|
||||
// 成功提交后调用 cleanupUploadParts,避免 metadata 写失败时丢失可重试的上传分片。
|
||||
func (s *Service) assembleUploadBlob(ctx context.Context, ownerUserID, fileID int64, expectedParts int) (assembledUploadBlob, error) {
|
||||
parts, _, err := s.loadAndValidateUploadParts(ctx, ownerUserID, fileID, expectedParts)
|
||||
parts, total, err := s.loadAndValidateUploadParts(ctx, ownerUserID, fileID, expectedParts)
|
||||
if err != nil {
|
||||
return assembledUploadBlob{}, err
|
||||
}
|
||||
// Re-check free space against the full assembled size right before
|
||||
// committing to the permanent blob backend: SaveFilePart already
|
||||
// checked each chunk, but free space may have dropped since then over
|
||||
// the lifetime of a large multi-part upload.
|
||||
if allowed, err := s.spaceGuard.Allow(total); err != nil {
|
||||
return assembledUploadBlob{}, err
|
||||
} else if !allowed {
|
||||
return assembledUploadBlob{}, domain.ErrStorageFull
|
||||
}
|
||||
if s.uploadParts == nil {
|
||||
return assembledUploadBlob{}, fmt.Errorf("upload part backend not configured")
|
||||
}
|
||||
|
|
|
|||
27
internal/app/maintenance/media_retention.go
Normal file
27
internal/app/maintenance/media_retention.go
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
package maintenance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// OrphanedMediaRetentionStore deletes documents/photos that have been
|
||||
// orphaned (no live message/profile-photo/sticker-set reference remains,
|
||||
// tracked via media_references + orphaned_at) for at least the configured
|
||||
// age, along with their underlying blob once no other file_blobs row on
|
||||
// its backend still needs it. Never touches media that still has a live
|
||||
// reference, regardless of age.
|
||||
type OrphanedMediaRetentionStore interface {
|
||||
DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error)
|
||||
}
|
||||
|
||||
// WithOrphanedMediaRetention enables the storage retention sweep. maxAge is
|
||||
// how long a document/photo must have been orphaned before it's actually
|
||||
// deleted (not how old the media itself is) -- <=0 leaves the sweep
|
||||
// disabled even if a store is provided, matching
|
||||
// TELESRV_STORAGE_RETENTION_ENABLE=false being the safe default.
|
||||
func (w *RetentionWorker) WithOrphanedMediaRetention(store OrphanedMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
|
||||
w.orphanedMedia = store
|
||||
w.orphanedMediaMaxAge = maxAge
|
||||
return w
|
||||
}
|
||||
|
|
@ -118,6 +118,7 @@ type RetentionWorker struct {
|
|||
orphanAuthKeys OrphanAuthKeyRetentionStore
|
||||
activeAuthKeys ActiveRawAuthKeyProvider
|
||||
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
|
||||
orphanedMedia OrphanedMediaRetentionStore
|
||||
logger *zap.Logger
|
||||
retention time.Duration
|
||||
botAPIRetention time.Duration
|
||||
|
|
@ -126,6 +127,7 @@ type RetentionWorker struct {
|
|||
authDeliveryReportRetention time.Duration
|
||||
outboxPoisonRetention time.Duration
|
||||
outboxPoisonInterval time.Duration
|
||||
orphanedMediaMaxAge time.Duration
|
||||
interval time.Duration
|
||||
batch int
|
||||
}
|
||||
|
|
@ -412,6 +414,17 @@ func (w *RetentionWorker) runRetentionOnce(ctx context.Context) {
|
|||
w.logger.Info("expired channel_update_events contiguous-prefix cleanup complete", zap.Int("deleted", channelDeleted))
|
||||
}
|
||||
}
|
||||
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
|
||||
// Only ever touches documents/photos already marked orphaned (no live
|
||||
// message/profile-photo/sticker-set reference remains) -- media still
|
||||
// visible in a conversation is never a candidate, regardless of age.
|
||||
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
|
||||
if err != nil {
|
||||
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
|
||||
} else if mediaDeleted > 0 {
|
||||
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) orphanHeartbeatInterval() time.Duration {
|
||||
|
|
|
|||
|
|
@ -253,6 +253,45 @@ type Config struct {
|
|||
StarGiftTONStartingGrant int64
|
||||
// BlobDir 是本地磁盘 blob backend 根目录(媒体文件字节内容)。
|
||||
BlobDir string
|
||||
// BlobBackendKind selects the blob storage backend: "localfs" (default)
|
||||
// or "s3". Transient upload parts always stay on local disk (BlobDir)
|
||||
// regardless of this setting -- only the permanent blob store moves.
|
||||
BlobBackendKind string
|
||||
// S3Endpoint/S3Region/S3Bucket/S3AccessKeyID/S3SecretAccessKey/S3UseSSL/
|
||||
// S3PathStyle configure the s3 blob backend; only used when
|
||||
// BlobBackendKind == "s3". Works against self-hosted MinIO or AWS S3.
|
||||
S3Endpoint string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3AccessKeyID string
|
||||
S3SecretAccessKey string
|
||||
S3UseSSL bool
|
||||
S3PathStyle bool
|
||||
// StorageLowSpaceGuardEnable turns on the pre-upload free-space check.
|
||||
StorageLowSpaceGuardEnable bool
|
||||
// StorageMinFreeBytes: for the localfs backend, reject new uploads once
|
||||
// real free disk bytes fall below this. <=0 disables this check.
|
||||
StorageMinFreeBytes int64
|
||||
// StorageMaxTotalBytes: reject new uploads once total tracked blob
|
||||
// bytes would exceed this -- the only meaningful "low space" signal for
|
||||
// the s3 backend (no OS-level free space concept), and usable as an
|
||||
// optional soft budget cap on localfs too. <=0 disables this check.
|
||||
StorageMaxTotalBytes int64
|
||||
// StorageUsageRefreshInterval is how often the cached free-space/budget
|
||||
// usage gauge refreshes; <=0 uses a 1 minute default.
|
||||
StorageUsageRefreshInterval time.Duration
|
||||
// StorageRetentionEnable turns on the orphaned-media age sweep. Off by
|
||||
// default: orphaned media (no live message/profile-photo/sticker-set
|
||||
// reference) is tracked and visible in the admin panel, but nothing is
|
||||
// auto-deleted until the operator explicitly opts in.
|
||||
StorageRetentionEnable bool
|
||||
// StorageRetentionMaxAge is how long a document/photo must have been
|
||||
// orphaned (not how old the media itself is) before the sweep deletes
|
||||
// it. Never deletes media that still has a live reference, regardless
|
||||
// of age. <=0 uses a 30 day default. The sweep itself runs on the
|
||||
// shared RetentionInterval/RetentionBatch cadence alongside every other
|
||||
// retention check (see maintenance.RetentionWorker).
|
||||
StorageRetentionMaxAge time.Duration
|
||||
// StickerSeedDir 是 reaction / sticker 资源种子目录(导入到 documents/sticker_sets + blob)。
|
||||
StickerSeedDir string
|
||||
// StickerSeedMaxSets 限制导入的常规贴纸集数量(避免启动时导入过多包),<=0 表示不限。
|
||||
|
|
@ -756,6 +795,20 @@ func Load() (Config, error) {
|
|||
OfficialGiftsDir: envOr("TELESRV_OFFICIAL_GIFTS_DIR", "data/official-gifts"),
|
||||
StarGiftTONStartingGrant: envInt64Or("TELESRV_STARGIFT_TON_STARTING_GRANT", 10_000_000_000),
|
||||
BlobDir: envOr("TELESRV_BLOB_DIR", "data/blobs"),
|
||||
BlobBackendKind: strings.ToLower(strings.TrimSpace(envOr("TELESRV_BLOB_BACKEND", "localfs"))),
|
||||
S3Endpoint: envOr("TELESRV_S3_ENDPOINT", ""),
|
||||
S3Region: envOr("TELESRV_S3_REGION", "us-east-1"),
|
||||
S3Bucket: envOr("TELESRV_S3_BUCKET", ""),
|
||||
S3AccessKeyID: envOr("TELESRV_S3_ACCESS_KEY_ID", ""),
|
||||
S3SecretAccessKey: envOr("TELESRV_S3_SECRET_ACCESS_KEY", ""),
|
||||
S3UseSSL: envBoolOr("TELESRV_S3_USE_SSL", true),
|
||||
S3PathStyle: envBoolOr("TELESRV_S3_PATH_STYLE", false),
|
||||
StorageLowSpaceGuardEnable: envBoolOr("TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE", true),
|
||||
StorageMinFreeBytes: envInt64Or("TELESRV_STORAGE_MIN_FREE_BYTES", 1<<30),
|
||||
StorageMaxTotalBytes: envInt64Or("TELESRV_STORAGE_MAX_TOTAL_BYTES", 0),
|
||||
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
|
||||
StorageRetentionEnable: envBoolOr("TELESRV_STORAGE_RETENTION_ENABLE", false),
|
||||
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
|
||||
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
|
||||
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
|
||||
PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"),
|
||||
|
|
@ -930,6 +983,9 @@ func Load() (Config, error) {
|
|||
if err := validateTelegramLoginConfig(cfg); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if err := validateStorageConfig(cfg); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
|
|
@ -1132,6 +1188,40 @@ func validateVerificationConfig(cfg Config) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// validateStorageConfig checks the blob backend selection and storage
|
||||
// management (low-space guard, retention sweep) settings.
|
||||
func validateStorageConfig(cfg Config) error {
|
||||
switch cfg.BlobBackendKind {
|
||||
case "localfs":
|
||||
// no extra requirements
|
||||
case "s3":
|
||||
if strings.TrimSpace(cfg.S3Endpoint) == "" {
|
||||
return fmt.Errorf("TELESRV_S3_ENDPOINT is required when TELESRV_BLOB_BACKEND=s3")
|
||||
}
|
||||
if strings.TrimSpace(cfg.S3Bucket) == "" {
|
||||
return fmt.Errorf("TELESRV_S3_BUCKET is required when TELESRV_BLOB_BACKEND=s3")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("TELESRV_BLOB_BACKEND must be \"localfs\" or \"s3\", got %q", cfg.BlobBackendKind)
|
||||
}
|
||||
if cfg.StorageMinFreeBytes < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_MIN_FREE_BYTES must be non-negative")
|
||||
}
|
||||
if cfg.StorageMaxTotalBytes < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
|
||||
}
|
||||
if cfg.StorageUsageRefreshInterval < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be non-negative")
|
||||
}
|
||||
if cfg.StorageRetentionMaxAge < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
|
||||
}
|
||||
if cfg.StorageRetentionEnable && cfg.StorageRetentionMaxAge <= 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_ENABLE is true")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateAdminRBACConfig checks the panel/adminapi permission configuration.
|
||||
// An unparsable permission name is refused rather than ignored: a silently
|
||||
// dropped permission is either a lockout or an unintended grant.
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ type MediaBackend string
|
|||
const (
|
||||
// MediaBackendLocalFS 表示 blob 字节存在本地磁盘(object_key 为相对路径)。
|
||||
MediaBackendLocalFS MediaBackend = "localfs"
|
||||
// MediaBackendS3 表示 blob 字节存在 S3 兼容对象存储(MinIO 或 AWS S3)。
|
||||
MediaBackendS3 MediaBackend = "s3"
|
||||
)
|
||||
|
||||
// FileBlob 是一个可下载的二进制对象的索引项:location_key → 后端/对象键/大小/mime。
|
||||
|
|
@ -233,6 +235,10 @@ type Document struct {
|
|||
DCID int `json:"dc_id,omitempty"`
|
||||
Attributes []DocumentAttribute `json:"attributes,omitempty"`
|
||||
Thumbs []PhotoSize `json:"thumbs,omitempty"`
|
||||
// OwnerUserID is who uploaded this document; 0 for rows predating storage
|
||||
// accounting or for system-originated documents. Used for per-account
|
||||
// storage usage reporting, not part of the tg wire protocol.
|
||||
OwnerUserID int64 `json:"owner_user_id,omitempty"`
|
||||
}
|
||||
|
||||
// StickerSetRef 返回该文档归属的贴纸集引用(若有 sticker/custom_emoji 属性)。
|
||||
|
|
@ -384,6 +390,9 @@ type Photo struct {
|
|||
DCID int `json:"dc_id,omitempty"`
|
||||
HasStickers bool `json:"has_stickers,omitempty"`
|
||||
Sizes []PhotoSize `json:"sizes,omitempty"`
|
||||
// OwnerUserID is who uploaded this photo; 0 for rows predating storage
|
||||
// accounting or for system-originated photos (e.g. webpage previews).
|
||||
OwnerUserID int64 `json:"owner_user_id,omitempty"`
|
||||
}
|
||||
|
||||
func ClonePhotoPtr(photo *Photo) *Photo {
|
||||
|
|
|
|||
|
|
@ -11,4 +11,8 @@ var (
|
|||
ErrUploadQuotaExceeded = errors.New("upload quota exceeded")
|
||||
ErrPhotoInvalid = errors.New("photo invalid")
|
||||
ErrDocumentInvalid = errors.New("document invalid")
|
||||
// ErrStorageFull is returned when the configured low-space guard rejects a
|
||||
// write: local disk free bytes (or, on the s3 backend, the configured
|
||||
// total-bytes budget) has fallen below the configured threshold.
|
||||
ErrStorageFull = errors.New("storage full")
|
||||
)
|
||||
|
|
|
|||
90
internal/domain/media_refs.go
Normal file
90
internal/domain/media_refs.go
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
package domain
|
||||
|
||||
// MediaKind identifies which table a media_references row points into.
|
||||
type MediaKind string
|
||||
|
||||
const (
|
||||
MediaKindDocument MediaKind = "document"
|
||||
MediaKindPhoto MediaKind = "photo"
|
||||
)
|
||||
|
||||
// MediaRefKind identifies why a document/photo is still considered "live" --
|
||||
// each distinct kind of place that can hold a durable pointer to media gets
|
||||
// its own ref_kind, so removing one kind of reference (e.g. a deleted
|
||||
// message) doesn't accidentally drop a still-live reference of another kind
|
||||
// (e.g. the same document also set as someone's profile photo).
|
||||
type MediaRefKind string
|
||||
|
||||
const (
|
||||
MediaRefKindMessageBox MediaRefKind = "message_box"
|
||||
MediaRefKindChannelMessage MediaRefKind = "channel_message"
|
||||
MediaRefKindProfilePhoto MediaRefKind = "profile_photo"
|
||||
MediaRefKindStickerSet MediaRefKind = "sticker_set"
|
||||
MediaRefKindGift MediaRefKind = "gift"
|
||||
)
|
||||
|
||||
// MediaReference is one live pointer to a document/photo. GC (the storage
|
||||
// retention sweep) only considers a document/photo eligible for deletion
|
||||
// once every reference to it has been removed.
|
||||
type MediaReference struct {
|
||||
Kind MediaKind
|
||||
MediaID int64
|
||||
RefKind MediaRefKind
|
||||
RefKey string
|
||||
}
|
||||
|
||||
// OrphanCandidate is a document/photo whose last reference has been removed
|
||||
// (orphaned_at is set) and is old enough to be considered for the storage
|
||||
// retention sweep.
|
||||
type OrphanCandidate struct {
|
||||
Kind MediaKind
|
||||
MediaID int64
|
||||
Backend MediaBackend
|
||||
ObjectKey string
|
||||
Size int64
|
||||
OrphanedAt int64 // unix seconds
|
||||
}
|
||||
|
||||
// MediaRefTarget identifies one document/photo embedded in a message's media
|
||||
// snapshot.
|
||||
type MediaRefTarget struct {
|
||||
Kind MediaKind
|
||||
ID int64
|
||||
}
|
||||
|
||||
// ExtractMediaRefTargets returns every document/photo id embedded in a
|
||||
// message's media snapshot (including nested ones -- a live photo's video
|
||||
// document, a webpage preview's photo), deduplicated. Used to register/drop
|
||||
// media_references rows when a message carrying this media is
|
||||
// created/edited/deleted.
|
||||
func ExtractMediaRefTargets(media *MessageMedia) []MediaRefTarget {
|
||||
if media == nil {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[MediaRefTarget]struct{}, 2)
|
||||
var out []MediaRefTarget
|
||||
add := func(kind MediaKind, id int64) {
|
||||
if id == 0 {
|
||||
return
|
||||
}
|
||||
t := MediaRefTarget{Kind: kind, ID: id}
|
||||
if _, ok := seen[t]; ok {
|
||||
return
|
||||
}
|
||||
seen[t] = struct{}{}
|
||||
out = append(out, t)
|
||||
}
|
||||
if media.Photo != nil {
|
||||
add(MediaKindPhoto, media.Photo.ID)
|
||||
}
|
||||
if media.Document != nil {
|
||||
add(MediaKindDocument, media.Document.ID)
|
||||
}
|
||||
if media.LivePhotoVideo != nil {
|
||||
add(MediaKindDocument, media.LivePhotoVideo.ID)
|
||||
}
|
||||
if media.WebPage != nil && media.WebPage.Photo != nil {
|
||||
add(MediaKindPhoto, media.WebPage.Photo.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
@ -107,6 +107,11 @@ func locationInvalidErr() error { return tgerr.New(400, "LOCATION_INVALID")
|
|||
func fileIDInvalidErr() error { return tgerr.New(400, "FILE_ID_INVALID") }
|
||||
func documentInvalidErr() error { return tgerr.New(400, "DOCUMENT_INVALID") }
|
||||
|
||||
// storageFullErr surfaces the low-disk-space upload guard. Deliberately not
|
||||
// a flood-wait: a full disk won't resolve itself in 60 seconds, and telling
|
||||
// the client to retry shortly would be misleading.
|
||||
func storageFullErr() error { return tgerr.New(400, "STORAGE_FULL") }
|
||||
|
||||
func mediaEmptyErr() error { return tgerr.New(400, "MEDIA_EMPTY") }
|
||||
|
||||
func frozenMethodInvalidErr() error { return tgerr.New(420, "FROZEN_METHOD_INVALID") }
|
||||
|
|
|
|||
|
|
@ -684,6 +684,8 @@ func photoUploadErr(err error) error {
|
|||
return filePartsInvalidErr()
|
||||
case errors.Is(err, domain.ErrPhotoInvalid):
|
||||
return photoInvalidErr()
|
||||
case errors.Is(err, domain.ErrStorageFull):
|
||||
return storageFullErr()
|
||||
default:
|
||||
return internalErr()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1144,6 +1144,8 @@ func mediaUploadErr(err error) error {
|
|||
return photoInvalidErr()
|
||||
case errors.Is(err, domain.ErrDocumentInvalid):
|
||||
return mediaInvalidErr()
|
||||
case errors.Is(err, domain.ErrStorageFull):
|
||||
return storageFullErr()
|
||||
default:
|
||||
return internalErr()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -327,6 +327,8 @@ func fileSaveErr(err error) error {
|
|||
return filePartTooBigErr()
|
||||
case errors.Is(err, domain.ErrUploadQuotaExceeded):
|
||||
return floodWaitErr(60)
|
||||
case errors.Is(err, domain.ErrStorageFull):
|
||||
return storageFullErr()
|
||||
default:
|
||||
return internalErr()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,10 @@ type MediaStore interface {
|
|||
// GetFileBlobs 批量按 location_key 取 FileBlob 元数据(缺失的 key 不出现在返回 map 中)。
|
||||
// 供启动预热等需一次性加载大量 blob 的路径,替代逐个 GetFileBlob 的 N+1 往返。
|
||||
GetFileBlobs(ctx context.Context, locationKeys []string) (map[string]domain.FileBlob, error)
|
||||
// SumFileBlobBytes 返回全部 blob 的物理字节总量(去重后,同内容只算一次)。
|
||||
// 供低磁盘空间守卫的周期性用量刷新使用(尤其是 s3 backend 的预算模式,没有
|
||||
// 操作系统级"剩余空间"概念,只能靠这个累计值和配置的预算比较)。
|
||||
SumFileBlobBytes(ctx context.Context) (int64, error)
|
||||
|
||||
// seed 状态。只记录静态资源 catalog 的内容 hash,用于启动时跳过未变化的重复导入;
|
||||
// 真实可服务性仍由 documents/file_blobs 校验保证,不能只相信这里的 hash。
|
||||
|
|
|
|||
|
|
@ -479,6 +479,11 @@ ORDER BY id`, channel.ID, id32)
|
|||
// 删除入口统一静默跳过(官方客户端对它禁用删除)。
|
||||
continue
|
||||
}
|
||||
// Drop this message's media_references (storage GC); orphans the
|
||||
// document/photo if this was its last live reference anywhere.
|
||||
if err := removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindChannelMessage, channelMessageRefKey(channel.ID, id)); err != nil {
|
||||
return nil, domain.ChannelUpdateEvent{}, channel, fmt.Errorf("remove deleted channel message media references: %w", err)
|
||||
}
|
||||
deleted = append(deleted, id)
|
||||
if err := s.insertChannelAdminLogTx(ctx, tx, domain.ChannelAdminLogEvent{
|
||||
ChannelID: channel.ID,
|
||||
|
|
|
|||
|
|
@ -272,6 +272,10 @@ WHERE location_key = ANY($1::text[])`, locationKeys)
|
|||
return out, nil
|
||||
}
|
||||
|
||||
func (s *MediaStore) SumFileBlobBytes(ctx context.Context) (int64, error) {
|
||||
return s.q.SumFileBlobBytes(ctx)
|
||||
}
|
||||
|
||||
func (s *MediaStore) GetSeedState(ctx context.Context, key string) (string, bool, error) {
|
||||
var hash string
|
||||
if err := s.db.QueryRow(ctx, `
|
||||
|
|
@ -332,6 +336,7 @@ func putDocumentParams(doc domain.Document) (sqlcgen.PutDocumentParams, error) {
|
|||
DcID: int32(doc.DCID),
|
||||
AttributesJson: attrs,
|
||||
ThumbsJson: thumbs,
|
||||
OwnerUserID: doc.OwnerUserID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -472,6 +477,20 @@ func (c *documentMetaCache) put(id int64, doc domain.Document) {
|
|||
}
|
||||
}
|
||||
|
||||
// remove evicts id, e.g. after the row is permanently deleted (storage
|
||||
// retention sweep) so a stale cache hit can't outlive the row.
|
||||
func (c *documentMetaCache) remove(id int64) {
|
||||
if c == nil || id == 0 {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if el, ok := c.m[id]; ok {
|
||||
c.ll.Remove(el)
|
||||
delete(c.m, id)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneDocument(doc domain.Document) domain.Document {
|
||||
doc.FileReference = append([]byte(nil), doc.FileReference...)
|
||||
if len(doc.Attributes) > 0 {
|
||||
|
|
@ -514,6 +533,7 @@ func documentFromRow(row sqlcgen.GetDocumentRow) (domain.Document, error) {
|
|||
DCID: int(row.DcID),
|
||||
Attributes: attrs,
|
||||
Thumbs: thumbs,
|
||||
OwnerUserID: row.OwnerUserID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -532,6 +552,7 @@ func (s *MediaStore) PutPhoto(ctx context.Context, photo domain.Photo) error {
|
|||
DcID: int32(photo.DCID),
|
||||
HasStickers: photo.HasStickers,
|
||||
SizesJson: sizes,
|
||||
OwnerUserID: photo.OwnerUserID,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -543,7 +564,7 @@ func (s *MediaStore) GetPhoto(ctx context.Context, id int64) (domain.Photo, bool
|
|||
}
|
||||
return domain.Photo{}, false, err
|
||||
}
|
||||
photo, err := photoFromFields(row.ID, row.AccessHash, row.FileReference, int(row.Date), int(row.DcID), row.HasStickers, row.SizesJson)
|
||||
photo, err := photoFromFields(row.ID, row.AccessHash, row.FileReference, int(row.Date), int(row.DcID), row.HasStickers, row.SizesJson, row.OwnerUserID)
|
||||
if err != nil {
|
||||
return domain.Photo{}, false, err
|
||||
}
|
||||
|
|
@ -572,7 +593,7 @@ func (s *MediaStore) GetPhotos(ctx context.Context, ids []int64) ([]domain.Photo
|
|||
return nil, nil
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, access_hash, file_reference, date, dc_id, has_stickers, sizes::text
|
||||
SELECT id, access_hash, file_reference, date, dc_id, has_stickers, sizes::text, owner_user_id
|
||||
FROM photos
|
||||
WHERE id = ANY($1::bigint[])
|
||||
`, unique)
|
||||
|
|
@ -613,14 +634,15 @@ func scanPhotoRow(row photoScanner) (domain.Photo, error) {
|
|||
dcID int32
|
||||
hasStickers bool
|
||||
sizesJSON string
|
||||
ownerUserID int64
|
||||
)
|
||||
if err := row.Scan(&id, &accessHash, &fileReference, &date, &dcID, &hasStickers, &sizesJSON); err != nil {
|
||||
if err := row.Scan(&id, &accessHash, &fileReference, &date, &dcID, &hasStickers, &sizesJSON, &ownerUserID); err != nil {
|
||||
return domain.Photo{}, err
|
||||
}
|
||||
return photoFromFields(id, accessHash, fileReference, int(date), int(dcID), hasStickers, sizesJSON)
|
||||
return photoFromFields(id, accessHash, fileReference, int(date), int(dcID), hasStickers, sizesJSON, ownerUserID)
|
||||
}
|
||||
|
||||
func photoFromFields(id, accessHash int64, fileReference []byte, date, dcID int, hasStickers bool, sizesJSON string) (domain.Photo, error) {
|
||||
func photoFromFields(id, accessHash int64, fileReference []byte, date, dcID int, hasStickers bool, sizesJSON string, ownerUserID int64) (domain.Photo, error) {
|
||||
sizes, err := decodePhotoSizes(sizesJSON)
|
||||
if err != nil {
|
||||
return domain.Photo{}, err
|
||||
|
|
@ -633,6 +655,7 @@ func photoFromFields(id, accessHash int64, fileReference []byte, date, dcID int,
|
|||
DCID: dcID,
|
||||
HasStickers: hasStickers,
|
||||
Sizes: sizes,
|
||||
OwnerUserID: ownerUserID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
|
@ -697,7 +720,7 @@ func (s *MediaStore) CreateStickerSet(ctx context.Context, set domain.StickerSet
|
|||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return addStickerSetMediaReferencesTx(ctx, qtx, set.ID, docs)
|
||||
})
|
||||
if err != nil {
|
||||
if stickerSetShortNameConflict(err) {
|
||||
|
|
@ -726,7 +749,13 @@ func (s *MediaStore) UpdateStickerSet(ctx context.Context, set domain.StickerSet
|
|||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
// Re-register from scratch: drops references for any document no
|
||||
// longer in the set (candidate for storage GC once orphaned long
|
||||
// enough) and refreshes the rest.
|
||||
if err := removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindStickerSet, stickerSetRefKey(set.ID)); err != nil {
|
||||
return err
|
||||
}
|
||||
return addStickerSetMediaReferencesTx(ctx, qtx, set.ID, docs)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -751,8 +780,10 @@ WHERE id = $1
|
|||
if tag.RowsAffected() == 0 {
|
||||
return domain.ErrStickerSetInvalid
|
||||
}
|
||||
_, err = tx.Exec(ctx, `DELETE FROM user_sticker_sets WHERE sticker_set_id = $1`, setID)
|
||||
return err
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM user_sticker_sets WHERE sticker_set_id = $1`, setID); err != nil {
|
||||
return err
|
||||
}
|
||||
return removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindStickerSet, stickerSetRefKey(setID))
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -769,11 +800,40 @@ WHERE id = $1
|
|||
if tag.RowsAffected() == 0 {
|
||||
return domain.ErrStickerSetInvalid
|
||||
}
|
||||
_, err = tx.Exec(ctx, `DELETE FROM user_sticker_sets WHERE sticker_set_id = $1`, setID)
|
||||
return err
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM user_sticker_sets WHERE sticker_set_id = $1`, setID); err != nil {
|
||||
return err
|
||||
}
|
||||
return removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindStickerSet, stickerSetRefKey(setID))
|
||||
})
|
||||
}
|
||||
|
||||
func stickerSetRefKey(setID int64) string {
|
||||
return fmt.Sprintf("stickerset:%d", setID)
|
||||
}
|
||||
|
||||
// addStickerSetMediaReferencesTx registers every document belonging to a
|
||||
// sticker set as referenced (storage GC), clearing orphaned_at on each.
|
||||
func addStickerSetMediaReferencesTx(ctx context.Context, qtx *sqlcgen.Queries, setID int64, docs []domain.Document) error {
|
||||
refKey := stickerSetRefKey(setID)
|
||||
for _, doc := range docs {
|
||||
if doc.ID == 0 {
|
||||
continue
|
||||
}
|
||||
if err := qtx.InsertMediaReference(ctx, sqlcgen.InsertMediaReferenceParams{
|
||||
MediaKind: string(domain.MediaKindDocument),
|
||||
MediaID: doc.ID,
|
||||
RefKind: string(domain.MediaRefKindStickerSet),
|
||||
RefKey: refKey,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("register sticker set media reference: %w", err)
|
||||
}
|
||||
if err := qtx.ClearDocumentOrphan(ctx, doc.ID); err != nil {
|
||||
return fmt.Errorf("clear sticker document orphan: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func insertStickerSet(ctx context.Context, db sqlcgen.DBTX, set domain.StickerSet) error {
|
||||
thumbs, err := jsonArrayOrEmpty(set.Thumbs)
|
||||
if err != nil {
|
||||
|
|
@ -1197,15 +1257,29 @@ func (s *MediaStore) AddProfilePhotoKind(ctx context.Context, ownerType domain.P
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.Exec(ctx, `
|
||||
if _, err := s.db.Exec(ctx, `
|
||||
INSERT INTO profile_photos (owner_peer_type, owner_peer_id, kind, photo_id, date, active, sort_order)
|
||||
VALUES ($1, $2, $3, $4, $5, true, $6)
|
||||
ON CONFLICT (owner_peer_type, owner_peer_id, kind, photo_id) DO UPDATE SET
|
||||
date = EXCLUDED.date,
|
||||
active = true,
|
||||
sort_order = EXCLUDED.sort_order
|
||||
`, string(ownerType), ownerID, string(kind), photoID, date, next+1)
|
||||
return err
|
||||
`, string(ownerType), ownerID, string(kind), photoID, date, next+1); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.q.InsertMediaReference(ctx, sqlcgen.InsertMediaReferenceParams{
|
||||
MediaKind: string(domain.MediaKindPhoto),
|
||||
MediaID: photoID,
|
||||
RefKind: string(domain.MediaRefKindProfilePhoto),
|
||||
RefKey: profilePhotoRefKey(ownerType, ownerID, kind, photoID),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("register profile photo reference: %w", err)
|
||||
}
|
||||
return s.q.ClearPhotoOrphan(ctx, photoID)
|
||||
}
|
||||
|
||||
func profilePhotoRefKey(ownerType domain.PeerType, ownerID int64, kind domain.ProfilePhotoKind, photoID int64) string {
|
||||
return fmt.Sprintf("%s:%d:kind:%s:photo:%d", ownerType, ownerID, kind, photoID)
|
||||
}
|
||||
|
||||
func (s *MediaStore) CurrentProfilePhotoKind(ctx context.Context, ownerType domain.PeerType, ownerID int64, kind domain.ProfilePhotoKind) (int64, bool, error) {
|
||||
|
|
@ -1336,7 +1410,7 @@ func (s *MediaStore) ListProfilePhotoDetailsKind(ctx context.Context, ownerType
|
|||
var err error
|
||||
if offset < 0 && maxID > 0 {
|
||||
rows, err = s.db.Query(ctx, `
|
||||
SELECT ph.id, ph.access_hash, ph.file_reference, ph.date, ph.dc_id, ph.has_stickers, ph.sizes::text AS sizes_json
|
||||
SELECT ph.id, ph.access_hash, ph.file_reference, ph.date, ph.dc_id, ph.has_stickers, ph.sizes::text AS sizes_json, ph.owner_user_id
|
||||
FROM profile_photos pp
|
||||
JOIN photos ph ON ph.id = pp.photo_id
|
||||
WHERE pp.owner_peer_type = $1
|
||||
|
|
@ -1352,7 +1426,7 @@ LIMIT $5
|
|||
offset = 0
|
||||
}
|
||||
rows, err = s.db.Query(ctx, `
|
||||
SELECT ph.id, ph.access_hash, ph.file_reference, ph.date, ph.dc_id, ph.has_stickers, ph.sizes::text AS sizes_json
|
||||
SELECT ph.id, ph.access_hash, ph.file_reference, ph.date, ph.dc_id, ph.has_stickers, ph.sizes::text AS sizes_json, ph.owner_user_id
|
||||
FROM profile_photos pp
|
||||
JOIN photos ph ON ph.id = pp.photo_id
|
||||
WHERE pp.owner_peer_type = $1
|
||||
|
|
@ -1429,6 +1503,19 @@ RETURNING photo_id
|
|||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, id := range deleted {
|
||||
if err := s.q.RemoveMediaReference(ctx, sqlcgen.RemoveMediaReferenceParams{
|
||||
MediaKind: string(domain.MediaKindPhoto),
|
||||
MediaID: id,
|
||||
RefKind: string(domain.MediaRefKindProfilePhoto),
|
||||
RefKey: profilePhotoRefKey(ownerType, ownerID, kind, id),
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("remove profile photo reference: %w", err)
|
||||
}
|
||||
if err := s.q.OrphanPhotoIfUnreferenced(ctx, id); err != nil {
|
||||
return nil, fmt.Errorf("orphan check profile photo: %w", err)
|
||||
}
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ import (
|
|||
// 写路径只在「创建」和「编辑改媒体」两处维护,删除靠读查询 JOIN 过滤 deleted、不在此维护。
|
||||
|
||||
// insertChannelMediaIndexTx 为一条频道消息按其媒体类别写索引行(无类别则 no-op)。
|
||||
// 同时登记该消息内嵌 document/photo 的 media_references(存储回收用),与分类
|
||||
// 索引共用同一事务。
|
||||
func insertChannelMediaIndexTx(ctx context.Context, tx pgx.Tx, channelID int64, id, date int, media *domain.MessageMedia, entities []domain.MessageEntity) error {
|
||||
for _, c := range domain.ClassifyMediaCategories(media, entities) {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
|
|
@ -22,15 +24,18 @@ ON CONFLICT (channel_id, id, category) DO NOTHING`, channelID, id, int16(c), dat
|
|||
return fmt.Errorf("insert channel media index: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindChannelMessage, channelMessageRefKey(channelID, id))
|
||||
}
|
||||
|
||||
// deleteChannelMediaIndexTx 清掉一条频道消息的全部索引行(编辑改媒体前先清后插)。
|
||||
// 只在 replaceChannelMediaIndexTx 内被调用;真正的消息删除不清 *_media 分类索引
|
||||
// (读时靠 JOIN deleted 过滤,见文件头注释),但仍需在此清掉 media_references,
|
||||
// 否则 replace 场景下旧媒体永远不会被判定为孤儿。
|
||||
func deleteChannelMediaIndexTx(ctx context.Context, tx pgx.Tx, channelID int64, id int) error {
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM channel_message_media WHERE channel_id = $1 AND id = $2`, channelID, id); err != nil {
|
||||
return fmt.Errorf("delete channel media index: %w", err)
|
||||
}
|
||||
return nil
|
||||
return removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindChannelMessage, channelMessageRefKey(channelID, id))
|
||||
}
|
||||
|
||||
// replaceChannelMediaIndexTx 在编辑替换媒体后重建索引行(类别可能变化)。
|
||||
|
|
@ -41,7 +46,8 @@ func replaceChannelMediaIndexTx(ctx context.Context, tx pgx.Tx, channelID int64,
|
|||
return insertChannelMediaIndexTx(ctx, tx, channelID, id, date, media, entities)
|
||||
}
|
||||
|
||||
// insertMessageBoxMediaIndexTx 为一条私聊 owner box 按其媒体类别写索引行。
|
||||
// insertMessageBoxMediaIndexTx 为一条私聊 owner box 按其媒体类别写索引行。同时登记
|
||||
// media_references(存储回收用)。
|
||||
func insertMessageBoxMediaIndexTx(ctx context.Context, tx pgx.Tx, ownerUserID, peerID int64, boxID, date int, media *domain.MessageMedia, entities []domain.MessageEntity) error {
|
||||
for _, c := range domain.ClassifyMediaCategories(media, entities) {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
|
|
@ -51,15 +57,18 @@ ON CONFLICT (owner_user_id, box_id, category) DO NOTHING`, ownerUserID, boxID, p
|
|||
return fmt.Errorf("insert message box media index: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindMessageBox, messageBoxRefKey(ownerUserID, boxID))
|
||||
}
|
||||
|
||||
// deleteMessageBoxMediaIndexTx 清掉一条私聊 owner box 的全部索引行。
|
||||
// deleteMessageBoxMediaIndexTx 清掉一条私聊 owner box 的全部索引行。只在
|
||||
// replaceMessageBoxMediaIndexTx 内被调用(编辑改媒体前先清后插);真正的消息
|
||||
// 删除不清 message_box_media(读时靠 JOIN deleted 过滤),但仍需在此清掉
|
||||
// media_references,否则 replace 场景下旧媒体永远不会被判定为孤儿。
|
||||
func deleteMessageBoxMediaIndexTx(ctx context.Context, tx pgx.Tx, ownerUserID int64, boxID int) error {
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM message_box_media WHERE owner_user_id = $1 AND box_id = $2`, ownerUserID, boxID); err != nil {
|
||||
return fmt.Errorf("delete message box media index: %w", err)
|
||||
}
|
||||
return nil
|
||||
return removeMediaReferencesByKeyTx(ctx, tx, domain.MediaRefKindMessageBox, messageBoxRefKey(ownerUserID, boxID))
|
||||
}
|
||||
|
||||
// replaceMessageBoxMediaIndexTx 在编辑替换媒体后重建索引行。
|
||||
|
|
|
|||
212
internal/store/postgres/media_refs.go
Normal file
212
internal/store/postgres/media_refs.go
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
"telesrv/internal/store/postgres/sqlcgen"
|
||||
)
|
||||
|
||||
// messageBoxRefKey/channelMessageRefKey are the ref_key encodings used by
|
||||
// media_references rows registered from the private-mailbox and channel
|
||||
// message write paths, respectively. Kept as named helpers so the add and
|
||||
// remove sides can never drift apart on format.
|
||||
func messageBoxRefKey(ownerUserID int64, boxID int) string {
|
||||
return fmt.Sprintf("user:%d:box:%d", ownerUserID, boxID)
|
||||
}
|
||||
|
||||
func channelMessageRefKey(channelID int64, messageID int) string {
|
||||
return fmt.Sprintf("channel:%d:msg:%d", channelID, messageID)
|
||||
}
|
||||
|
||||
// addMediaReferencesTx registers every document/photo embedded in media as
|
||||
// referenced by refKind/refKey, clearing orphaned_at on each if it had been
|
||||
// set by an earlier removal. Must run in the same transaction as the write
|
||||
// that creates the reference (message send/edit).
|
||||
func addMediaReferencesTx(ctx context.Context, tx sqlcgen.DBTX, media *domain.MessageMedia, refKind domain.MediaRefKind, refKey string) error {
|
||||
targets := domain.ExtractMediaRefTargets(media)
|
||||
if len(targets) == 0 {
|
||||
return nil
|
||||
}
|
||||
q := sqlcgen.New(tx)
|
||||
for _, t := range targets {
|
||||
if err := q.InsertMediaReference(ctx, sqlcgen.InsertMediaReferenceParams{
|
||||
MediaKind: string(t.Kind),
|
||||
MediaID: t.ID,
|
||||
RefKind: string(refKind),
|
||||
RefKey: refKey,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("insert media reference: %w", err)
|
||||
}
|
||||
var clearErr error
|
||||
switch t.Kind {
|
||||
case domain.MediaKindDocument:
|
||||
clearErr = q.ClearDocumentOrphan(ctx, t.ID)
|
||||
case domain.MediaKindPhoto:
|
||||
clearErr = q.ClearPhotoOrphan(ctx, t.ID)
|
||||
}
|
||||
if clearErr != nil {
|
||||
return fmt.Errorf("clear media orphan: %w", clearErr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeMediaReferencesByKeyTx drops every media_references row registered
|
||||
// under refKind/refKey (no need to know which document/photo ids those were
|
||||
// -- the delete finds them) and, for each one that becomes fully
|
||||
// unreferenced as a result, marks it orphaned so the storage retention
|
||||
// sweep can consider it once old enough. Must run in the same transaction
|
||||
// as the write that removes the reference (a message being soft-deleted).
|
||||
func removeMediaReferencesByKeyTx(ctx context.Context, tx sqlcgen.DBTX, refKind domain.MediaRefKind, refKey string) error {
|
||||
rows, err := tx.Query(ctx, `
|
||||
DELETE FROM media_references
|
||||
WHERE ref_kind = $1 AND ref_key = $2
|
||||
RETURNING media_kind, media_id`, string(refKind), refKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("remove media references: %w", err)
|
||||
}
|
||||
type removedRef struct {
|
||||
kind string
|
||||
id int64
|
||||
}
|
||||
var removed []removedRef
|
||||
for rows.Next() {
|
||||
var r removedRef
|
||||
if err := rows.Scan(&r.kind, &r.id); err != nil {
|
||||
rows.Close()
|
||||
return fmt.Errorf("scan removed media reference: %w", err)
|
||||
}
|
||||
removed = append(removed, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("remove media references: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
q := sqlcgen.New(tx)
|
||||
for _, r := range removed {
|
||||
var orphanErr error
|
||||
switch domain.MediaKind(r.kind) {
|
||||
case domain.MediaKindDocument:
|
||||
orphanErr = q.OrphanDocumentIfUnreferenced(ctx, r.id)
|
||||
case domain.MediaKindPhoto:
|
||||
orphanErr = q.OrphanPhotoIfUnreferenced(ctx, r.id)
|
||||
}
|
||||
if orphanErr != nil {
|
||||
return fmt.Errorf("orphan check media: %w", orphanErr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---- storage retention sweep ----
|
||||
|
||||
// ListOrphanedDocumentIDsOlderThan returns document ids whose orphaned_at is
|
||||
// set and older than cutoff, oldest first, up to limit.
|
||||
func (s *MediaStore) ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListOrphanedDocumentIDsOlderThan(ctx, sqlcgen.ListOrphanedDocumentIDsOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// ListOrphanedPhotoIDsOlderThan returns photo ids whose orphaned_at is set
|
||||
// and older than cutoff, oldest first, up to limit.
|
||||
func (s *MediaStore) ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListOrphanedPhotoIDsOlderThan(ctx, sqlcgen.ListOrphanedPhotoIDsOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// CountFileBlobRefs reports how many file_blobs rows still point at
|
||||
// (backend, objectKey) -- the caller must not physically delete the object
|
||||
// from that backend while this is > 0 (content-addressed storage: the same
|
||||
// object can be shared by multiple documents/photos).
|
||||
func (s *MediaStore) CountFileBlobRefs(ctx context.Context, backend, objectKey string) (int, error) {
|
||||
n, err := s.q.CountFileBlobRefs(ctx, sqlcgen.CountFileBlobRefsParams{Backend: backend, ObjectKey: objectKey})
|
||||
return int(n), err
|
||||
}
|
||||
|
||||
// DeleteDocumentAndBlobs deletes a document row and every file_blobs row it
|
||||
// owns (main body + thumbnail variants), returning what was deleted so the
|
||||
// caller can physically remove each object from its backend once confirming
|
||||
// (via CountFileBlobRefs, after this call) no other row still needs it.
|
||||
// Assumes the document is already orphaned -- does not check references.
|
||||
func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error) {
|
||||
var blobs []domain.FileBlob
|
||||
err := withTx(ctx, s.db, "delete document and blobs", func(tx pgx.Tx) error {
|
||||
qtx := s.q.WithTx(tx)
|
||||
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
|
||||
ExactKey: fmt.Sprintf("doc:%d", id),
|
||||
PrefixPattern: fmt.Sprintf("doc:%d:%%", id),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list document blobs: %w", err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
blobs = append(blobs, domain.FileBlob{
|
||||
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
|
||||
})
|
||||
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
|
||||
return fmt.Errorf("delete file blob row: %w", err)
|
||||
}
|
||||
}
|
||||
if err := qtx.DeleteDocumentRow(ctx, id); err != nil {
|
||||
return fmt.Errorf("delete document row: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.documents.remove(id)
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
|
||||
// (one per rendition size), returning what was deleted so the caller can
|
||||
// physically remove each object from its backend once confirming (via
|
||||
// CountFileBlobRefs, after this call) no other row still needs it. Assumes
|
||||
// the photo is already orphaned -- does not check references.
|
||||
func (s *MediaStore) DeletePhotoAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error) {
|
||||
var blobs []domain.FileBlob
|
||||
err := withTx(ctx, s.db, "delete photo and blobs", func(tx pgx.Tx) error {
|
||||
qtx := s.q.WithTx(tx)
|
||||
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
|
||||
ExactKey: fmt.Sprintf("photo:%d", id),
|
||||
PrefixPattern: fmt.Sprintf("photo:%d:%%", id),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list photo blobs: %w", err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
blobs = append(blobs, domain.FileBlob{
|
||||
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
|
||||
})
|
||||
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
|
||||
return fmt.Errorf("delete file blob row: %w", err)
|
||||
}
|
||||
}
|
||||
if err := qtx.DeletePhotoRow(ctx, id); err != nil {
|
||||
return fmt.Errorf("delete photo row: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return blobs, nil
|
||||
}
|
||||
109
internal/store/postgres/media_refs_integration_test.go
Normal file
109
internal/store/postgres/media_refs_integration_test.go
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// TestMediaReferenceOrphanTransitions proves the core storage-retention
|
||||
// safety invariant: a document's orphaned_at is set only once every
|
||||
// reference to it is gone, and cleared the instant a new one appears --
|
||||
// so the retention sweep never targets media still visible in a
|
||||
// conversation, regardless of how many places reference it.
|
||||
func TestMediaReferenceOrphanTransitions(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
s := NewMediaStore(pool)
|
||||
|
||||
const docID = int64(9100000000000000101)
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM media_references WHERE media_kind = 'document' AND media_id = $1`, docID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM documents WHERE id = $1`, docID)
|
||||
})
|
||||
|
||||
if err := s.PutDocument(ctx, domain.Document{ID: docID, MimeType: "text/plain", Size: 10}); err != nil {
|
||||
t.Fatalf("put document: %v", err)
|
||||
}
|
||||
|
||||
media := &domain.MessageMedia{Kind: domain.MessageMediaKindDocument, Document: &domain.Document{ID: docID}}
|
||||
|
||||
// A freshly created document has no orphaned_at yet either way -- it's
|
||||
// simply unreferenced until a message send registers the first
|
||||
// reference, at which point normal tracking takes over.
|
||||
orphaned, err := documentOrphanedAt(ctx, pool, docID)
|
||||
if err != nil {
|
||||
t.Fatalf("query orphaned_at: %v", err)
|
||||
}
|
||||
if orphaned {
|
||||
t.Fatal("expected a freshly inserted document to not be marked orphaned yet")
|
||||
}
|
||||
|
||||
// Adding a reference (as if a message carrying it was sent) clears it.
|
||||
mustAddRef(t, pool, media, domain.MediaRefKindMessageBox, "user:1:box:1")
|
||||
if orphaned, err := documentOrphanedAt(ctx, pool, docID); err != nil || orphaned {
|
||||
t.Fatalf("expected referenced document to not be orphaned, orphaned=%v err=%v", orphaned, err)
|
||||
}
|
||||
|
||||
// A second, independent reference (e.g. forwarded to another box).
|
||||
mustAddRef(t, pool, media, domain.MediaRefKindMessageBox, "user:2:box:5")
|
||||
|
||||
// Removing only one of the two references must NOT orphan the document.
|
||||
mustRemoveRefsByKey(t, pool, domain.MediaRefKindMessageBox, "user:1:box:1")
|
||||
if orphaned, err := documentOrphanedAt(ctx, pool, docID); err != nil || orphaned {
|
||||
t.Fatalf("expected document with a remaining reference to survive, orphaned=%v err=%v", orphaned, err)
|
||||
}
|
||||
|
||||
// Removing the last reference orphans it.
|
||||
mustRemoveRefsByKey(t, pool, domain.MediaRefKindMessageBox, "user:2:box:5")
|
||||
if orphaned, err := documentOrphanedAt(ctx, pool, docID); err != nil || !orphaned {
|
||||
t.Fatalf("expected document with no remaining reference to be orphaned, orphaned=%v err=%v", orphaned, err)
|
||||
}
|
||||
|
||||
// A reference reappearing after orphaning (e.g. re-sent) clears it again.
|
||||
mustAddRef(t, pool, media, domain.MediaRefKindMessageBox, "user:3:box:9")
|
||||
if orphaned, err := documentOrphanedAt(ctx, pool, docID); err != nil || orphaned {
|
||||
t.Fatalf("expected re-referenced document to no longer be orphaned, orphaned=%v err=%v", orphaned, err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustAddRef(t *testing.T, pool *pgxpool.Pool, media *domain.MessageMedia, refKind domain.MediaRefKind, refKey string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("begin tx: %v", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if err := addMediaReferencesTx(ctx, tx, media, refKind, refKey); err != nil {
|
||||
t.Fatalf("add media reference: %v", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
t.Fatalf("commit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRemoveRefsByKey(t *testing.T, pool *pgxpool.Pool, refKind domain.MediaRefKind, refKey string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("begin tx: %v", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if err := removeMediaReferencesByKeyTx(ctx, tx, refKind, refKey); err != nil {
|
||||
t.Fatalf("remove media references: %v", err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
t.Fatalf("commit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func documentOrphanedAt(ctx context.Context, pool *pgxpool.Pool, id int64) (bool, error) {
|
||||
var orphaned bool
|
||||
err := pool.QueryRow(ctx, `SELECT orphaned_at IS NOT NULL FROM documents WHERE id = $1`, id).Scan(&orphaned)
|
||||
return orphaned, err
|
||||
}
|
||||
|
|
@ -174,6 +174,11 @@ func (s *MessageStore) finishDeleteMessagesTx(ctx context.Context, db sqlcgen.DB
|
|||
if row.ownerUserID == 0 || row.boxID == 0 {
|
||||
continue
|
||||
}
|
||||
// Drop this box's media_references (storage GC); orphans the
|
||||
// document/photo if this was its last live reference anywhere.
|
||||
if err := removeMediaReferencesByKeyTx(ctx, db, domain.MediaRefKindMessageBox, messageBoxRefKey(row.ownerUserID, row.boxID)); err != nil {
|
||||
return res, fmt.Errorf("remove deleted message media references: %w", err)
|
||||
}
|
||||
idsByOwner[row.ownerUserID] = append(idsByOwner[row.ownerUserID], row.boxID)
|
||||
if row.peer.ID != 0 {
|
||||
if peersByOwner[row.ownerUserID] == nil {
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ WHERE location_key = sqlc.arg(location_key)::text;
|
|||
-- documents -------------------------------------------------------------------
|
||||
|
||||
-- name: PutDocument :exec
|
||||
INSERT INTO documents (id, access_hash, file_reference, date, mime_type, size, dc_id, attributes, thumbs)
|
||||
INSERT INTO documents (id, access_hash, file_reference, date, mime_type, size, dc_id, attributes, thumbs, owner_user_id)
|
||||
VALUES (
|
||||
sqlc.arg(id)::bigint,
|
||||
sqlc.arg(access_hash)::bigint,
|
||||
|
|
@ -118,8 +118,13 @@ VALUES (
|
|||
sqlc.arg(size)::bigint,
|
||||
sqlc.arg(dc_id)::int,
|
||||
sqlc.arg(attributes_json)::jsonb,
|
||||
sqlc.arg(thumbs_json)::jsonb
|
||||
sqlc.arg(thumbs_json)::jsonb,
|
||||
sqlc.arg(owner_user_id)::bigint
|
||||
)
|
||||
-- owner_user_id is intentionally NOT in the UPDATE SET list: a document id is
|
||||
-- only ever (re-)upserted by its original uploader's own request replay, and
|
||||
-- keeping the first-write owner sticky avoids any risk of a later call
|
||||
-- (e.g. a forward re-touching the row) reassigning ownership.
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
access_hash = EXCLUDED.access_hash,
|
||||
file_reference = EXCLUDED.file_reference,
|
||||
|
|
@ -133,21 +138,26 @@ ON CONFLICT (id) DO UPDATE SET
|
|||
-- name: GetDocument :one
|
||||
SELECT id, access_hash, file_reference, date, mime_type, size, dc_id,
|
||||
attributes::text AS attributes_json,
|
||||
thumbs::text AS thumbs_json
|
||||
thumbs::text AS thumbs_json,
|
||||
owner_user_id
|
||||
FROM documents
|
||||
WHERE id = sqlc.arg(id)::bigint;
|
||||
|
||||
-- name: GetDocuments :many
|
||||
SELECT id, access_hash, file_reference, date, mime_type, size, dc_id,
|
||||
attributes::text AS attributes_json,
|
||||
thumbs::text AS thumbs_json
|
||||
thumbs::text AS thumbs_json,
|
||||
owner_user_id
|
||||
FROM documents
|
||||
WHERE id = ANY(sqlc.arg(ids)::bigint[]);
|
||||
|
||||
-- name: DeleteDocumentRow :exec
|
||||
DELETE FROM documents WHERE id = sqlc.arg(id)::bigint;
|
||||
|
||||
-- photos ----------------------------------------------------------------------
|
||||
|
||||
-- name: PutPhoto :exec
|
||||
INSERT INTO photos (id, access_hash, file_reference, date, dc_id, has_stickers, sizes)
|
||||
INSERT INTO photos (id, access_hash, file_reference, date, dc_id, has_stickers, sizes, owner_user_id)
|
||||
VALUES (
|
||||
sqlc.arg(id)::bigint,
|
||||
sqlc.arg(access_hash)::bigint,
|
||||
|
|
@ -155,8 +165,10 @@ VALUES (
|
|||
sqlc.arg(date)::int,
|
||||
sqlc.arg(dc_id)::int,
|
||||
sqlc.arg(has_stickers)::boolean,
|
||||
sqlc.arg(sizes_json)::jsonb
|
||||
sqlc.arg(sizes_json)::jsonb,
|
||||
sqlc.arg(owner_user_id)::bigint
|
||||
)
|
||||
-- owner_user_id intentionally not updated on conflict, see PutDocument.
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
access_hash = EXCLUDED.access_hash,
|
||||
file_reference = EXCLUDED.file_reference,
|
||||
|
|
@ -167,10 +179,87 @@ ON CONFLICT (id) DO UPDATE SET
|
|||
|
||||
-- name: GetPhoto :one
|
||||
SELECT id, access_hash, file_reference, date, dc_id, has_stickers,
|
||||
sizes::text AS sizes_json
|
||||
sizes::text AS sizes_json,
|
||||
owner_user_id
|
||||
FROM photos
|
||||
WHERE id = sqlc.arg(id)::bigint;
|
||||
|
||||
-- name: DeletePhotoRow :exec
|
||||
DELETE FROM photos WHERE id = sqlc.arg(id)::bigint;
|
||||
|
||||
-- media_references / storage retention -----------------------------------------
|
||||
|
||||
-- name: InsertMediaReference :exec
|
||||
INSERT INTO media_references (media_kind, media_id, ref_kind, ref_key)
|
||||
VALUES (sqlc.arg(media_kind)::text, sqlc.arg(media_id)::bigint, sqlc.arg(ref_kind)::text, sqlc.arg(ref_key)::text)
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- name: ClearDocumentOrphan :exec
|
||||
UPDATE documents SET orphaned_at = NULL
|
||||
WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
|
||||
|
||||
-- name: ClearPhotoOrphan :exec
|
||||
UPDATE photos SET orphaned_at = NULL
|
||||
WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
|
||||
|
||||
-- name: RemoveMediaReference :exec
|
||||
DELETE FROM media_references
|
||||
WHERE media_kind = sqlc.arg(media_kind)::text
|
||||
AND media_id = sqlc.arg(media_id)::bigint
|
||||
AND ref_kind = sqlc.arg(ref_kind)::text
|
||||
AND ref_key = sqlc.arg(ref_key)::text;
|
||||
|
||||
-- name: OrphanDocumentIfUnreferenced :exec
|
||||
UPDATE documents SET orphaned_at = now()
|
||||
WHERE id = sqlc.arg(media_id)::bigint
|
||||
AND orphaned_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_references WHERE media_kind = 'document' AND media_id = sqlc.arg(media_id)::bigint
|
||||
);
|
||||
|
||||
-- name: OrphanPhotoIfUnreferenced :exec
|
||||
UPDATE photos SET orphaned_at = now()
|
||||
WHERE id = sqlc.arg(media_id)::bigint
|
||||
AND orphaned_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_references WHERE media_kind = 'photo' AND media_id = sqlc.arg(media_id)::bigint
|
||||
);
|
||||
|
||||
-- name: ListOrphanedDocumentIDsOlderThan :many
|
||||
SELECT id FROM documents
|
||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
|
||||
ORDER BY orphaned_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: ListOrphanedPhotoIDsOlderThan :many
|
||||
SELECT id FROM photos
|
||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
|
||||
ORDER BY orphaned_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: CountFileBlobRefs :one
|
||||
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;
|
||||
|
||||
-- name: DeleteFileBlobRow :exec
|
||||
DELETE FROM file_blobs WHERE location_key = sqlc.arg(location_key)::text;
|
||||
|
||||
-- name: ListFileBlobsByLocationPrefix :many
|
||||
-- Matches a media's main blob (exact_key, e.g. "doc:123") plus every
|
||||
-- variant keyed off it (prefix_pattern, e.g. "doc:123:%" for thumbnails /
|
||||
-- "photo:456:%" for each rendition size) -- a document/photo can own
|
||||
-- multiple file_blobs rows.
|
||||
SELECT location_key, backend, object_key, size
|
||||
FROM file_blobs
|
||||
WHERE location_key = sqlc.arg(exact_key)::text
|
||||
OR location_key LIKE sqlc.arg(prefix_pattern)::text;
|
||||
|
||||
-- name: SumFileBlobBytes :one
|
||||
-- Physical bytes actually held by the blob backend (dedup-aware: identical
|
||||
-- content uploaded by different users is one row here). Used by the
|
||||
-- low-space guard's cached usage gauge and the admin panel's "physical
|
||||
-- usage" stat.
|
||||
SELECT COALESCE(SUM(size), 0)::bigint FROM file_blobs;
|
||||
|
||||
-- sticker_sets ----------------------------------------------------------------
|
||||
|
||||
-- name: PutStickerSet :exec
|
||||
|
|
|
|||
|
|
@ -48,6 +48,26 @@ func (q *Queries) AddProfilePhoto(ctx context.Context, arg AddProfilePhotoParams
|
|||
return err
|
||||
}
|
||||
|
||||
const clearDocumentOrphan = `-- name: ClearDocumentOrphan :exec
|
||||
UPDATE documents SET orphaned_at = NULL
|
||||
WHERE id = $1::bigint AND orphaned_at IS NOT NULL
|
||||
`
|
||||
|
||||
func (q *Queries) ClearDocumentOrphan(ctx context.Context, mediaID int64) error {
|
||||
_, err := q.db.Exec(ctx, clearDocumentOrphan, mediaID)
|
||||
return err
|
||||
}
|
||||
|
||||
const clearPhotoOrphan = `-- name: ClearPhotoOrphan :exec
|
||||
UPDATE photos SET orphaned_at = NULL
|
||||
WHERE id = $1::bigint AND orphaned_at IS NOT NULL
|
||||
`
|
||||
|
||||
func (q *Queries) ClearPhotoOrphan(ctx context.Context, mediaID int64) error {
|
||||
_, err := q.db.Exec(ctx, clearPhotoOrphan, mediaID)
|
||||
return err
|
||||
}
|
||||
|
||||
const countAvailableReactions = `-- name: CountAvailableReactions :one
|
||||
SELECT count(*)::int AS total FROM available_reactions
|
||||
`
|
||||
|
|
@ -59,6 +79,22 @@ func (q *Queries) CountAvailableReactions(ctx context.Context) (int32, error) {
|
|||
return total, err
|
||||
}
|
||||
|
||||
const countFileBlobRefs = `-- name: CountFileBlobRefs :one
|
||||
SELECT COUNT(*)::int FROM file_blobs WHERE backend = $1::text AND object_key = $2::text
|
||||
`
|
||||
|
||||
type CountFileBlobRefsParams struct {
|
||||
Backend string
|
||||
ObjectKey string
|
||||
}
|
||||
|
||||
func (q *Queries) CountFileBlobRefs(ctx context.Context, arg CountFileBlobRefsParams) (int32, error) {
|
||||
row := q.db.QueryRow(ctx, countFileBlobRefs, arg.Backend, arg.ObjectKey)
|
||||
var column_1 int32
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const countProfilePhotos = `-- name: CountProfilePhotos :one
|
||||
SELECT count(*)::int AS total
|
||||
FROM profile_photos
|
||||
|
|
@ -199,6 +235,15 @@ func (q *Queries) DeactivateProfilePhotos(ctx context.Context, arg DeactivatePro
|
|||
return items, nil
|
||||
}
|
||||
|
||||
const deleteDocumentRow = `-- name: DeleteDocumentRow :exec
|
||||
DELETE FROM documents WHERE id = $1::bigint
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteDocumentRow(ctx context.Context, id int64) error {
|
||||
_, err := q.db.Exec(ctx, deleteDocumentRow, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteExpiredUploadParts = `-- name: DeleteExpiredUploadParts :many
|
||||
WITH doomed AS (
|
||||
SELECT owner_user_id, file_id, part
|
||||
|
|
@ -240,6 +285,24 @@ func (q *Queries) DeleteExpiredUploadParts(ctx context.Context, arg DeleteExpire
|
|||
return items, nil
|
||||
}
|
||||
|
||||
const deleteFileBlobRow = `-- name: DeleteFileBlobRow :exec
|
||||
DELETE FROM file_blobs WHERE location_key = $1::text
|
||||
`
|
||||
|
||||
func (q *Queries) DeleteFileBlobRow(ctx context.Context, locationKey string) error {
|
||||
_, err := q.db.Exec(ctx, deleteFileBlobRow, locationKey)
|
||||
return err
|
||||
}
|
||||
|
||||
const deletePhotoRow = `-- name: DeletePhotoRow :exec
|
||||
DELETE FROM photos WHERE id = $1::bigint
|
||||
`
|
||||
|
||||
func (q *Queries) DeletePhotoRow(ctx context.Context, id int64) error {
|
||||
_, err := q.db.Exec(ctx, deletePhotoRow, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const deleteUploadParts = `-- name: DeleteUploadParts :many
|
||||
DELETE FROM upload_parts
|
||||
WHERE owner_user_id = $1::bigint
|
||||
|
|
@ -275,7 +338,8 @@ func (q *Queries) DeleteUploadParts(ctx context.Context, arg DeleteUploadPartsPa
|
|||
const getDocument = `-- name: GetDocument :one
|
||||
SELECT id, access_hash, file_reference, date, mime_type, size, dc_id,
|
||||
attributes::text AS attributes_json,
|
||||
thumbs::text AS thumbs_json
|
||||
thumbs::text AS thumbs_json,
|
||||
owner_user_id
|
||||
FROM documents
|
||||
WHERE id = $1::bigint
|
||||
`
|
||||
|
|
@ -290,6 +354,7 @@ type GetDocumentRow struct {
|
|||
DcID int32
|
||||
AttributesJson string
|
||||
ThumbsJson string
|
||||
OwnerUserID int64
|
||||
}
|
||||
|
||||
func (q *Queries) GetDocument(ctx context.Context, id int64) (GetDocumentRow, error) {
|
||||
|
|
@ -305,6 +370,7 @@ func (q *Queries) GetDocument(ctx context.Context, id int64) (GetDocumentRow, er
|
|||
&i.DcID,
|
||||
&i.AttributesJson,
|
||||
&i.ThumbsJson,
|
||||
&i.OwnerUserID,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
|
@ -312,7 +378,8 @@ func (q *Queries) GetDocument(ctx context.Context, id int64) (GetDocumentRow, er
|
|||
const getDocuments = `-- name: GetDocuments :many
|
||||
SELECT id, access_hash, file_reference, date, mime_type, size, dc_id,
|
||||
attributes::text AS attributes_json,
|
||||
thumbs::text AS thumbs_json
|
||||
thumbs::text AS thumbs_json,
|
||||
owner_user_id
|
||||
FROM documents
|
||||
WHERE id = ANY($1::bigint[])
|
||||
`
|
||||
|
|
@ -327,6 +394,7 @@ type GetDocumentsRow struct {
|
|||
DcID int32
|
||||
AttributesJson string
|
||||
ThumbsJson string
|
||||
OwnerUserID int64
|
||||
}
|
||||
|
||||
func (q *Queries) GetDocuments(ctx context.Context, ids []int64) ([]GetDocumentsRow, error) {
|
||||
|
|
@ -348,6 +416,7 @@ func (q *Queries) GetDocuments(ctx context.Context, ids []int64) ([]GetDocuments
|
|||
&i.DcID,
|
||||
&i.AttributesJson,
|
||||
&i.ThumbsJson,
|
||||
&i.OwnerUserID,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -390,7 +459,8 @@ func (q *Queries) GetFileBlob(ctx context.Context, locationKey string) (GetFileB
|
|||
|
||||
const getPhoto = `-- name: GetPhoto :one
|
||||
SELECT id, access_hash, file_reference, date, dc_id, has_stickers,
|
||||
sizes::text AS sizes_json
|
||||
sizes::text AS sizes_json,
|
||||
owner_user_id
|
||||
FROM photos
|
||||
WHERE id = $1::bigint
|
||||
`
|
||||
|
|
@ -403,6 +473,7 @@ type GetPhotoRow struct {
|
|||
DcID int32
|
||||
HasStickers bool
|
||||
SizesJson string
|
||||
OwnerUserID int64
|
||||
}
|
||||
|
||||
func (q *Queries) GetPhoto(ctx context.Context, id int64) (GetPhotoRow, error) {
|
||||
|
|
@ -416,6 +487,7 @@ func (q *Queries) GetPhoto(ctx context.Context, id int64) (GetPhotoRow, error) {
|
|||
&i.DcID,
|
||||
&i.HasStickers,
|
||||
&i.SizesJson,
|
||||
&i.OwnerUserID,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
|
@ -686,6 +758,31 @@ func (q *Queries) GetUploadPartUsage(ctx context.Context, ownerUserID int64) (Ge
|
|||
return i, err
|
||||
}
|
||||
|
||||
const insertMediaReference = `-- name: InsertMediaReference :exec
|
||||
|
||||
INSERT INTO media_references (media_kind, media_id, ref_kind, ref_key)
|
||||
VALUES ($1::text, $2::bigint, $3::text, $4::text)
|
||||
ON CONFLICT DO NOTHING
|
||||
`
|
||||
|
||||
type InsertMediaReferenceParams struct {
|
||||
MediaKind string
|
||||
MediaID int64
|
||||
RefKind string
|
||||
RefKey string
|
||||
}
|
||||
|
||||
// media_references / storage retention -----------------------------------------
|
||||
func (q *Queries) InsertMediaReference(ctx context.Context, arg InsertMediaReferenceParams) error {
|
||||
_, err := q.db.Exec(ctx, insertMediaReference,
|
||||
arg.MediaKind,
|
||||
arg.MediaID,
|
||||
arg.RefKind,
|
||||
arg.RefKey,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const listAvailableReactions = `-- name: ListAvailableReactions :many
|
||||
SELECT
|
||||
reaction, title, inactive, premium,
|
||||
|
|
@ -728,6 +825,118 @@ func (q *Queries) ListAvailableReactions(ctx context.Context) ([]AvailableReacti
|
|||
return items, nil
|
||||
}
|
||||
|
||||
const listFileBlobsByLocationPrefix = `-- name: ListFileBlobsByLocationPrefix :many
|
||||
SELECT location_key, backend, object_key, size
|
||||
FROM file_blobs
|
||||
WHERE location_key = $1::text
|
||||
OR location_key LIKE $2::text
|
||||
`
|
||||
|
||||
type ListFileBlobsByLocationPrefixParams struct {
|
||||
ExactKey string
|
||||
PrefixPattern string
|
||||
}
|
||||
|
||||
type ListFileBlobsByLocationPrefixRow struct {
|
||||
LocationKey string
|
||||
Backend string
|
||||
ObjectKey string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// Matches a media's main blob (exact_key, e.g. "doc:123") plus every
|
||||
// variant keyed off it (prefix_pattern, e.g. "doc:123:%" for thumbnails /
|
||||
// "photo:456:%" for each rendition size) -- a document/photo can own
|
||||
// multiple file_blobs rows.
|
||||
func (q *Queries) ListFileBlobsByLocationPrefix(ctx context.Context, arg ListFileBlobsByLocationPrefixParams) ([]ListFileBlobsByLocationPrefixRow, error) {
|
||||
rows, err := q.db.Query(ctx, listFileBlobsByLocationPrefix, arg.ExactKey, arg.PrefixPattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []ListFileBlobsByLocationPrefixRow
|
||||
for rows.Next() {
|
||||
var i ListFileBlobsByLocationPrefixRow
|
||||
if err := rows.Scan(
|
||||
&i.LocationKey,
|
||||
&i.Backend,
|
||||
&i.ObjectKey,
|
||||
&i.Size,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listOrphanedDocumentIDsOlderThan = `-- name: ListOrphanedDocumentIDsOlderThan :many
|
||||
SELECT id FROM documents
|
||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
|
||||
ORDER BY orphaned_at ASC
|
||||
LIMIT $2::int
|
||||
`
|
||||
|
||||
type ListOrphanedDocumentIDsOlderThanParams struct {
|
||||
Cutoff pgtype.Timestamptz
|
||||
BatchLimit int32
|
||||
}
|
||||
|
||||
func (q *Queries) ListOrphanedDocumentIDsOlderThan(ctx context.Context, arg ListOrphanedDocumentIDsOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listOrphanedDocumentIDsOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listOrphanedPhotoIDsOlderThan = `-- name: ListOrphanedPhotoIDsOlderThan :many
|
||||
SELECT id FROM photos
|
||||
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
|
||||
ORDER BY orphaned_at ASC
|
||||
LIMIT $2::int
|
||||
`
|
||||
|
||||
type ListOrphanedPhotoIDsOlderThanParams struct {
|
||||
Cutoff pgtype.Timestamptz
|
||||
BatchLimit int32
|
||||
}
|
||||
|
||||
func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrphanedPhotoIDsOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listOrphanedPhotoIDsOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listProfilePhotos = `-- name: ListProfilePhotos :many
|
||||
SELECT photo_id
|
||||
FROM profile_photos
|
||||
|
|
@ -925,6 +1134,34 @@ func (q *Queries) NextProfilePhotoOrder(ctx context.Context, arg NextProfilePhot
|
|||
return max_order, err
|
||||
}
|
||||
|
||||
const orphanDocumentIfUnreferenced = `-- name: OrphanDocumentIfUnreferenced :exec
|
||||
UPDATE documents SET orphaned_at = now()
|
||||
WHERE id = $1::bigint
|
||||
AND orphaned_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_references WHERE media_kind = 'document' AND media_id = $1::bigint
|
||||
)
|
||||
`
|
||||
|
||||
func (q *Queries) OrphanDocumentIfUnreferenced(ctx context.Context, mediaID int64) error {
|
||||
_, err := q.db.Exec(ctx, orphanDocumentIfUnreferenced, mediaID)
|
||||
return err
|
||||
}
|
||||
|
||||
const orphanPhotoIfUnreferenced = `-- name: OrphanPhotoIfUnreferenced :exec
|
||||
UPDATE photos SET orphaned_at = now()
|
||||
WHERE id = $1::bigint
|
||||
AND orphaned_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM media_references WHERE media_kind = 'photo' AND media_id = $1::bigint
|
||||
)
|
||||
`
|
||||
|
||||
func (q *Queries) OrphanPhotoIfUnreferenced(ctx context.Context, mediaID int64) error {
|
||||
_, err := q.db.Exec(ctx, orphanPhotoIfUnreferenced, mediaID)
|
||||
return err
|
||||
}
|
||||
|
||||
const putAvailableReaction = `-- name: PutAvailableReaction :exec
|
||||
|
||||
INSERT INTO available_reactions (
|
||||
|
|
@ -995,7 +1232,7 @@ func (q *Queries) PutAvailableReaction(ctx context.Context, arg PutAvailableReac
|
|||
|
||||
const putDocument = `-- name: PutDocument :exec
|
||||
|
||||
INSERT INTO documents (id, access_hash, file_reference, date, mime_type, size, dc_id, attributes, thumbs)
|
||||
INSERT INTO documents (id, access_hash, file_reference, date, mime_type, size, dc_id, attributes, thumbs, owner_user_id)
|
||||
VALUES (
|
||||
$1::bigint,
|
||||
$2::bigint,
|
||||
|
|
@ -1005,7 +1242,8 @@ VALUES (
|
|||
$6::bigint,
|
||||
$7::int,
|
||||
$8::jsonb,
|
||||
$9::jsonb
|
||||
$9::jsonb,
|
||||
$10::bigint
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
access_hash = EXCLUDED.access_hash,
|
||||
|
|
@ -1028,9 +1266,14 @@ type PutDocumentParams struct {
|
|||
DcID int32
|
||||
AttributesJson []byte
|
||||
ThumbsJson []byte
|
||||
OwnerUserID int64
|
||||
}
|
||||
|
||||
// documents -------------------------------------------------------------------
|
||||
// owner_user_id is intentionally NOT in the UPDATE SET list: a document id is
|
||||
// only ever (re-)upserted by its original uploader's own request replay, and
|
||||
// keeping the first-write owner sticky avoids any risk of a later call
|
||||
// (e.g. a forward re-touching the row) reassigning ownership.
|
||||
func (q *Queries) PutDocument(ctx context.Context, arg PutDocumentParams) error {
|
||||
_, err := q.db.Exec(ctx, putDocument,
|
||||
arg.ID,
|
||||
|
|
@ -1042,6 +1285,7 @@ func (q *Queries) PutDocument(ctx context.Context, arg PutDocumentParams) error
|
|||
arg.DcID,
|
||||
arg.AttributesJson,
|
||||
arg.ThumbsJson,
|
||||
arg.OwnerUserID,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
|
@ -1089,7 +1333,7 @@ func (q *Queries) PutFileBlob(ctx context.Context, arg PutFileBlobParams) error
|
|||
|
||||
const putPhoto = `-- name: PutPhoto :exec
|
||||
|
||||
INSERT INTO photos (id, access_hash, file_reference, date, dc_id, has_stickers, sizes)
|
||||
INSERT INTO photos (id, access_hash, file_reference, date, dc_id, has_stickers, sizes, owner_user_id)
|
||||
VALUES (
|
||||
$1::bigint,
|
||||
$2::bigint,
|
||||
|
|
@ -1097,7 +1341,8 @@ VALUES (
|
|||
$4::int,
|
||||
$5::int,
|
||||
$6::boolean,
|
||||
$7::jsonb
|
||||
$7::jsonb,
|
||||
$8::bigint
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
access_hash = EXCLUDED.access_hash,
|
||||
|
|
@ -1116,9 +1361,11 @@ type PutPhotoParams struct {
|
|||
DcID int32
|
||||
HasStickers bool
|
||||
SizesJson []byte
|
||||
OwnerUserID int64
|
||||
}
|
||||
|
||||
// photos ----------------------------------------------------------------------
|
||||
// owner_user_id intentionally not updated on conflict, see PutDocument.
|
||||
func (q *Queries) PutPhoto(ctx context.Context, arg PutPhotoParams) error {
|
||||
_, err := q.db.Exec(ctx, putPhoto,
|
||||
arg.ID,
|
||||
|
|
@ -1128,6 +1375,7 @@ func (q *Queries) PutPhoto(ctx context.Context, arg PutPhotoParams) error {
|
|||
arg.DcID,
|
||||
arg.HasStickers,
|
||||
arg.SizesJson,
|
||||
arg.OwnerUserID,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
|
@ -1244,6 +1492,31 @@ func (q *Queries) PutStickerSet(ctx context.Context, arg PutStickerSetParams) er
|
|||
return err
|
||||
}
|
||||
|
||||
const removeMediaReference = `-- name: RemoveMediaReference :exec
|
||||
DELETE FROM media_references
|
||||
WHERE media_kind = $1::text
|
||||
AND media_id = $2::bigint
|
||||
AND ref_kind = $3::text
|
||||
AND ref_key = $4::text
|
||||
`
|
||||
|
||||
type RemoveMediaReferenceParams struct {
|
||||
MediaKind string
|
||||
MediaID int64
|
||||
RefKind string
|
||||
RefKey string
|
||||
}
|
||||
|
||||
func (q *Queries) RemoveMediaReference(ctx context.Context, arg RemoveMediaReferenceParams) error {
|
||||
_, err := q.db.Exec(ctx, removeMediaReference,
|
||||
arg.MediaKind,
|
||||
arg.MediaID,
|
||||
arg.RefKind,
|
||||
arg.RefKey,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const saveUploadPart = `-- name: SaveUploadPart :exec
|
||||
|
||||
INSERT INTO upload_parts (owner_user_id, file_id, part, total_parts, is_big, backend, object_key, size, sha256)
|
||||
|
|
@ -1295,3 +1568,18 @@ func (q *Queries) SaveUploadPart(ctx context.Context, arg SaveUploadPartParams)
|
|||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const sumFileBlobBytes = `-- name: SumFileBlobBytes :one
|
||||
SELECT COALESCE(SUM(size), 0)::bigint FROM file_blobs
|
||||
`
|
||||
|
||||
// Physical bytes actually held by the blob backend (dedup-aware: identical
|
||||
// content uploaded by different users is one row here). Used by the
|
||||
// low-space guard's cached usage gauge and the admin panel's "physical
|
||||
// usage" stat.
|
||||
func (q *Queries) SumFileBlobBytes(ctx context.Context) (int64, error) {
|
||||
row := q.db.QueryRow(ctx, sumFileBlobBytes)
|
||||
var column_1 int64
|
||||
err := row.Scan(&column_1)
|
||||
return column_1, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,6 +79,34 @@ type AccountPrivacyRule struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type AccountRating struct {
|
||||
UserID int64
|
||||
Level int32
|
||||
Stars int64
|
||||
CurrentLevelStars int64
|
||||
NextLevelStars *int64
|
||||
StarsComponent int64
|
||||
ActivityComponent int64
|
||||
PenaltyComponent int64
|
||||
ManualComponent int64
|
||||
PendingStars int64
|
||||
PendingDate pgtype.Timestamptz
|
||||
ComputedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
Version int64
|
||||
}
|
||||
|
||||
type AccountRatingEvent struct {
|
||||
ID int64
|
||||
UserID int64
|
||||
Kind string
|
||||
Amount int64
|
||||
Reason string
|
||||
Actor string
|
||||
CommandKey *string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type AccountReactionSetting struct {
|
||||
UserID int64
|
||||
MessagesNotifyFrom string
|
||||
|
|
@ -218,6 +246,21 @@ type AttachMenuUserState struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type AuthDeliveryReport struct {
|
||||
ID int64
|
||||
AuthKeyID []byte
|
||||
SessionID int64
|
||||
ClientType string
|
||||
PhoneHash []byte
|
||||
CodeHash []byte
|
||||
IssuedUserID int64
|
||||
DeliveryID string
|
||||
Channel string
|
||||
Mnc string
|
||||
Fingerprint []byte
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type AuthKey struct {
|
||||
AuthKeyID int64
|
||||
Body []byte
|
||||
|
|
@ -448,6 +491,20 @@ type BotUserPermission struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type BotVerifierSetting struct {
|
||||
BotID int64
|
||||
IconDocumentID int64
|
||||
CompanyName string
|
||||
DefaultDescription string
|
||||
CanModifyCustomDescription bool
|
||||
Enabled bool
|
||||
GrantedBy string
|
||||
GrantReason string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
Version int64
|
||||
}
|
||||
|
||||
type BusinessAutomationDelivery struct {
|
||||
OwnerUserID int64
|
||||
PeerUserID int64
|
||||
|
|
@ -577,6 +634,18 @@ type ChannelAdminLogEvent struct {
|
|||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ChannelAntispamDecision struct {
|
||||
ID int64
|
||||
ChannelID int64
|
||||
MessageID int32
|
||||
AuthorUserID int64
|
||||
EvidenceSchemaVersion int16
|
||||
Evidence []byte
|
||||
EvidenceHash []byte
|
||||
ReportID *int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ChannelBoostSlot struct {
|
||||
UserID int64
|
||||
Slot int32
|
||||
|
|
@ -689,25 +758,27 @@ type ChannelMediaCategoryCount struct {
|
|||
}
|
||||
|
||||
type ChannelMember struct {
|
||||
ChannelID int64
|
||||
UserID int64
|
||||
InviterUserID int64
|
||||
Role string
|
||||
Status string
|
||||
JoinedAt int32
|
||||
LeftAt int32
|
||||
AdminRights []byte
|
||||
BannedRights []byte
|
||||
Rank string
|
||||
AvailableMinID int32
|
||||
AvailableMinPts int32
|
||||
ReadInboxMaxID int32
|
||||
ReadInboxDate int32
|
||||
ReadOutboxMaxID int32
|
||||
UnreadMark bool
|
||||
SlowmodeLastSendDate int32
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
ChannelID int64
|
||||
UserID int64
|
||||
InviterUserID int64
|
||||
Role string
|
||||
Status string
|
||||
JoinedAt int32
|
||||
LeftAt int32
|
||||
AdminRights []byte
|
||||
BannedRights []byte
|
||||
Rank string
|
||||
AvailableMinID int32
|
||||
AvailableMinPts int32
|
||||
ReadInboxMaxID int32
|
||||
ReadInboxDate int32
|
||||
ReadOutboxMaxID int32
|
||||
UnreadMark bool
|
||||
SlowmodeLastSendDate int32
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
HistoryClearAnchorID int32
|
||||
HistoryClearAnchorDate int32
|
||||
}
|
||||
|
||||
type ChannelMessage struct {
|
||||
|
|
@ -910,6 +981,55 @@ type ChatlistMembership struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ClientTelemetryEvent struct {
|
||||
ID int64
|
||||
UserID int64
|
||||
Kind string
|
||||
PeerType string
|
||||
PeerID int64
|
||||
SubjectIds []int64
|
||||
Payload []byte
|
||||
Fingerprint []byte
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type CollectibleUsername struct {
|
||||
ID int64
|
||||
Username string
|
||||
UsernameLower string
|
||||
Status string
|
||||
OwnerPeerType string
|
||||
OwnerPeerID int64
|
||||
PurchaseDate pgtype.Timestamptz
|
||||
Currency string
|
||||
Amount int64
|
||||
CryptoCurrency string
|
||||
CryptoAmount int64
|
||||
Url string
|
||||
OriginalOwnerPeerType string
|
||||
OriginalOwnerPeerID int64
|
||||
TransferCount int32
|
||||
Version int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type CollectibleUsernameTransfer struct {
|
||||
ID int64
|
||||
CollectibleID int64
|
||||
Kind string
|
||||
FromPeerType string
|
||||
FromPeerID int64
|
||||
ToPeerType string
|
||||
ToPeerID int64
|
||||
Currency string
|
||||
Amount int64
|
||||
Actor string
|
||||
Reason string
|
||||
CommandKey *string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type Community struct {
|
||||
ID int64
|
||||
AccessHash int64
|
||||
|
|
@ -1008,6 +1128,41 @@ type CountryCode struct {
|
|||
OrderIndex int32
|
||||
}
|
||||
|
||||
type CustomVerification struct {
|
||||
ID int64
|
||||
VerifierBotID int64
|
||||
PeerType string
|
||||
PeerID int64
|
||||
IconDocumentID int64
|
||||
Description string
|
||||
GrantedByUserID int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
Version int64
|
||||
}
|
||||
|
||||
type CustomVerificationRequest struct {
|
||||
ID int64
|
||||
VerifierBotID int64
|
||||
ApplicantUserID int64
|
||||
PeerType string
|
||||
PeerID int64
|
||||
PeerTitle string
|
||||
PeerUsername string
|
||||
Reason string
|
||||
RequestedDescription string
|
||||
Status string
|
||||
DecidedBy string
|
||||
DecisionReason string
|
||||
InternalNote string
|
||||
CorrelationID string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
ApprovedAt pgtype.Timestamptz
|
||||
RejectedAt pgtype.Timestamptz
|
||||
Version int64
|
||||
}
|
||||
|
||||
type Dialog struct {
|
||||
UserID int64
|
||||
PeerType string
|
||||
|
|
@ -1095,6 +1250,8 @@ type Document struct {
|
|||
Attributes []byte
|
||||
Thumbs []byte
|
||||
CreatedAt pgtype.Timestamptz
|
||||
OwnerUserID int64
|
||||
OrphanedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type EncryptedFile struct {
|
||||
|
|
@ -1286,6 +1443,14 @@ type LoginCodeMessageDelivery struct {
|
|||
ExpiresAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type MediaReference struct {
|
||||
MediaKind string
|
||||
MediaID int64
|
||||
RefKind string
|
||||
RefKey string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type MessageBox struct {
|
||||
OwnerUserID int64
|
||||
BoxID int32
|
||||
|
|
@ -1343,6 +1508,126 @@ type MessageBoxMedium struct {
|
|||
MessageDate int32
|
||||
}
|
||||
|
||||
type ModerationAction struct {
|
||||
ID int64
|
||||
CaseID int64
|
||||
DecisionID int64
|
||||
Kind string
|
||||
Payload []byte
|
||||
Status string
|
||||
Attempts int32
|
||||
AvailableAt pgtype.Timestamptz
|
||||
LeaseUntil pgtype.Timestamptz
|
||||
LastError string
|
||||
CommandID string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationAppeal struct {
|
||||
ID int64
|
||||
CaseID int64
|
||||
AppellantUserID int64
|
||||
AppealText string
|
||||
TextHash []byte
|
||||
Fingerprint []byte
|
||||
Status string
|
||||
PreviousCaseStatus string
|
||||
Reviewer string
|
||||
ReviewReason string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
ReviewedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationAppealLink struct {
|
||||
ID int64
|
||||
CaseID int64
|
||||
AppellantUserID int64
|
||||
TokenHash []byte
|
||||
ExpiresAt pgtype.Timestamptz
|
||||
AppealID *int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
ConsumedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationCase struct {
|
||||
ID int64
|
||||
TargetPeerType string
|
||||
TargetPeerID int64
|
||||
Status string
|
||||
Severity int16
|
||||
AssignedTo string
|
||||
Version int64
|
||||
ReportCount int32
|
||||
DistinctReporterCount int32
|
||||
FirstReportAt pgtype.Timestamptz
|
||||
LastReportAt pgtype.Timestamptz
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationCaseReport struct {
|
||||
CaseID int64
|
||||
ReportID int64
|
||||
AttachedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationDecision struct {
|
||||
ID int64
|
||||
CaseID int64
|
||||
AppealID *int64
|
||||
Kind string
|
||||
Actor string
|
||||
Reason string
|
||||
CommandID string
|
||||
Fingerprint []byte
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationLegacyEphemeralMigration struct {
|
||||
LegacyReportID int64
|
||||
ModerationReportID int64
|
||||
MigratedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationMediaHold struct {
|
||||
ReportID int64
|
||||
ItemOrdinal int16
|
||||
MediaKind string
|
||||
StorageKey string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
ReleasedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationReport struct {
|
||||
ID int64
|
||||
ReporterUserID int64
|
||||
Source string
|
||||
TargetPeerType string
|
||||
TargetPeerID int64
|
||||
Reason string
|
||||
ReportOption string
|
||||
ReportComment string
|
||||
CommentHash []byte
|
||||
Fingerprint []byte
|
||||
TaxonomyVersion int16
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ModerationReportItem struct {
|
||||
ReportID int64
|
||||
Ordinal int16
|
||||
ItemKind string
|
||||
PeerType string
|
||||
PeerID int64
|
||||
ItemID int64
|
||||
SecondaryID int64
|
||||
AuthorUserID int64
|
||||
EvidenceSchemaVersion int16
|
||||
Evidence []byte
|
||||
EvidenceHash []byte
|
||||
}
|
||||
|
||||
type NotifySetting struct {
|
||||
OwnerUserID int64
|
||||
ScopeKind string
|
||||
|
|
@ -1412,6 +1697,11 @@ type PeerUsername struct {
|
|||
PeerType string
|
||||
PeerID int64
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
Username string
|
||||
Active bool
|
||||
Editable bool
|
||||
SortOrder int32
|
||||
CollectibleID *int64
|
||||
}
|
||||
|
||||
type Photo struct {
|
||||
|
|
@ -1423,6 +1713,8 @@ type Photo struct {
|
|||
HasStickers bool
|
||||
Sizes []byte
|
||||
CreatedAt pgtype.Timestamptz
|
||||
OwnerUserID int64
|
||||
OrphanedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type Poll struct {
|
||||
|
|
@ -1518,6 +1810,24 @@ type PrivateMessageReaction struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type PrivateNoForwardsChat struct {
|
||||
UserLowID int64
|
||||
UserHighID int64
|
||||
EnabledByUserID *int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type PrivateNoForwardsRequest struct {
|
||||
PrivateMessageSenderUserID int64
|
||||
PrivateMessageID int64
|
||||
RequesterUserID int64
|
||||
ResponderUserID int64
|
||||
ExpiresAt int32
|
||||
HandledAt int32
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ProfilePhoto struct {
|
||||
OwnerPeerType string
|
||||
OwnerPeerID int64
|
||||
|
|
@ -1568,6 +1878,16 @@ type SavedDialogPin struct {
|
|||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type SavedMessageReactionTag struct {
|
||||
UserID int64
|
||||
MessageBoxID int32
|
||||
ReactionType string
|
||||
ReactionValue string
|
||||
ChosenOrder int32
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type SavedMusic struct {
|
||||
UserID int64
|
||||
DocumentID int64
|
||||
|
|
@ -1645,6 +1965,21 @@ type SeedState struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type SponsoredMessageImpression struct {
|
||||
ID int64
|
||||
UserID int64
|
||||
RandomIDHash []byte
|
||||
TargetPeerType string
|
||||
TargetPeerID int64
|
||||
AuthorUserID int64
|
||||
EvidenceSchemaVersion int16
|
||||
Evidence []byte
|
||||
EvidenceHash []byte
|
||||
ReportID *int64
|
||||
CreatedAt pgtype.Timestamptz
|
||||
ExpiresAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type StarGiftAdminGrantCommand struct {
|
||||
RecipientUserID int64
|
||||
CommandKey string
|
||||
|
|
@ -1781,6 +2116,22 @@ type StarGiftCatalogRevision struct {
|
|||
BackgroundTextColor *int32
|
||||
}
|
||||
|
||||
// Purchase-time snapshot of per-admin channel gift notification intents; delivery uses deterministic private-message replay.
|
||||
type StarGiftChannelNotificationJob struct {
|
||||
SavedGiftID int64
|
||||
TargetUserID int64
|
||||
GiftDate int32
|
||||
Action []byte
|
||||
Attempts int32
|
||||
NextAttemptAt int32
|
||||
LeaseUntil int32
|
||||
DeliveredAt int32
|
||||
MessageID int32
|
||||
LastError string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type StarGiftCollectibleBackdrop struct {
|
||||
ID int64
|
||||
CollectibleRevisionID int64
|
||||
|
|
@ -2072,7 +2423,7 @@ type StarGiftUpgradeCommand struct {
|
|||
SourceEditPts int32
|
||||
}
|
||||
|
||||
// Owner-local service-message aliases (unique outputs and separate prepaid-upgrade notifications) for one saved gift aggregate.
|
||||
// Viewer-local private service-message aliases to saved gift aggregates; the saved gift owner may be that user or an authorized channel.
|
||||
type StarGiftUserMessageRef struct {
|
||||
OwnerUserID int64
|
||||
MsgID int32
|
||||
|
|
@ -2115,6 +2466,55 @@ type StarsBalance struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type StarsGiveaway struct {
|
||||
ID int64
|
||||
BuyerUserID int64
|
||||
FormID int64
|
||||
ChannelID int64
|
||||
LaunchMessageID int32
|
||||
RandomID int64
|
||||
Stars int64
|
||||
Users int32
|
||||
PerUserStars int64
|
||||
YearlyBoosts int32
|
||||
UntilDate int32
|
||||
PurposeJson []byte
|
||||
State string
|
||||
CreatedAt int32
|
||||
}
|
||||
|
||||
type StarsPurchaseCommand struct {
|
||||
BuyerUserID int64
|
||||
FormID int64
|
||||
RequestFingerprint []byte
|
||||
RecipientUserID *int64
|
||||
Stars int64
|
||||
Currency string
|
||||
Amount int64
|
||||
BalanceAfter int64
|
||||
TransactionID string
|
||||
CreatedAt int32
|
||||
Kind string
|
||||
SpendPeerType *string
|
||||
SpendPeerID *int64
|
||||
PurposeJson []byte
|
||||
}
|
||||
|
||||
type StarsPurchaseForm struct {
|
||||
BuyerUserID int64
|
||||
FormID int64
|
||||
RecipientUserID *int64
|
||||
Stars int64
|
||||
Currency string
|
||||
Amount int64
|
||||
IssuedAt int32
|
||||
ExpiresAt int32
|
||||
Kind string
|
||||
SpendPeerType *string
|
||||
SpendPeerID *int64
|
||||
PurposeJson []byte
|
||||
}
|
||||
|
||||
type StarsTransaction struct {
|
||||
ID int64
|
||||
UserID int64
|
||||
|
|
@ -2502,18 +2902,21 @@ type UserBusinessProfile struct {
|
|||
}
|
||||
|
||||
type UserChannelMemberIndex struct {
|
||||
UserID int64
|
||||
ChannelID int64
|
||||
Status string
|
||||
Megagroup bool
|
||||
Broadcast bool
|
||||
Deleted bool
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
Role string
|
||||
LeftAt int32
|
||||
Forum bool
|
||||
PublicUsername bool
|
||||
CanPinMessages bool
|
||||
UserID int64
|
||||
ChannelID int64
|
||||
Status string
|
||||
Megagroup bool
|
||||
Broadcast bool
|
||||
Deleted bool
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
Role string
|
||||
LeftAt int32
|
||||
Forum bool
|
||||
PublicUsername bool
|
||||
CanPinMessages bool
|
||||
AvailableMinID int32
|
||||
HistoryClearAnchorID int32
|
||||
HistoryClearUpdatedAt int32
|
||||
}
|
||||
|
||||
type UserRecentReaction struct {
|
||||
|
|
@ -2530,6 +2933,7 @@ type UserSavedReactionTag struct {
|
|||
ReactionType string
|
||||
ReactionValue string
|
||||
Title string
|
||||
// Legacy unused column; visible counts are aggregated from saved_message_reaction_tags.
|
||||
ReactionCount int32
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
|
|
@ -2607,6 +3011,67 @@ type UserUpdateWatermark struct {
|
|||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type VerificationApplication struct {
|
||||
ID int64
|
||||
ApplicantUserID int64
|
||||
TargetType string
|
||||
TargetID int64
|
||||
TargetTitle string
|
||||
TargetUsername string
|
||||
TargetAccessHash int64
|
||||
Category string
|
||||
Description string
|
||||
OfficialWebsite string
|
||||
SocialLinks []string
|
||||
PressLinks []string
|
||||
AdditionalNote string
|
||||
Status string
|
||||
ReviewerAdminID string
|
||||
DecisionReason string
|
||||
InternalNote string
|
||||
CorrelationID string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
SubmittedAt pgtype.Timestamptz
|
||||
ReviewedAt pgtype.Timestamptz
|
||||
Version int64
|
||||
}
|
||||
|
||||
type VerificationApplicationEvent struct {
|
||||
ID int64
|
||||
ApplicationID int64
|
||||
Kind string
|
||||
FromStatus string
|
||||
ToStatus string
|
||||
Actor string
|
||||
Reason string
|
||||
Note string
|
||||
CorrelationID string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type VerificationIcon struct {
|
||||
ID int64
|
||||
DocumentID int64
|
||||
OwnerBotID int64
|
||||
Name string
|
||||
Active bool
|
||||
CreatedAt pgtype.Timestamptz
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type VerificationNotificationOutbox struct {
|
||||
ID int64
|
||||
ApplicationID int64
|
||||
RecipientUserID int64
|
||||
Kind string
|
||||
Payload []byte
|
||||
Attempts int32
|
||||
DeliveredAt pgtype.Timestamptz
|
||||
LastError string
|
||||
CreatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type WebAuthorization struct {
|
||||
Hash int64
|
||||
RequestID int64
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ func TestStarGiftLifecycleMigrationsApply(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("migrate star gift lifecycle schema: %v", err)
|
||||
}
|
||||
if status.Dirty || status.Empty || status.Version != 20260714003127 {
|
||||
t.Fatalf("migration status = %+v, want clean version 20260714003127", status)
|
||||
if status.Dirty || status.Empty || status.Version != 20260714003129 {
|
||||
t.Fatalf("migration status = %+v, want clean version 20260714003129", status)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue