Initial open source release
This commit is contained in:
A
commit
74992e893f
377 changed files with 118084 additions and 0 deletions
236
internal/app/files/blobcache.go
Normal file
236
internal/app/files/blobcache.go
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"sync"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// blobMetaCache 是 location_key → FileBlob 元数据的进程内 LRU，用于消除 upload.getFile
|
||||
// 每个 chunk 一次 GetFileBlob 的 PG 往返（一个文件按 ≤512KB/1MB 分多次 getFile，热门贴纸/
|
||||
// reaction/头像更被大量用户重复拉）。
|
||||
//
|
||||
// FileBlob 元数据小（约百字节）且内容不可变：location_key 一旦写入即固定指向同一 object_key，
|
||||
// 新建 blob 用随机 id 生成 location_key 不会与已缓存项冲突，故只读填充、无需失效。
|
||||
type blobMetaCache struct {
|
||||
mu sync.Mutex
|
||||
cap int
|
||||
ll *list.List
|
||||
m map[string]*list.Element
|
||||
}
|
||||
|
||||
type blobMetaEntry struct {
|
||||
key string
|
||||
blob domain.FileBlob
|
||||
}
|
||||
|
||||
func newBlobMetaCache(capacity int) *blobMetaCache {
|
||||
if capacity <= 0 {
|
||||
capacity = 1
|
||||
}
|
||||
return &blobMetaCache{
|
||||
cap: capacity,
|
||||
ll: list.New(),
|
||||
m: make(map[string]*list.Element, capacity),
|
||||
}
|
||||
}
|
||||
|
||||
// get 返回缓存的 FileBlob 并把其移到 LRU 头部；未命中返回 ok=false。
|
||||
func (c *blobMetaCache) get(key string) (domain.FileBlob, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if el, ok := c.m[key]; ok {
|
||||
c.ll.MoveToFront(el)
|
||||
return el.Value.(*blobMetaEntry).blob, true
|
||||
}
|
||||
return domain.FileBlob{}, false
|
||||
}
|
||||
|
||||
// put 写入/更新缓存，超出容量时淘汰最久未用项。
|
||||
func (c *blobMetaCache) put(key string, blob domain.FileBlob) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if el, ok := c.m[key]; ok {
|
||||
el.Value.(*blobMetaEntry).blob = blob
|
||||
c.ll.MoveToFront(el)
|
||||
return
|
||||
}
|
||||
c.m[key] = c.ll.PushFront(&blobMetaEntry{key: key, blob: blob})
|
||||
if c.ll.Len() > c.cap {
|
||||
if oldest := c.ll.Back(); oldest != nil {
|
||||
c.ll.Remove(oldest)
|
||||
delete(c.m, oldest.Value.(*blobMetaEntry).key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// blobBytesCache 是 object_key → 小 blob 全量字节的 LRU。Sticker / reaction /
|
||||
// 缩略图通常只有几 KB 到几十 KB，缓存全量内容可以避开点击历史时的本地磁盘冷读抖动；
|
||||
// 大媒体仍由 BlobBackend.GetRange 分段读取，避免把大文件放进内存。
|
||||
type blobBytesCache struct {
|
||||
mu sync.Mutex
|
||||
maxBytes int
|
||||
used int
|
||||
ll *list.List
|
||||
m map[string]*list.Element
|
||||
}
|
||||
|
||||
type blobBytesEntry struct {
|
||||
key string
|
||||
bytes []byte
|
||||
size int
|
||||
}
|
||||
|
||||
func newBlobBytesCache(maxBytes int) *blobBytesCache {
|
||||
if maxBytes <= 0 {
|
||||
maxBytes = 1
|
||||
}
|
||||
return &blobBytesCache{
|
||||
maxBytes: maxBytes,
|
||||
ll: list.New(),
|
||||
m: make(map[string]*list.Element),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *blobBytesCache) get(key string) ([]byte, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if el, ok := c.m[key]; ok {
|
||||
c.ll.MoveToFront(el)
|
||||
entry := el.Value.(*blobBytesEntry)
|
||||
return append([]byte(nil), entry.bytes...), true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (c *blobBytesCache) has(key string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if el, ok := c.m[key]; ok {
|
||||
c.ll.MoveToFront(el)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *blobBytesCache) put(key string, bytes []byte) {
|
||||
if len(bytes) > c.maxBytes {
|
||||
return
|
||||
}
|
||||
copied := append([]byte(nil), bytes...)
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if el, ok := c.m[key]; ok {
|
||||
entry := el.Value.(*blobBytesEntry)
|
||||
c.used += len(copied) - entry.size
|
||||
entry.bytes = copied
|
||||
entry.size = len(copied)
|
||||
c.ll.MoveToFront(el)
|
||||
} else {
|
||||
entry := &blobBytesEntry{key: key, bytes: copied, size: len(copied)}
|
||||
c.m[key] = c.ll.PushFront(entry)
|
||||
c.used += entry.size
|
||||
}
|
||||
for c.used > c.maxBytes {
|
||||
oldest := c.ll.Back()
|
||||
if oldest == nil {
|
||||
break
|
||||
}
|
||||
c.ll.Remove(oldest)
|
||||
entry := oldest.Value.(*blobBytesEntry)
|
||||
delete(c.m, entry.key)
|
||||
c.used -= entry.size
|
||||
}
|
||||
}
|
||||
|
||||
type stickerSetFullCache struct {
|
||||
mu sync.RWMutex
|
||||
byID map[int64]stickerSetFullEntry
|
||||
byShort map[string]int64
|
||||
bySystem map[string]int64
|
||||
}
|
||||
|
||||
type stickerSetFullEntry struct {
|
||||
set domain.StickerSet
|
||||
docs []domain.Document
|
||||
}
|
||||
|
||||
func newStickerSetFullCache() *stickerSetFullCache {
|
||||
return &stickerSetFullCache{
|
||||
byID: map[int64]stickerSetFullEntry{},
|
||||
byShort: map[string]int64{},
|
||||
bySystem: map[string]int64{},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *stickerSetFullCache) get(ref domain.StickerSetRef) (domain.StickerSet, []domain.Document, bool) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
var id int64
|
||||
switch ref.Kind {
|
||||
case domain.StickerSetRefByID:
|
||||
id = ref.ID
|
||||
case domain.StickerSetRefByShortName:
|
||||
id = c.byShort[ref.ShortName]
|
||||
case domain.StickerSetRefBySystem:
|
||||
id = c.bySystem[ref.SystemKey]
|
||||
default:
|
||||
return domain.StickerSet{}, nil, false
|
||||
}
|
||||
entry, ok := c.byID[id]
|
||||
if !ok {
|
||||
return domain.StickerSet{}, nil, false
|
||||
}
|
||||
return copyStickerSet(entry.set), copyDocuments(entry.docs), true
|
||||
}
|
||||
|
||||
func (c *stickerSetFullCache) put(set domain.StickerSet, docs []domain.Document) {
|
||||
if set.ID == 0 {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.byID[set.ID] = stickerSetFullEntry{
|
||||
set: copyStickerSet(set),
|
||||
docs: copyDocuments(docs),
|
||||
}
|
||||
if set.ShortName != "" {
|
||||
c.byShort[set.ShortName] = set.ID
|
||||
}
|
||||
if set.SystemKey != "" {
|
||||
c.bySystem[set.SystemKey] = set.ID
|
||||
}
|
||||
}
|
||||
|
||||
func copyStickerSet(set domain.StickerSet) domain.StickerSet {
|
||||
set.DocumentIDs = append([]int64(nil), set.DocumentIDs...)
|
||||
set.Packs = append([]domain.StickerPack(nil), set.Packs...)
|
||||
for i := range set.Packs {
|
||||
set.Packs[i].DocumentIDs = append([]int64(nil), set.Packs[i].DocumentIDs...)
|
||||
}
|
||||
set.Thumbs = copyPhotoSizes(set.Thumbs)
|
||||
return set
|
||||
}
|
||||
|
||||
func copyDocuments(docs []domain.Document) []domain.Document {
|
||||
out := append([]domain.Document(nil), docs...)
|
||||
for i := range out {
|
||||
out[i].FileReference = append([]byte(nil), out[i].FileReference...)
|
||||
out[i].Attributes = append([]domain.DocumentAttribute(nil), out[i].Attributes...)
|
||||
for j := range out[i].Attributes {
|
||||
out[i].Attributes[j].Waveform = append([]byte(nil), out[i].Attributes[j].Waveform...)
|
||||
}
|
||||
out[i].Thumbs = copyPhotoSizes(out[i].Thumbs)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func copyPhotoSizes(sizes []domain.PhotoSize) []domain.PhotoSize {
|
||||
out := append([]domain.PhotoSize(nil), sizes...)
|
||||
for i := range out {
|
||||
out[i].Bytes = append([]byte(nil), out[i].Bytes...)
|
||||
out[i].Sizes = append([]int(nil), out[i].Sizes...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
233
internal/app/files/blobcache_test.go
Normal file
233
internal/app/files/blobcache_test.go
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
func TestBlobMetaCacheGetPutEvict(t *testing.T) {
|
||||
c := newBlobMetaCache(2)
|
||||
c.put("a", domain.FileBlob{LocationKey: "a", ObjectKey: "oa"})
|
||||
c.put("b", domain.FileBlob{LocationKey: "b", ObjectKey: "ob"})
|
||||
if b, ok := c.get("a"); !ok || b.ObjectKey != "oa" {
|
||||
t.Fatalf("get a = %+v ok=%v", b, ok)
|
||||
}
|
||||
// 容量 2：刚 access 过 a，再 put c 应淘汰最久未用的 b。
|
||||
c.put("c", domain.FileBlob{LocationKey: "c", ObjectKey: "oc"})
|
||||
if _, ok := c.get("b"); ok {
|
||||
t.Error("b should be evicted (least recently used)")
|
||||
}
|
||||
if _, ok := c.get("a"); !ok {
|
||||
t.Error("a should remain (recently used)")
|
||||
}
|
||||
if _, ok := c.get("c"); !ok {
|
||||
t.Error("c should be present")
|
||||
}
|
||||
}
|
||||
|
||||
// countingMediaStore 统计 GetFileBlob 次数，验证元数据缓存命中后不再查 PG。
|
||||
type countingMediaStore struct {
|
||||
*fakeMediaStore
|
||||
getBlobCalls int
|
||||
getSetByIDCalls int
|
||||
}
|
||||
|
||||
func (c *countingMediaStore) GetFileBlob(ctx context.Context, key string) (domain.FileBlob, bool, error) {
|
||||
c.getBlobCalls++
|
||||
return c.fakeMediaStore.GetFileBlob(ctx, key)
|
||||
}
|
||||
|
||||
func (c *countingMediaStore) GetStickerSetByID(ctx context.Context, id int64) (domain.StickerSet, bool, error) {
|
||||
c.getSetByIDCalls++
|
||||
return c.fakeMediaStore.GetStickerSetByID(ctx, id)
|
||||
}
|
||||
|
||||
type countingBlobBackend struct {
|
||||
BlobBackend
|
||||
getRangeCalls int
|
||||
}
|
||||
|
||||
func (c *countingBlobBackend) GetRange(ctx context.Context, objectKey string, offset, limit int64) ([]byte, int64, error) {
|
||||
c.getRangeCalls++
|
||||
return c.BlobBackend.GetRange(ctx, objectKey, offset, limit)
|
||||
}
|
||||
|
||||
func TestGetFileCachesMetadataAndSmallBlobBytes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
local, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("local fs: %v", err)
|
||||
}
|
||||
objectKey, err := local.Put(ctx, []byte("0123456789"))
|
||||
if err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
media := newFakeMediaStore()
|
||||
if err := media.PutFileBlob(ctx, domain.FileBlob{LocationKey: "doc:42", ObjectKey: objectKey, Size: 10, MimeType: "application/octet-stream"}); err != nil {
|
||||
t.Fatalf("put blob: %v", err)
|
||||
}
|
||||
counting := &countingMediaStore{fakeMediaStore: media}
|
||||
blobs := &countingBlobBackend{BlobBackend: local}
|
||||
svc := NewService(counting, blobs, 2)
|
||||
|
||||
// 第一次：查 PG 一次并填充元数据缓存；小 blob 读整块进字节缓存后返回 [0,5)。
|
||||
c1, ok, err := svc.GetFile(ctx, domain.FileDownloadRequest{LocationKey: "doc:42", Offset: 0, Limit: 5})
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("getfile1 ok=%v err=%v", ok, err)
|
||||
}
|
||||
if string(c1.Bytes) != "01234" {
|
||||
t.Errorf("chunk1 = %q, want 01234", c1.Bytes)
|
||||
}
|
||||
if c1.Total != 10 {
|
||||
t.Errorf("total = %d, want 10", c1.Total)
|
||||
}
|
||||
if counting.getBlobCalls != 1 {
|
||||
t.Errorf("getBlobCalls = %d, want 1", counting.getBlobCalls)
|
||||
}
|
||||
if blobs.getRangeCalls != 1 {
|
||||
t.Errorf("getRangeCalls = %d, want 1", blobs.getRangeCalls)
|
||||
}
|
||||
|
||||
// 第二次：同 location 命中元数据与字节缓存；[5,10) 直接从内存切片。
|
||||
c2, ok, err := svc.GetFile(ctx, domain.FileDownloadRequest{LocationKey: "doc:42", Offset: 5, Limit: 5})
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("getfile2 ok=%v err=%v", ok, err)
|
||||
}
|
||||
if string(c2.Bytes) != "56789" {
|
||||
t.Errorf("chunk2 = %q, want 56789", c2.Bytes)
|
||||
}
|
||||
if counting.getBlobCalls != 1 {
|
||||
t.Errorf("getBlobCalls = %d, want 1 (cache hit)", counting.getBlobCalls)
|
||||
}
|
||||
if blobs.getRangeCalls != 1 {
|
||||
t.Errorf("getRangeCalls = %d, want 1 (byte cache hit)", blobs.getRangeCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetFileDoesNotByteCacheLargeBlob(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
local, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("local fs: %v", err)
|
||||
}
|
||||
content := bytes.Repeat([]byte("x"), blobBytesCacheMaxEntryBytes+2)
|
||||
objectKey, err := local.Put(ctx, content)
|
||||
if err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
media := newFakeMediaStore()
|
||||
if err := media.PutFileBlob(ctx, domain.FileBlob{
|
||||
LocationKey: "doc:large",
|
||||
ObjectKey: objectKey,
|
||||
Size: int64(len(content)),
|
||||
MimeType: "application/octet-stream",
|
||||
}); err != nil {
|
||||
t.Fatalf("put blob: %v", err)
|
||||
}
|
||||
blobs := &countingBlobBackend{BlobBackend: local}
|
||||
svc := NewService(media, blobs, 2)
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
chunk, ok, err := svc.GetFile(ctx, domain.FileDownloadRequest{LocationKey: "doc:large", Offset: 1, Limit: 7})
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("getfile %d ok=%v err=%v", i, ok, err)
|
||||
}
|
||||
if string(chunk.Bytes) != "xxxxxxx" {
|
||||
t.Fatalf("chunk %d = %q, want seven x bytes", i, chunk.Bytes)
|
||||
}
|
||||
}
|
||||
if blobs.getRangeCalls != 2 {
|
||||
t.Errorf("getRangeCalls = %d, want 2 (large blob is not byte cached)", blobs.getRangeCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWarmCachesPreloadsStickerSetAndSmallBlobs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
local, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("local fs: %v", err)
|
||||
}
|
||||
mainKey, err := local.Put(ctx, []byte("sticker"))
|
||||
if err != nil {
|
||||
t.Fatalf("put main: %v", err)
|
||||
}
|
||||
thumbKey, err := local.Put(ctx, []byte("thumb"))
|
||||
if err != nil {
|
||||
t.Fatalf("put thumb: %v", err)
|
||||
}
|
||||
media := newFakeMediaStore()
|
||||
doc := domain.Document{
|
||||
ID: 100,
|
||||
AccessHash: 1,
|
||||
DCID: 2,
|
||||
MimeType: "application/x-tgsticker",
|
||||
Size: 7,
|
||||
Thumbs: []domain.PhotoSize{
|
||||
{Kind: domain.PhotoSizeKindDefault, Type: "m", W: 128, H: 128, Size: 5},
|
||||
},
|
||||
}
|
||||
if err := media.PutDocument(ctx, doc); err != nil {
|
||||
t.Fatalf("put doc: %v", err)
|
||||
}
|
||||
if err := media.PutFileBlob(ctx, domain.FileBlob{LocationKey: "doc:100", ObjectKey: mainKey, Size: 7, MimeType: doc.MimeType}); err != nil {
|
||||
t.Fatalf("put main blob: %v", err)
|
||||
}
|
||||
if err := media.PutFileBlob(ctx, domain.FileBlob{LocationKey: "doc:100:m", ObjectKey: thumbKey, Size: 5, MimeType: "image/jpeg"}); err != nil {
|
||||
t.Fatalf("put thumb blob: %v", err)
|
||||
}
|
||||
set := domain.StickerSet{
|
||||
ID: 200,
|
||||
AccessHash: 2,
|
||||
ShortName: "pack",
|
||||
Title: "Pack",
|
||||
Kind: domain.StickerSetKindStickers,
|
||||
Count: 1,
|
||||
DocumentIDs: []int64{
|
||||
doc.ID,
|
||||
},
|
||||
}
|
||||
if err := media.PutStickerSet(ctx, set); err != nil {
|
||||
t.Fatalf("put set: %v", err)
|
||||
}
|
||||
counting := &countingMediaStore{fakeMediaStore: media}
|
||||
blobs := &countingBlobBackend{BlobBackend: local}
|
||||
svc := NewService(counting, blobs, 2)
|
||||
|
||||
stats, err := svc.WarmCaches(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("warm caches: %v", err)
|
||||
}
|
||||
if stats.StickerSets != 1 || stats.Documents != 1 || stats.Blobs != 2 {
|
||||
t.Fatalf("warm stats = %+v, want 1 set, 1 doc, 2 blobs", stats)
|
||||
}
|
||||
blobs.getRangeCalls = 0
|
||||
chunk, ok, err := svc.GetFile(ctx, domain.FileDownloadRequest{LocationKey: "doc:100", Offset: 0, Limit: 7})
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("getfile ok=%v err=%v", ok, err)
|
||||
}
|
||||
if string(chunk.Bytes) != "sticker" {
|
||||
t.Fatalf("chunk = %q, want sticker", chunk.Bytes)
|
||||
}
|
||||
if blobs.getRangeCalls != 0 {
|
||||
t.Fatalf("prewarmed blob should be served from byte cache, GetRange calls = %d", blobs.getRangeCalls)
|
||||
}
|
||||
|
||||
gotSet, docs, found, err := svc.ResolveStickerSet(ctx, domain.StickerSetRef{Kind: domain.StickerSetRefByID, ID: set.ID})
|
||||
if err != nil || !found {
|
||||
t.Fatalf("resolve found=%v err=%v", found, err)
|
||||
}
|
||||
if gotSet.ID != set.ID || len(docs) != 1 || docs[0].ID != doc.ID {
|
||||
t.Fatalf("resolve = set %+v docs %+v", gotSet, docs)
|
||||
}
|
||||
if counting.getSetByIDCalls != 0 {
|
||||
t.Fatalf("ResolveStickerSet should hit full-set cache, GetStickerSetByID calls = %d", counting.getSetByIDCalls)
|
||||
}
|
||||
docs[0].ID = 999
|
||||
_, docsAgain, found, err := svc.ResolveStickerSet(ctx, domain.StickerSetRef{Kind: domain.StickerSetRefByID, ID: set.ID})
|
||||
if err != nil || !found || docsAgain[0].ID != doc.ID {
|
||||
t.Fatalf("cached docs were mutated: found=%v err=%v docs=%+v", found, err, docsAgain)
|
||||
}
|
||||
}
|
||||
105
internal/app/files/blobfs.go
Normal file
105
internal/app/files/blobfs.go
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// BlobBackend 是 blob 字节内容的存储后端。第一阶段只有本地磁盘实现。
|
||||
// 内容寻址：Put 返回的 objectKey 是内容 sha256，相同内容自动去重。
|
||||
type BlobBackend interface {
|
||||
Name() string
|
||||
Put(ctx context.Context, data []byte) (objectKey string, err error)
|
||||
Get(ctx context.Context, objectKey string) ([]byte, error)
|
||||
// GetRange 只读 [offset, offset+limit) 段并返回该段字节与文件总大小（limit<=0 读到末尾），
|
||||
// 避免大文件每个 chunk 都整文件读入内存（getFile 按 chunk 多次请求 ⇒ 否则 O(N²) 放大）。
|
||||
GetRange(ctx context.Context, objectKey string, offset, limit int64) (data []byte, total int64, err error)
|
||||
}
|
||||
|
||||
// LocalFS 把 blob 字节存到本地磁盘根目录下，路径按内容 hash 两级 fanout。
|
||||
type LocalFS struct {
|
||||
root string
|
||||
}
|
||||
|
||||
// NewLocalFS 创建本地磁盘 blob backend，确保根目录存在。
|
||||
func NewLocalFS(root string) (*LocalFS, error) {
|
||||
if root == "" {
|
||||
return nil, fmt.Errorf("blob root dir is empty")
|
||||
}
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create blob root %q: %w", root, err)
|
||||
}
|
||||
return &LocalFS{root: root}, nil
|
||||
}
|
||||
|
||||
// Name 返回后端标识，与 file_blobs.backend 一致。
|
||||
func (l *LocalFS) Name() string { return "localfs" }
|
||||
|
||||
func (l *LocalFS) pathFor(objectKey string) string {
|
||||
if len(objectKey) < 4 {
|
||||
return filepath.Join(l.root, "_", objectKey)
|
||||
}
|
||||
return filepath.Join(l.root, objectKey[:2], objectKey[2:4], objectKey)
|
||||
}
|
||||
|
||||
// Put 写入内容并返回 sha256 hex 作为 objectKey；同内容已存在则跳过写入（去重）。
|
||||
func (l *LocalFS) Put(_ context.Context, data []byte) (string, error) {
|
||||
sum := sha256.Sum256(data)
|
||||
key := hex.EncodeToString(sum[:])
|
||||
path := l.pathFor(key)
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return key, nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return "", fmt.Errorf("create blob dir: %w", err)
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0o644); err != nil {
|
||||
return "", fmt.Errorf("write blob: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
return "", fmt.Errorf("commit blob: %w", err)
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// Get 读取 objectKey 对应的全部字节。
|
||||
func (l *LocalFS) Get(_ context.Context, objectKey string) ([]byte, error) {
|
||||
return os.ReadFile(l.pathFor(objectKey))
|
||||
}
|
||||
|
||||
// GetRange 用 ReadAt 只读 [offset, offset+limit) 段，total 取自文件大小；
|
||||
// n 受 total 约束，故即便客户端传超大 limit 也只分配文件实际大小，不会按客户端巨值分配。
|
||||
func (l *LocalFS) GetRange(_ context.Context, objectKey string, offset, limit int64) ([]byte, int64, error) {
|
||||
f, err := os.Open(l.pathFor(objectKey))
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
total := info.Size()
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
if offset >= total {
|
||||
return []byte{}, total, nil
|
||||
}
|
||||
n := total - offset
|
||||
if limit > 0 && limit < n {
|
||||
n = limit
|
||||
}
|
||||
buf := make([]byte, n)
|
||||
read, err := f.ReadAt(buf, offset)
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, 0, err
|
||||
}
|
||||
return buf[:read], total, nil
|
||||
}
|
||||
92
internal/app/files/blobfs_test.go
Normal file
92
internal/app/files/blobfs_test.go
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLocalFSPutGetRoundTrip(t *testing.T) {
|
||||
fs, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("new local fs: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
data := []byte("hello telesrv media blob 你好")
|
||||
|
||||
key, err := fs.Put(ctx, data)
|
||||
if err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
if key == "" {
|
||||
t.Fatal("empty object key")
|
||||
}
|
||||
|
||||
// 内容寻址：相同内容应得到相同 key（去重）。
|
||||
key2, err := fs.Put(ctx, data)
|
||||
if err != nil {
|
||||
t.Fatalf("put again: %v", err)
|
||||
}
|
||||
if key != key2 {
|
||||
t.Fatalf("expected dedup key %q == %q", key, key2)
|
||||
}
|
||||
|
||||
got, err := fs.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Fatalf("roundtrip mismatch: got %q want %q", got, data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalFSDistinctContent(t *testing.T) {
|
||||
fs, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("new local fs: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
k1, _ := fs.Put(ctx, []byte("aaa"))
|
||||
k2, _ := fs.Put(ctx, []byte("bbb"))
|
||||
if k1 == k2 {
|
||||
t.Fatal("distinct content must yield distinct keys")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalFSGetRange(t *testing.T) {
|
||||
fs, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("new local fs: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
key, err := fs.Put(ctx, []byte("0123456789"))
|
||||
if err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
offset, limit int64
|
||||
want string
|
||||
}{
|
||||
{"head", 0, 4, "0123"},
|
||||
{"middle", 3, 4, "3456"},
|
||||
{"limit-exceeds-remaining", 7, 100, "789"},
|
||||
{"zero-limit-reads-to-end", 2, 0, "23456789"},
|
||||
{"offset-at-end", 10, 5, ""},
|
||||
{"offset-past-end", 20, 5, ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
data, total, err := fs.GetRange(ctx, key, tc.offset, tc.limit)
|
||||
if err != nil {
|
||||
t.Fatalf("getrange: %v", err)
|
||||
}
|
||||
if string(data) != tc.want {
|
||||
t.Errorf("data = %q, want %q", data, tc.want)
|
||||
}
|
||||
if total != 10 {
|
||||
t.Errorf("total = %d, want 10", total)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
6
internal/app/files/doc.go
Normal file
6
internal/app/files/doc.go
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
// Package files 是文件应用服务：upload 分片累积、blob 落盘、getFile 下载，
|
||||
// 以及把上传文件组装成 Photo / Document（头像、图片/文件/贴纸消息）。
|
||||
//
|
||||
// 类型边界：本包只用 domain / store 类型，不依赖 tg.*；
|
||||
// rpc 层负责 tg.InputFileLocation / InputMedia ↔ domain 的转换。
|
||||
package files
|
||||
278
internal/app/files/photos.go
Normal file
278
internal/app/files/photos.go
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/jpeg" // 注册 jpeg DecodeConfig，用于读取上传头像/图片尺寸
|
||||
_ "image/png" // 注册 png DecodeConfig
|
||||
"time"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// 头像与图片消息共用的尺寸 type：'a' 小图（≤160），'c' 大图，'x' 通用下载尺寸。
|
||||
// 同一份上传字节在多个 location_key 下建 blob（不做实际缩放，dev 主路径足够）。
|
||||
|
||||
// UploadProfilePhoto 把已上传文件组装成头像 Photo，落 blob/photos/profile_photos，并设为当前头像。
|
||||
func (s *Service) UploadProfilePhoto(ctx context.Context, ownerType domain.PeerType, ownerID int64, file domain.UploadedFileRef, date int) (domain.Photo, error) {
|
||||
data, err := s.assembleUpload(ctx, file.OwnerUserID, file.FileID, file.Parts)
|
||||
if err != nil {
|
||||
return domain.Photo{}, err
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return domain.Photo{}, domain.ErrPhotoInvalid
|
||||
}
|
||||
if date == 0 {
|
||||
date = int(time.Now().Unix())
|
||||
}
|
||||
photo, err := s.createPhoto(ctx, data, photoSizeSpecsForAvatar(data))
|
||||
if err != nil {
|
||||
return domain.Photo{}, err
|
||||
}
|
||||
if err := s.media.AddProfilePhoto(ctx, ownerType, ownerID, photo.ID, date); err != nil {
|
||||
return domain.Photo{}, err
|
||||
}
|
||||
return photo, nil
|
||||
}
|
||||
|
||||
// CreatePhotoFromUpload 把已上传文件组装成 Photo（不绑定 profile_photos），用于频道头像 / 图片消息。
|
||||
func (s *Service) CreatePhotoFromUpload(ctx context.Context, file domain.UploadedFileRef) (domain.Photo, error) {
|
||||
data, err := s.assembleUpload(ctx, file.OwnerUserID, file.FileID, file.Parts)
|
||||
if err != nil {
|
||||
return domain.Photo{}, err
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return domain.Photo{}, domain.ErrPhotoInvalid
|
||||
}
|
||||
return s.createPhoto(ctx, data, photoSizeSpecsForMessage(data))
|
||||
}
|
||||
|
||||
// GetPhoto 按 id 返回已存储照片。
|
||||
func (s *Service) GetPhoto(ctx context.Context, id int64) (domain.Photo, bool, error) {
|
||||
return s.media.GetPhoto(ctx, id)
|
||||
}
|
||||
|
||||
// GetDocument 按 id 返回已存储文档（贴纸 / 文件）。
|
||||
func (s *Service) GetDocument(ctx context.Context, id int64) (domain.Document, bool, error) {
|
||||
return s.media.GetDocument(ctx, id)
|
||||
}
|
||||
|
||||
// CreateAvatarFromUpload 把已上传文件组装成头像 Photo（'a'/'c' 尺寸，匹配 InputPeerPhotoFileLocation
|
||||
// big/small 与 channelFull 合成尺寸的下载路径），不绑定 profile_photos。用于频道 editPhoto。
|
||||
func (s *Service) CreateAvatarFromUpload(ctx context.Context, file domain.UploadedFileRef) (domain.Photo, error) {
|
||||
data, err := s.assembleUpload(ctx, file.OwnerUserID, file.FileID, file.Parts)
|
||||
if err != nil {
|
||||
return domain.Photo{}, err
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return domain.Photo{}, domain.ErrPhotoInvalid
|
||||
}
|
||||
return s.createPhoto(ctx, data, photoSizeSpecsForAvatar(data))
|
||||
}
|
||||
|
||||
// CreateDocumentFromUpload 把已上传文件组装成 Document（文件/视频/音频/gif/贴纸消息），落 blob + documents。
|
||||
func (s *Service) CreateDocumentFromUpload(ctx context.Context, file domain.UploadedFileRef, spec domain.DocumentSpec) (domain.Document, error) {
|
||||
data, err := s.assembleUpload(ctx, file.OwnerUserID, file.FileID, file.Parts)
|
||||
if err != nil {
|
||||
return domain.Document{}, err
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return domain.Document{}, domain.ErrDocumentInvalid
|
||||
}
|
||||
objectKey, err := s.blobs.Put(ctx, data)
|
||||
if err != nil {
|
||||
return domain.Document{}, err
|
||||
}
|
||||
docID := randomID()
|
||||
if err := s.media.PutFileBlob(ctx, domain.FileBlob{
|
||||
LocationKey: fmt.Sprintf("doc:%d", docID),
|
||||
Backend: domain.MediaBackend(s.blobs.Name()),
|
||||
ObjectKey: objectKey,
|
||||
Size: int64(len(data)),
|
||||
MimeType: spec.MimeType,
|
||||
}); err != nil {
|
||||
return domain.Document{}, err
|
||||
}
|
||||
doc := domain.Document{
|
||||
ID: docID,
|
||||
AccessHash: randomID(),
|
||||
FileReference: randomFileReference(),
|
||||
Date: int(time.Now().Unix()),
|
||||
MimeType: spec.MimeType,
|
||||
Size: int64(len(data)),
|
||||
DCID: s.dc,
|
||||
Attributes: spec.Attributes,
|
||||
}
|
||||
if spec.Thumb != nil {
|
||||
thumbData, err := s.assembleUpload(ctx, spec.Thumb.OwnerUserID, spec.Thumb.FileID, spec.Thumb.Parts)
|
||||
if err == nil && len(thumbData) > 0 {
|
||||
thumbKey, err := s.blobs.Put(ctx, thumbData)
|
||||
if err == nil {
|
||||
w, h := imageDimensions(thumbData, 0, 0)
|
||||
if err := s.media.PutFileBlob(ctx, domain.FileBlob{
|
||||
LocationKey: fmt.Sprintf("doc:%d:m", docID),
|
||||
Backend: domain.MediaBackend(s.blobs.Name()),
|
||||
ObjectKey: thumbKey,
|
||||
Size: int64(len(thumbData)),
|
||||
MimeType: "image/jpeg",
|
||||
}); err == nil {
|
||||
doc.Thumbs = []domain.PhotoSize{{Kind: domain.PhotoSizeKindDefault, Type: "m", W: w, H: h, Size: len(thumbData)}}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := s.media.PutDocument(ctx, doc); err != nil {
|
||||
return domain.Document{}, err
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
// SetCurrentProfilePhoto 把已存在的 photo 设为当前头像（updateProfilePhoto 选历史头像）。
|
||||
func (s *Service) SetCurrentProfilePhoto(ctx context.Context, ownerType domain.PeerType, ownerID, photoID int64, date int) (domain.Photo, bool, error) {
|
||||
photo, ok, err := s.media.GetPhoto(ctx, photoID)
|
||||
if err != nil || !ok {
|
||||
return domain.Photo{}, ok, err
|
||||
}
|
||||
if date == 0 {
|
||||
date = int(time.Now().Unix())
|
||||
}
|
||||
if err := s.media.AddProfilePhoto(ctx, ownerType, ownerID, photoID, date); err != nil {
|
||||
return domain.Photo{}, false, err
|
||||
}
|
||||
return photo, true, nil
|
||||
}
|
||||
|
||||
// CurrentProfilePhoto 返回某 owner 的当前头像 Photo。
|
||||
func (s *Service) CurrentProfilePhoto(ctx context.Context, ownerType domain.PeerType, ownerID int64) (domain.Photo, bool, error) {
|
||||
id, ok, err := s.media.CurrentProfilePhoto(ctx, ownerType, ownerID)
|
||||
if err != nil || !ok {
|
||||
return domain.Photo{}, ok, err
|
||||
}
|
||||
return s.media.GetPhoto(ctx, id)
|
||||
}
|
||||
|
||||
// GetProfilePhotos 返回 owner 的头像历史（最新在前）。
|
||||
func (s *Service) GetProfilePhotos(ctx context.Context, ownerType domain.PeerType, ownerID int64, offset, limit int, maxID int64) ([]domain.Photo, int, error) {
|
||||
ids, total, err := s.media.ListProfilePhotos(ctx, ownerType, ownerID, offset, limit, maxID)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
photos := make([]domain.Photo, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
if p, ok, err := s.media.GetPhoto(ctx, id); err != nil {
|
||||
return nil, 0, err
|
||||
} else if ok {
|
||||
photos = append(photos, p)
|
||||
}
|
||||
}
|
||||
return photos, total, nil
|
||||
}
|
||||
|
||||
// DeleteProfilePhotos 停用指定头像，返回成功停用数量。
|
||||
func (s *Service) DeleteProfilePhotos(ctx context.Context, ownerType domain.PeerType, ownerID int64, photoIDs []int64) (int, error) {
|
||||
deleted, err := s.media.DeleteProfilePhotos(ctx, ownerType, ownerID, photoIDs)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(deleted), nil
|
||||
}
|
||||
|
||||
// createPhoto 把字节落 blob（每个尺寸一个 location_key，指向同一内容）并写 photos 表。
|
||||
func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSizeSpec) (domain.Photo, error) {
|
||||
objectKey, err := s.blobs.Put(ctx, data)
|
||||
if err != nil {
|
||||
return domain.Photo{}, err
|
||||
}
|
||||
photoID := randomID()
|
||||
sizes := make([]domain.PhotoSize, 0, len(specs))
|
||||
for _, spec := range specs {
|
||||
if err := s.media.PutFileBlob(ctx, domain.FileBlob{
|
||||
LocationKey: fmt.Sprintf("photo:%d:%s", photoID, spec.Type),
|
||||
Backend: domain.MediaBackend(s.blobs.Name()),
|
||||
ObjectKey: objectKey,
|
||||
Size: int64(len(data)),
|
||||
MimeType: "image/jpeg",
|
||||
}); err != nil {
|
||||
return domain.Photo{}, err
|
||||
}
|
||||
sizes = append(sizes, domain.PhotoSize{Kind: domain.PhotoSizeKindDefault, Type: spec.Type, W: spec.W, H: spec.H, Size: len(data)})
|
||||
}
|
||||
photo := domain.Photo{
|
||||
ID: photoID,
|
||||
AccessHash: randomID(),
|
||||
FileReference: randomFileReference(),
|
||||
Date: int(time.Now().Unix()),
|
||||
DCID: s.dc,
|
||||
Sizes: sizes,
|
||||
}
|
||||
if err := s.media.PutPhoto(ctx, photo); err != nil {
|
||||
return domain.Photo{}, err
|
||||
}
|
||||
return photo, nil
|
||||
}
|
||||
|
||||
type photoSizeSpec struct {
|
||||
Type string
|
||||
W int
|
||||
H int
|
||||
}
|
||||
|
||||
func photoSizeSpecsForAvatar(data []byte) []photoSizeSpec {
|
||||
w, h := imageDimensions(data, 640, 640)
|
||||
small := 160
|
||||
if w < small {
|
||||
small = w
|
||||
}
|
||||
return []photoSizeSpec{
|
||||
{Type: "a", W: small, H: small},
|
||||
{Type: "c", W: w, H: h},
|
||||
}
|
||||
}
|
||||
|
||||
// photoSizeSpecsForMessage 给图片消息生成下载尺寸（'m' 缩略 + 'x'/'y' 大图）。
|
||||
func photoSizeSpecsForMessage(data []byte) []photoSizeSpec {
|
||||
w, h := imageDimensions(data, 1280, 1280)
|
||||
thumbW, thumbH := scaleDown(w, h, 320)
|
||||
return []photoSizeSpec{
|
||||
{Type: "m", W: thumbW, H: thumbH},
|
||||
{Type: "x", W: w, H: h},
|
||||
}
|
||||
}
|
||||
|
||||
func imageDimensions(data []byte, defW, defH int) (int, int) {
|
||||
cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil || cfg.Width <= 0 || cfg.Height <= 0 {
|
||||
return defW, defH
|
||||
}
|
||||
return cfg.Width, cfg.Height
|
||||
}
|
||||
|
||||
func scaleDown(w, h, max int) (int, int) {
|
||||
if w <= max && h <= max {
|
||||
return w, h
|
||||
}
|
||||
if w >= h {
|
||||
return max, max * h / w
|
||||
}
|
||||
return max * w / h, max
|
||||
}
|
||||
|
||||
func randomID() int64 {
|
||||
var b [8]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
v := int64(binary.BigEndian.Uint64(b[:]) >> 1)
|
||||
if v == 0 {
|
||||
v = 1
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func randomFileReference() []byte {
|
||||
b := make([]byte, 16)
|
||||
_, _ = rand.Read(b)
|
||||
return b
|
||||
}
|
||||
860
internal/app/files/seed.go
Normal file
860
internal/app/files/seed.go
Normal file
|
|
@ -0,0 +1,860 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// 本文件实现从外部导出的 reaction / sticker 资源目录导入媒体种子：
|
||||
// JSON 元数据（外部导出 document id/access_hash/file_reference/attributes/thumbs）落 documents /
|
||||
// sticker_sets / available_reactions 表，二进制 .tgs/.webp/缩略图落 blob backend。
|
||||
// dc_id 统一重写为本 server 的 DC，使客户端从本 DC 下载。导入幂等（已存在则跳过）。
|
||||
|
||||
// SeedStats 汇报一次种子导入结果。
|
||||
type SeedStats struct {
|
||||
Reactions int
|
||||
StickerSets int
|
||||
Documents int
|
||||
Blobs int
|
||||
Skipped bool
|
||||
}
|
||||
|
||||
// SeedMedia 从导出根目录导入 reaction 与 sticker 资源。maxRegularSets<=0 表示不限。
|
||||
func (s *Service) SeedMedia(ctx context.Context, root string, maxRegularSets int) (SeedStats, error) {
|
||||
var stats SeedStats
|
||||
if root == "" {
|
||||
stats.Skipped = true
|
||||
return stats, nil
|
||||
}
|
||||
if _, err := os.Stat(root); err != nil {
|
||||
// 目录不存在：跳过而非失败（开发机可能未放资源）。
|
||||
stats.Skipped = true
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// reactions
|
||||
if n, err := s.media.CountAvailableReactions(ctx); err != nil {
|
||||
return stats, err
|
||||
} else if n == 0 {
|
||||
if err := s.seedReactions(ctx, root, &stats); err != nil {
|
||||
return stats, fmt.Errorf("seed reactions: %w", err)
|
||||
}
|
||||
} else if incomplete, err := s.availableReactionSeedNeedsRepair(ctx); err != nil {
|
||||
return stats, err
|
||||
} else if incomplete {
|
||||
if err := s.seedReactions(ctx, root, &stats); err != nil {
|
||||
return stats, fmt.Errorf("repair reactions: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// sticker sets（default 系统集 + 常规集）
|
||||
if n, err := s.media.CountStickerSets(ctx); err != nil {
|
||||
return stats, err
|
||||
} else if n == 0 {
|
||||
if err := s.seedStickerSets(ctx, root, maxRegularSets, &stats); err != nil {
|
||||
return stats, fmt.Errorf("seed sticker sets: %w", err)
|
||||
}
|
||||
} else if stale, err := s.stickerSetDocumentThumbsNeedInlineCache(ctx); err != nil {
|
||||
return stats, err
|
||||
} else if stale {
|
||||
if err := s.seedStickerSets(ctx, root, maxRegularSets, &stats); err != nil {
|
||||
return stats, fmt.Errorf("repair sticker set thumbs: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if stats.Reactions == 0 && stats.StickerSets == 0 {
|
||||
stats.Skipped = true
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// ---- reactions ----
|
||||
|
||||
func (s *Service) seedReactions(ctx context.Context, root string, stats *SeedStats) error {
|
||||
reactionsDir := filepath.Join(root, "telegram_reactions_export", "reactions")
|
||||
rawPath := filepath.Join(root, "telegram_reactions_export", "global_json", "available_reactions_raw.json")
|
||||
raw, err := os.ReadFile(rawPath)
|
||||
if err != nil {
|
||||
return nil // 没有 reaction 资源就跳过
|
||||
}
|
||||
var parsed struct {
|
||||
Result struct {
|
||||
Reactions []seedReactionJSON `json:"reactions"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &parsed); err != nil {
|
||||
return fmt.Errorf("parse available_reactions_raw.json: %w", err)
|
||||
}
|
||||
index, err := scanSeedDir(reactionsDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i, rj := range parsed.Result.Reactions {
|
||||
ar := domain.AvailableReaction{
|
||||
Reaction: rj.Reaction,
|
||||
Title: rj.Title,
|
||||
Inactive: rj.Inactive,
|
||||
Premium: rj.Premium,
|
||||
Order: i,
|
||||
}
|
||||
set := func(dst *int64, d *seedDocumentJSON) error {
|
||||
if d == nil || d.ID == 0 {
|
||||
return nil
|
||||
}
|
||||
doc, err := s.importDocument(ctx, *d, reactionsDir, index, stats)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*dst = doc.ID
|
||||
return nil
|
||||
}
|
||||
if err := set(&ar.StaticIconID, rj.StaticIcon); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := set(&ar.AppearAnimationID, rj.AppearAnimation); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := set(&ar.SelectAnimationID, rj.SelectAnimation); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := set(&ar.ActivateAnimationID, rj.ActivateAnimation); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := set(&ar.EffectAnimationID, rj.EffectAnimation); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := set(&ar.AroundAnimationID, rj.AroundAnimation); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := set(&ar.CenterIconID, rj.CenterIcon); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.media.PutAvailableReaction(ctx, ar); err != nil {
|
||||
return err
|
||||
}
|
||||
stats.Reactions++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) availableReactionSeedNeedsRepair(ctx context.Context) (bool, error) {
|
||||
reactions, err := s.media.ListAvailableReactions(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var docIDs []int64
|
||||
for _, r := range reactions {
|
||||
for _, id := range r.DocumentIDs() {
|
||||
if id == 0 {
|
||||
continue
|
||||
}
|
||||
docIDs = append(docIDs, id)
|
||||
if _, ok, err := s.media.GetFileBlob(ctx, fmt.Sprintf("doc:%d", id)); err != nil {
|
||||
return false, err
|
||||
} else if !ok {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if stale, err := s.documentsNeedInlineCachedThumbs(ctx, docIDs); err != nil || stale {
|
||||
return stale, err
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// ---- sticker sets ----
|
||||
|
||||
func (s *Service) seedStickerSets(ctx context.Context, root string, maxRegular int, stats *SeedStats) error {
|
||||
// default 系统集：目录名 → system_key。
|
||||
defaultDir := filepath.Join(root, "telegram_default_stickers_export")
|
||||
order := 0
|
||||
if entries, err := os.ReadDir(defaultDir); err == nil {
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
systemKey := systemKeyForDefaultSet(name)
|
||||
setDir := filepath.Join(defaultDir, name)
|
||||
if err := s.importStickerSetDir(ctx, setDir, systemKey, order, stats); err != nil {
|
||||
return fmt.Errorf("import default set %s: %w", name, err)
|
||||
}
|
||||
order++
|
||||
}
|
||||
}
|
||||
|
||||
// 常规贴纸集。
|
||||
regularDir := filepath.Join(root, "telegram_stickers_export")
|
||||
if entries, err := os.ReadDir(regularDir); err == nil {
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
imported := 0
|
||||
for _, name := range names {
|
||||
if maxRegular > 0 && imported >= maxRegular {
|
||||
break
|
||||
}
|
||||
setDir := filepath.Join(regularDir, name)
|
||||
if err := s.importStickerSetDir(ctx, setDir, "", order, stats); err != nil {
|
||||
return fmt.Errorf("import sticker set %s: %w", name, err)
|
||||
}
|
||||
order++
|
||||
imported++
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) importStickerSetDir(ctx context.Context, setDir, systemKey string, order int, stats *SeedStats) error {
|
||||
infoPath := filepath.Join(setDir, "set_info.json")
|
||||
raw, err := os.ReadFile(infoPath)
|
||||
if err != nil {
|
||||
return nil // 该目录无 set_info.json → 跳过
|
||||
}
|
||||
var info struct {
|
||||
Result seedStickerSetResultJSON `json:"result"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &info); err != nil {
|
||||
return fmt.Errorf("parse %s: %w", infoPath, err)
|
||||
}
|
||||
sj := info.Result.Set
|
||||
if sj.ID == 0 {
|
||||
return nil
|
||||
}
|
||||
stickersDir := filepath.Join(setDir, "stickers")
|
||||
index, err := scanSeedDir(stickersDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
docIDs := make([]int64, 0, len(info.Result.Documents))
|
||||
docs := make([]domain.Document, 0, len(info.Result.Documents))
|
||||
docIDBySource := make(map[int64]int64, len(info.Result.Documents))
|
||||
for _, dj := range info.Result.Documents {
|
||||
sourceID := dj.ID
|
||||
doc, err := s.importDocument(ctx, dj, stickersDir, index, stats)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if doc.ID != 0 {
|
||||
docIDBySource[sourceID] = doc.ID
|
||||
docIDs = append(docIDs, doc.ID)
|
||||
docs = append(docs, doc)
|
||||
}
|
||||
}
|
||||
|
||||
kind := stickerSetKind(sj, systemKey)
|
||||
set := domain.StickerSet{
|
||||
ID: sj.ID,
|
||||
AccessHash: sj.AccessHash,
|
||||
ShortName: sj.ShortName,
|
||||
Title: sj.Title,
|
||||
Count: sj.Count,
|
||||
Hash: sj.Hash,
|
||||
Kind: kind,
|
||||
Official: sj.Official,
|
||||
Animated: true, // 导出资源均为 .tgs 动画贴纸
|
||||
Emojis: sj.Emojis,
|
||||
Masks: sj.Masks,
|
||||
Archived: sj.Archived,
|
||||
Installed: seedStickerSetInstalled(kind),
|
||||
ThumbDocumentID: seedDocumentStorageID(derefInt64(sj.ThumbDocumentID)),
|
||||
Thumbs: seedStickerSetPhotoSizes(sj.Thumbs),
|
||||
ThumbDCID: s.dc,
|
||||
ThumbVersion: sj.ThumbVersion,
|
||||
DocumentIDs: docIDs,
|
||||
Packs: seedStickerPacks(sj.Packs, info.Result.Packs, docIDBySource),
|
||||
SortOrder: order,
|
||||
SystemKey: systemKey,
|
||||
}
|
||||
if set.Count == 0 {
|
||||
set.Count = len(docIDs)
|
||||
}
|
||||
if err := s.media.PutStickerSet(ctx, set); err != nil {
|
||||
return err
|
||||
}
|
||||
s.stickerSetCache.put(set, docs)
|
||||
stats.StickerSets++
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---- 单个 document 导入 ----
|
||||
|
||||
func (s *Service) importDocument(ctx context.Context, dj seedDocumentJSON, binDir string, index seedDirIndex, stats *SeedStats) (domain.Document, error) {
|
||||
if dj.ID == 0 {
|
||||
return domain.Document{}, nil
|
||||
}
|
||||
storageID := seedDocumentStorageID(dj.ID)
|
||||
ref, _ := hex.DecodeString(dj.FileReference)
|
||||
doc := domain.Document{
|
||||
ID: storageID,
|
||||
AccessHash: dj.AccessHash,
|
||||
FileReference: ref,
|
||||
Date: parseSeedDate(dj.Date),
|
||||
MimeType: dj.MimeType,
|
||||
Size: dj.Size,
|
||||
DCID: s.dc,
|
||||
Attributes: seedDocumentAttributes(dj.Attributes),
|
||||
}
|
||||
|
||||
// 主体 blob：doc:<server-owned-id>
|
||||
if mainPath, ok := index.main[dj.ID]; ok {
|
||||
data, err := os.ReadFile(mainPath)
|
||||
if err != nil {
|
||||
return domain.Document{}, err
|
||||
}
|
||||
objectKey, err := s.blobs.Put(ctx, data)
|
||||
if err != nil {
|
||||
return domain.Document{}, err
|
||||
}
|
||||
if err := s.media.PutFileBlob(ctx, domain.FileBlob{
|
||||
LocationKey: fmt.Sprintf("doc:%d", storageID),
|
||||
Backend: domain.MediaBackend(s.blobs.Name()),
|
||||
ObjectKey: objectKey,
|
||||
Size: int64(len(data)),
|
||||
MimeType: dj.MimeType,
|
||||
}); err != nil {
|
||||
return domain.Document{}, err
|
||||
}
|
||||
s.prewarmSmallBlob(objectKey, data)
|
||||
stats.Blobs++
|
||||
}
|
||||
|
||||
// 缩略图：PhotoPathSize 内联；小的 PhotoSize 静态图同时写 blob 并作为
|
||||
// PhotoCachedSize 返回，让 TDesktop 处理 document 元数据时即可填本地 image cache。
|
||||
thumbs := make([]domain.PhotoSize, 0, len(dj.Thumbs))
|
||||
for _, tj := range dj.Thumbs {
|
||||
ps, downloadable := seedPhotoSize(tj)
|
||||
if ps.Kind == "" {
|
||||
continue
|
||||
}
|
||||
if downloadable {
|
||||
thumbPath, ok := index.thumb[dj.ID][ps.Type]
|
||||
if !ok {
|
||||
continue // 无可服务的缩略图文件，丢弃该尺寸（保留 PhotoPathSize 占位）
|
||||
}
|
||||
data, err := os.ReadFile(thumbPath)
|
||||
if err != nil {
|
||||
return domain.Document{}, err
|
||||
}
|
||||
objectKey, err := s.blobs.Put(ctx, data)
|
||||
if err != nil {
|
||||
return domain.Document{}, err
|
||||
}
|
||||
ps.Size = len(data)
|
||||
if err := s.media.PutFileBlob(ctx, domain.FileBlob{
|
||||
LocationKey: fmt.Sprintf("doc:%d:%s", storageID, ps.Type),
|
||||
Backend: domain.MediaBackend(s.blobs.Name()),
|
||||
ObjectKey: objectKey,
|
||||
Size: int64(len(data)),
|
||||
MimeType: seedThumbMimeType(data),
|
||||
}); err != nil {
|
||||
return domain.Document{}, err
|
||||
}
|
||||
ps = seedInlineCachedDocumentThumb(ps, data)
|
||||
s.prewarmSmallBlob(objectKey, data)
|
||||
stats.Blobs++
|
||||
}
|
||||
thumbs = append(thumbs, ps)
|
||||
}
|
||||
doc.Thumbs = seedPreferRasterDocumentThumbs(thumbs)
|
||||
|
||||
if err := s.media.PutDocument(ctx, doc); err != nil {
|
||||
return domain.Document{}, err
|
||||
}
|
||||
stats.Documents++
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
func (s *Service) prewarmSmallBlob(objectKey string, data []byte) {
|
||||
if len(data) > 0 && len(data) <= blobBytesCacheMaxEntryBytes {
|
||||
s.byteCache.put(objectKey, data)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 目录扫描：docID → 主体文件 / 可下载缩略图 ----
|
||||
|
||||
type seedDirIndex struct {
|
||||
main map[int64]string // docID -> 主体文件路径
|
||||
thumb map[int64]map[string]string // docID -> thumbType -> 缩略图文件路径
|
||||
}
|
||||
|
||||
var seedTrailingDigits = regexp.MustCompile(`(\d{6,})`)
|
||||
var seedThumbMarker = regexp.MustCompile(`_thumb\d+_`)
|
||||
|
||||
const seedInlineCachedDocumentThumbMaxBytes = 32 * 1024
|
||||
|
||||
// Exported Telegram resources keep their original id in filenames/JSON, but
|
||||
// telesrv owns the document catalog it serves. Imported high source ids are
|
||||
// normalized once at seed time so RPC/storage use one server-owned id.
|
||||
const seedExternalDocumentIDOffset int64 = 4_000_000_000_000_000_000
|
||||
|
||||
var seedThumbType = regexp.MustCompile(`PhotoSize_type([a-z])`)
|
||||
|
||||
func seedDocumentStorageID(sourceID int64) int64 {
|
||||
if sourceID <= 0 {
|
||||
return 0
|
||||
}
|
||||
if sourceID > seedExternalDocumentIDOffset {
|
||||
return sourceID - seedExternalDocumentIDOffset
|
||||
}
|
||||
return sourceID
|
||||
}
|
||||
|
||||
func scanSeedDir(dir string) (seedDirIndex, error) {
|
||||
idx := seedDirIndex{main: map[int64]string{}, thumb: map[int64]map[string]string{}}
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return idx, nil // 目录不存在 → 空 index
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := e.Name()
|
||||
ext := strings.ToLower(filepath.Ext(name))
|
||||
full := filepath.Join(dir, name)
|
||||
if marker := seedThumbMarker.FindStringIndex(name); marker != nil {
|
||||
// 只收可下载的 PhotoSize 缩略图（jpg）；PhotoPathSize(svg) 内联在 JSON。
|
||||
if ext != ".jpg" && ext != ".jpeg" {
|
||||
continue
|
||||
}
|
||||
m := seedThumbType.FindStringSubmatch(name)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
docID := docIDFromName(name[:marker[0]])
|
||||
if docID == 0 {
|
||||
continue
|
||||
}
|
||||
if idx.thumb[docID] == nil {
|
||||
idx.thumb[docID] = map[string]string{}
|
||||
}
|
||||
idx.thumb[docID][m[1]] = full
|
||||
continue
|
||||
}
|
||||
if ext == ".svg" || ext == ".json" {
|
||||
continue
|
||||
}
|
||||
docID := docIDFromName(strings.TrimSuffix(name, filepath.Ext(name)))
|
||||
if docID == 0 {
|
||||
continue
|
||||
}
|
||||
idx.main[docID] = full
|
||||
}
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
// docIDFromName 取 base name 中末尾最长的数字串作为 document id。
|
||||
func docIDFromName(base string) int64 {
|
||||
matches := seedTrailingDigits.FindAllString(base, -1)
|
||||
if len(matches) == 0 {
|
||||
return 0
|
||||
}
|
||||
last := matches[len(matches)-1]
|
||||
id, err := strconv.ParseInt(last, 10, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func systemKeyForDefaultSet(dirName string) string {
|
||||
switch dirName {
|
||||
case "DefaultSet_AnimatedEmoji":
|
||||
return "animated_emoji"
|
||||
case "DefaultSet_AnimatedEmojiAnimations":
|
||||
return "animated_emoji_animations"
|
||||
case "DefaultSet_EmojiGenericAnimations":
|
||||
return "emoji_generic_animations"
|
||||
case "DefaultSet_Dice_Normal":
|
||||
return "dice:\U0001f3b2"
|
||||
case "DefaultSet_Dice_Dart":
|
||||
return "dice:\U0001f3af"
|
||||
case "DefaultSet_Dice_Basketball":
|
||||
return "dice:\U0001f3c0"
|
||||
case "DefaultSet_Dice_Football":
|
||||
return "dice:⚽"
|
||||
case "DefaultSet_Dice_Bowling":
|
||||
return "dice:\U0001f3b3"
|
||||
case "DefaultSet_Dice_Casino":
|
||||
return "dice:\U0001f3b0"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func stickerSetKind(sj seedStickerSetJSON, systemKey string) domain.StickerSetKind {
|
||||
switch {
|
||||
case systemKey != "":
|
||||
return domain.StickerSetKindSystem
|
||||
case sj.Emojis:
|
||||
return domain.StickerSetKindEmoji
|
||||
case sj.Masks:
|
||||
return domain.StickerSetKindMasks
|
||||
default:
|
||||
return domain.StickerSetKindStickers
|
||||
}
|
||||
}
|
||||
|
||||
func seedStickerSetInstalled(kind domain.StickerSetKind) bool {
|
||||
return kind != domain.StickerSetKindSystem
|
||||
}
|
||||
|
||||
// ---- JSON → domain 转换 ----
|
||||
|
||||
func seedDocumentAttributes(attrs []seedAttrJSON) []domain.DocumentAttribute {
|
||||
out := make([]domain.DocumentAttribute, 0, len(attrs))
|
||||
for _, a := range attrs {
|
||||
switch a.Type {
|
||||
case "DocumentAttributeImageSize":
|
||||
out = append(out, domain.DocumentAttribute{Kind: domain.DocAttrImageSize, W: a.W, H: a.H})
|
||||
case "DocumentAttributeAnimated":
|
||||
out = append(out, domain.DocumentAttribute{Kind: domain.DocAttrAnimated})
|
||||
case "DocumentAttributeSticker":
|
||||
attr := domain.DocumentAttribute{Kind: domain.DocAttrSticker, Alt: a.Alt, Mask: a.Mask}
|
||||
if a.Stickerset != nil {
|
||||
attr.StickerSetID = a.Stickerset.ID
|
||||
attr.StickerSetAccessHash = a.Stickerset.AccessHash
|
||||
}
|
||||
out = append(out, attr)
|
||||
case "DocumentAttributeCustomEmoji":
|
||||
attr := domain.DocumentAttribute{Kind: domain.DocAttrCustomEmoji, Alt: a.Alt, Free: a.Free, TextColor: a.TextColor}
|
||||
if a.Stickerset != nil {
|
||||
attr.StickerSetID = a.Stickerset.ID
|
||||
attr.StickerSetAccessHash = a.Stickerset.AccessHash
|
||||
}
|
||||
out = append(out, attr)
|
||||
case "DocumentAttributeVideo":
|
||||
out = append(out, domain.DocumentAttribute{Kind: domain.DocAttrVideo, W: a.W, H: a.H, Duration: a.Duration, RoundMessage: a.RoundMessage, SupportsStreaming: a.SupportsStreaming})
|
||||
case "DocumentAttributeAudio":
|
||||
out = append(out, domain.DocumentAttribute{Kind: domain.DocAttrAudio, AudioDuration: int(a.Duration), Voice: a.Voice, Title: a.Title, Performer: a.Performer})
|
||||
case "DocumentAttributeFilename":
|
||||
out = append(out, domain.DocumentAttribute{Kind: domain.DocAttrFilename, FileName: a.FileName})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func seedPhotoSizes(thumbs []seedThumbJSON) []domain.PhotoSize {
|
||||
out := make([]domain.PhotoSize, 0, len(thumbs))
|
||||
for _, t := range thumbs {
|
||||
ps, _ := seedPhotoSize(t)
|
||||
if ps.Kind != "" {
|
||||
out = append(out, ps)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func seedStickerSetPhotoSizes(thumbs []seedThumbJSON) []domain.PhotoSize {
|
||||
out := make([]domain.PhotoSize, 0, len(thumbs))
|
||||
for _, t := range thumbs {
|
||||
ps, downloadable := seedPhotoSize(t)
|
||||
if ps.Kind == "" || downloadable {
|
||||
continue
|
||||
}
|
||||
out = append(out, ps)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func seedPhotoSize(t seedThumbJSON) (domain.PhotoSize, bool) {
|
||||
switch t.Type {
|
||||
case "PhotoSize":
|
||||
return domain.PhotoSize{Kind: domain.PhotoSizeKindDefault, Type: t.SizeType, W: t.W, H: t.H, Size: t.Size}, true
|
||||
case "PhotoStrippedSize":
|
||||
b, _ := hex.DecodeString(t.Bytes)
|
||||
return domain.PhotoSize{Kind: domain.PhotoSizeKindStripped, Type: t.SizeType, Bytes: b}, false
|
||||
case "PhotoCachedSize":
|
||||
b, _ := hex.DecodeString(t.Bytes)
|
||||
return domain.PhotoSize{Kind: domain.PhotoSizeKindCached, Type: t.SizeType, W: t.W, H: t.H, Bytes: b}, false
|
||||
case "PhotoPathSize":
|
||||
b, _ := hex.DecodeString(t.Bytes)
|
||||
return domain.PhotoSize{Kind: domain.PhotoSizeKindPath, Type: t.SizeType, Bytes: b}, false
|
||||
case "PhotoSizeProgressive":
|
||||
return domain.PhotoSize{Kind: domain.PhotoSizeKindProgressive, Type: t.SizeType, W: t.W, H: t.H, Sizes: t.Sizes}, true
|
||||
default:
|
||||
return domain.PhotoSize{}, false
|
||||
}
|
||||
}
|
||||
|
||||
func seedInlineCachedDocumentThumb(ps domain.PhotoSize, data []byte) domain.PhotoSize {
|
||||
if ps.Kind != domain.PhotoSizeKindDefault || len(data) == 0 || len(data) > seedInlineCachedDocumentThumbMaxBytes {
|
||||
return ps
|
||||
}
|
||||
ps.Kind = domain.PhotoSizeKindCached
|
||||
ps.Size = 0
|
||||
ps.Bytes = append([]byte(nil), data...)
|
||||
return ps
|
||||
}
|
||||
|
||||
func seedPreferRasterDocumentThumbs(sizes []domain.PhotoSize) []domain.PhotoSize {
|
||||
if !documentThumbsHaveRaster(sizes) {
|
||||
return sizes
|
||||
}
|
||||
out := sizes[:0]
|
||||
for _, size := range sizes {
|
||||
if size.Kind == domain.PhotoSizeKindPath {
|
||||
continue
|
||||
}
|
||||
out = append(out, size)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func seedThumbMimeType(data []byte) string {
|
||||
switch {
|
||||
case len(data) >= 12 && data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' &&
|
||||
data[8] == 'W' && data[9] == 'E' && data[10] == 'B' && data[11] == 'P':
|
||||
return "image/webp"
|
||||
case len(data) >= 3 && data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF:
|
||||
return "image/jpeg"
|
||||
case len(data) >= 8 && data[0] == 0x89 && data[1] == 'P' && data[2] == 'N' && data[3] == 'G':
|
||||
return "image/png"
|
||||
case len(data) >= 6 && data[0] == 'G' && data[1] == 'I' && data[2] == 'F':
|
||||
return "image/gif"
|
||||
default:
|
||||
return "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) stickerSetDocumentThumbsNeedInlineCache(ctx context.Context) (bool, error) {
|
||||
var ids []int64
|
||||
for _, kind := range []domain.StickerSetKind{
|
||||
domain.StickerSetKindStickers,
|
||||
domain.StickerSetKindEmoji,
|
||||
domain.StickerSetKindMasks,
|
||||
domain.StickerSetKindSystem,
|
||||
} {
|
||||
sets, err := s.media.ListStickerSets(ctx, kind)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, set := range sets {
|
||||
ids = append(ids, set.DocumentIDs...)
|
||||
}
|
||||
}
|
||||
return s.documentsNeedInlineCachedThumbs(ctx, ids)
|
||||
}
|
||||
|
||||
func (s *Service) documentsNeedInlineCachedThumbs(ctx context.Context, ids []int64) (bool, error) {
|
||||
if len(ids) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
seen := make(map[int64]struct{}, len(ids))
|
||||
unique := make([]int64, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
if id == 0 {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
unique = append(unique, id)
|
||||
}
|
||||
docs, err := s.media.GetDocuments(ctx, unique)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, doc := range docs {
|
||||
if documentThumbsHaveRaster(doc.Thumbs) && documentThumbsHavePath(doc.Thumbs) {
|
||||
return true, nil
|
||||
}
|
||||
for _, thumb := range doc.Thumbs {
|
||||
if thumb.Kind == domain.PhotoSizeKindDefault && thumb.Size > 0 && thumb.Size <= seedInlineCachedDocumentThumbMaxBytes {
|
||||
return true, nil
|
||||
}
|
||||
if thumb.Kind == domain.PhotoSizeKindCached && len(thumb.Bytes) > 0 {
|
||||
blob, ok, err := s.media.GetFileBlob(ctx, fmt.Sprintf("doc:%d:%s", doc.ID, thumb.Type))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if ok {
|
||||
want := seedThumbMimeType(thumb.Bytes)
|
||||
if want != "application/octet-stream" && blob.MimeType != want {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func documentThumbsHaveRaster(sizes []domain.PhotoSize) bool {
|
||||
for _, size := range sizes {
|
||||
switch size.Kind {
|
||||
case domain.PhotoSizeKindCached:
|
||||
if len(size.Bytes) > 0 {
|
||||
return true
|
||||
}
|
||||
case domain.PhotoSizeKindDefault:
|
||||
if size.Type != "" && size.Size > 0 {
|
||||
return true
|
||||
}
|
||||
case domain.PhotoSizeKindProgressive:
|
||||
if size.Type != "" && len(size.Sizes) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func documentThumbsHavePath(sizes []domain.PhotoSize) bool {
|
||||
for _, size := range sizes {
|
||||
if size.Kind == domain.PhotoSizeKindPath && len(size.Bytes) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func seedStickerPacks(setPacks, resultPacks []seedStickerPackJSON, docIDBySource map[int64]int64) []domain.StickerPack {
|
||||
packs := setPacks
|
||||
if len(packs) == 0 {
|
||||
packs = resultPacks
|
||||
}
|
||||
out := make([]domain.StickerPack, 0, len(packs))
|
||||
for _, p := range packs {
|
||||
documents := make([]int64, 0, len(p.Documents))
|
||||
for _, sourceID := range p.Documents {
|
||||
if id, ok := docIDBySource[sourceID]; ok {
|
||||
documents = append(documents, id)
|
||||
continue
|
||||
}
|
||||
if id := seedDocumentStorageID(sourceID); id != 0 {
|
||||
documents = append(documents, id)
|
||||
}
|
||||
}
|
||||
out = append(out, domain.StickerPack{Emoticon: p.Emoticon, DocumentIDs: documents})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseSeedDate(s string) int {
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
if t, err := time.Parse(time.RFC3339, s); err == nil {
|
||||
return int(t.Unix())
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func derefInt64(v *int64) int64 {
|
||||
if v == nil {
|
||||
return 0
|
||||
}
|
||||
return *v
|
||||
}
|
||||
|
||||
// ---- seed JSON 结构 ----
|
||||
|
||||
type seedInputStickerSetJSON struct {
|
||||
ID int64 `json:"id"`
|
||||
AccessHash int64 `json:"access_hash"`
|
||||
}
|
||||
|
||||
type seedAttrJSON struct {
|
||||
Type string `json:"_"`
|
||||
W int `json:"w"`
|
||||
H int `json:"h"`
|
||||
Alt string `json:"alt"`
|
||||
Mask bool `json:"mask"`
|
||||
Duration float64 `json:"duration"`
|
||||
RoundMessage bool `json:"round_message"`
|
||||
SupportsStreaming bool `json:"supports_streaming"`
|
||||
Voice bool `json:"voice"`
|
||||
Title string `json:"title"`
|
||||
Performer string `json:"performer"`
|
||||
FileName string `json:"file_name"`
|
||||
Free bool `json:"free"`
|
||||
TextColor bool `json:"text_color"`
|
||||
Stickerset *seedInputStickerSetJSON `json:"stickerset"`
|
||||
}
|
||||
|
||||
type seedThumbJSON struct {
|
||||
Type string `json:"_"`
|
||||
SizeType string `json:"type"`
|
||||
W int `json:"w"`
|
||||
H int `json:"h"`
|
||||
Size int `json:"size"`
|
||||
Bytes string `json:"bytes"`
|
||||
Sizes []int `json:"sizes"`
|
||||
}
|
||||
|
||||
type seedDocumentJSON struct {
|
||||
ID int64 `json:"id"`
|
||||
AccessHash int64 `json:"access_hash"`
|
||||
FileReference string `json:"file_reference"`
|
||||
Date string `json:"date"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Size int64 `json:"size"`
|
||||
DCID int `json:"dc_id"`
|
||||
Attributes []seedAttrJSON `json:"attributes"`
|
||||
Thumbs []seedThumbJSON `json:"thumbs"`
|
||||
}
|
||||
|
||||
type seedStickerPackJSON struct {
|
||||
Emoticon string `json:"emoticon"`
|
||||
Documents []int64 `json:"documents"`
|
||||
}
|
||||
|
||||
type seedStickerSetJSON struct {
|
||||
ID int64 `json:"id"`
|
||||
AccessHash int64 `json:"access_hash"`
|
||||
Title string `json:"title"`
|
||||
ShortName string `json:"short_name"`
|
||||
Count int `json:"count"`
|
||||
Hash int `json:"hash"`
|
||||
Archived bool `json:"archived"`
|
||||
Official bool `json:"official"`
|
||||
Masks bool `json:"masks"`
|
||||
Emojis bool `json:"emojis"`
|
||||
Thumbs []seedThumbJSON `json:"thumbs"`
|
||||
ThumbDCID int `json:"thumb_dc_id"`
|
||||
ThumbVersion int `json:"thumb_version"`
|
||||
ThumbDocumentID *int64 `json:"thumb_document_id"`
|
||||
Packs []seedStickerPackJSON `json:"packs"`
|
||||
}
|
||||
|
||||
type seedStickerSetResultJSON struct {
|
||||
Set seedStickerSetJSON `json:"set"`
|
||||
Packs []seedStickerPackJSON `json:"packs"`
|
||||
Documents []seedDocumentJSON `json:"documents"`
|
||||
}
|
||||
|
||||
type seedReactionJSON struct {
|
||||
Reaction string `json:"reaction"`
|
||||
Title string `json:"title"`
|
||||
Inactive bool `json:"inactive"`
|
||||
Premium bool `json:"premium"`
|
||||
StaticIcon *seedDocumentJSON `json:"static_icon"`
|
||||
AppearAnimation *seedDocumentJSON `json:"appear_animation"`
|
||||
SelectAnimation *seedDocumentJSON `json:"select_animation"`
|
||||
ActivateAnimation *seedDocumentJSON `json:"activate_animation"`
|
||||
EffectAnimation *seedDocumentJSON `json:"effect_animation"`
|
||||
AroundAnimation *seedDocumentJSON `json:"around_animation"`
|
||||
CenterIcon *seedDocumentJSON `json:"center_icon"`
|
||||
}
|
||||
526
internal/app/files/seed_test.go
Normal file
526
internal/app/files/seed_test.go
Normal file
|
|
@ -0,0 +1,526 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// fakeMediaStore 是 store.MediaStore 的内存替身，用于在无 PG 时验证 seed 导入器。
|
||||
type fakeMediaStore struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string]domain.FileBlob
|
||||
docs map[int64]domain.Document
|
||||
photos map[int64]domain.Photo
|
||||
sets map[int64]domain.StickerSet
|
||||
reactions []domain.AvailableReaction
|
||||
parts map[string][]domain.UploadPart
|
||||
}
|
||||
|
||||
func newFakeMediaStore() *fakeMediaStore {
|
||||
return &fakeMediaStore{
|
||||
blobs: map[string]domain.FileBlob{},
|
||||
docs: map[int64]domain.Document{},
|
||||
photos: map[int64]domain.Photo{},
|
||||
sets: map[int64]domain.StickerSet{},
|
||||
parts: map[string][]domain.UploadPart{},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeMediaStore) SaveFilePart(_ context.Context, _ domain.UploadPart) error { return nil }
|
||||
func (f *fakeMediaStore) LoadFileParts(_ context.Context, _, _ int64) ([]domain.UploadPart, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakeMediaStore) DeleteFileParts(_ context.Context, _, _ int64) error { return nil }
|
||||
|
||||
func (f *fakeMediaStore) PutFileBlob(_ context.Context, blob domain.FileBlob) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.blobs[blob.LocationKey] = blob
|
||||
return nil
|
||||
}
|
||||
func (f *fakeMediaStore) GetFileBlob(_ context.Context, key string) (domain.FileBlob, bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
b, ok := f.blobs[key]
|
||||
return b, ok, nil
|
||||
}
|
||||
|
||||
func (f *fakeMediaStore) PutDocument(_ context.Context, doc domain.Document) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.docs[doc.ID] = doc
|
||||
return nil
|
||||
}
|
||||
func (f *fakeMediaStore) GetDocument(_ context.Context, id int64) (domain.Document, bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
d, ok := f.docs[id]
|
||||
return d, ok, nil
|
||||
}
|
||||
func (f *fakeMediaStore) GetDocuments(_ context.Context, ids []int64) ([]domain.Document, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := make([]domain.Document, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
if d, ok := f.docs[id]; ok {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (f *fakeMediaStore) PutPhoto(_ context.Context, p domain.Photo) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.photos[p.ID] = p
|
||||
return nil
|
||||
}
|
||||
func (f *fakeMediaStore) GetPhoto(_ context.Context, id int64) (domain.Photo, bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
p, ok := f.photos[id]
|
||||
return p, ok, nil
|
||||
}
|
||||
|
||||
func (f *fakeMediaStore) PutStickerSet(_ context.Context, set domain.StickerSet) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.sets[set.ID] = set
|
||||
return nil
|
||||
}
|
||||
func (f *fakeMediaStore) GetStickerSetByID(_ context.Context, id int64) (domain.StickerSet, bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
s, ok := f.sets[id]
|
||||
return s, ok, nil
|
||||
}
|
||||
func (f *fakeMediaStore) GetStickerSetByShortName(_ context.Context, name string) (domain.StickerSet, bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, s := range f.sets {
|
||||
if s.ShortName == name {
|
||||
return s, true, nil
|
||||
}
|
||||
}
|
||||
return domain.StickerSet{}, false, nil
|
||||
}
|
||||
func (f *fakeMediaStore) GetStickerSetBySystemKey(_ context.Context, key string) (domain.StickerSet, bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, s := range f.sets {
|
||||
if s.SystemKey == key {
|
||||
return s, true, nil
|
||||
}
|
||||
}
|
||||
return domain.StickerSet{}, false, nil
|
||||
}
|
||||
func (f *fakeMediaStore) ListStickerSets(_ context.Context, kind domain.StickerSetKind) ([]domain.StickerSet, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []domain.StickerSet
|
||||
for _, s := range f.sets {
|
||||
if s.Kind == kind {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (f *fakeMediaStore) CountStickerSets(_ context.Context) (int, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.sets), nil
|
||||
}
|
||||
func (f *fakeMediaStore) PutAvailableReaction(_ context.Context, r domain.AvailableReaction) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for i, existing := range f.reactions {
|
||||
if existing.Reaction == r.Reaction {
|
||||
f.reactions[i] = r
|
||||
return nil
|
||||
}
|
||||
}
|
||||
f.reactions = append(f.reactions, r)
|
||||
return nil
|
||||
}
|
||||
func (f *fakeMediaStore) ListAvailableReactions(_ context.Context) ([]domain.AvailableReaction, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]domain.AvailableReaction(nil), f.reactions...), nil
|
||||
}
|
||||
func (f *fakeMediaStore) CountAvailableReactions(_ context.Context) (int, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.reactions), nil
|
||||
}
|
||||
func (f *fakeMediaStore) AddProfilePhoto(_ context.Context, _ domain.PeerType, _, _ int64, _ int) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakeMediaStore) CurrentProfilePhoto(_ context.Context, _ domain.PeerType, _ int64) (int64, bool, error) {
|
||||
return 0, false, nil
|
||||
}
|
||||
func (f *fakeMediaStore) CurrentProfilePhotos(_ context.Context, _ domain.PeerType, _ []int64) (map[int64]domain.ProfilePhotoRef, error) {
|
||||
return map[int64]domain.ProfilePhotoRef{}, nil
|
||||
}
|
||||
func (f *fakeMediaStore) ListProfilePhotos(_ context.Context, _ domain.PeerType, _ int64, _, _ int, _ int64) ([]int64, int, error) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
func (f *fakeMediaStore) DeleteProfilePhotos(_ context.Context, _ domain.PeerType, _ int64, _ []int64) ([]int64, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestSeedMediaRepairsPartialReactionBlobs(t *testing.T) {
|
||||
seedDir := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(seedDir, "telegram_reactions_export", "global_json"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reactionsDir := filepath.Join(seedDir, "telegram_reactions_export", "reactions")
|
||||
if err := os.MkdirAll(reactionsDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := `{"result":{"reactions":[{"reaction":"👍","title":"Like","static_icon":{"id":1111111,"access_hash":1,"file_reference":"","date":"2026-06-03T00:00:00Z","mime_type":"image/webp","size":4,"attributes":[],"thumbs":[]},"select_animation":{"id":2222222,"access_hash":2,"file_reference":"","date":"2026-06-03T00:00:00Z","mime_type":"application/x-tgsticker","size":4,"attributes":[],"thumbs":[]}}]}}`
|
||||
if err := os.WriteFile(filepath.Join(seedDir, "telegram_reactions_export", "global_json", "available_reactions_raw.json"), []byte(raw), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(reactionsDir, "reaction_thumbs_up_sign_static_icon_Like_1111111.webp"), []byte("webp"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(reactionsDir, "reaction_thumbs_up_sign_static_icon_Like_1111111_thumb1_PhotoSize_types_72x72.jpg"), []byte("jpeg"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(reactionsDir, "reaction_select_2222222.tgs"), []byte("tgs!"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
media := newFakeMediaStore()
|
||||
local, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("local fs: %v", err)
|
||||
}
|
||||
blobs := &countingBlobBackend{BlobBackend: local}
|
||||
svc := NewService(media, blobs, 2)
|
||||
if stats, err := svc.SeedMedia(context.Background(), seedDir, 0); err != nil {
|
||||
t.Fatalf("initial seed: %v", err)
|
||||
} else if stats.Reactions != 1 || stats.Blobs != 2 {
|
||||
t.Fatalf("initial stats = %+v, want one reaction and two blobs", stats)
|
||||
}
|
||||
chunk, ok, err := svc.GetFile(context.Background(), domain.FileDownloadRequest{LocationKey: "doc:2222222", Offset: 0, Limit: 4})
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("prewarmed getfile ok=%v err=%v", ok, err)
|
||||
}
|
||||
if string(chunk.Bytes) != "tgs!" {
|
||||
t.Fatalf("prewarmed chunk = %q, want tgs!", chunk.Bytes)
|
||||
}
|
||||
if blobs.getRangeCalls != 0 {
|
||||
t.Fatalf("seeded small blob should be served from byte cache, GetRange calls = %d", blobs.getRangeCalls)
|
||||
}
|
||||
|
||||
media.mu.Lock()
|
||||
delete(media.blobs, "doc:2222222")
|
||||
media.mu.Unlock()
|
||||
|
||||
stats, err := svc.SeedMedia(context.Background(), seedDir, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("repair seed: %v", err)
|
||||
}
|
||||
if stats.Reactions != 1 || stats.Blobs != 2 || stats.Skipped {
|
||||
t.Fatalf("repair stats = %+v, want repair import", stats)
|
||||
}
|
||||
if _, ok, _ := media.GetFileBlob(context.Background(), "doc:2222222"); !ok {
|
||||
t.Fatal("missing reaction blob was not repaired")
|
||||
}
|
||||
if reactions, _ := media.ListAvailableReactions(context.Background()); len(reactions) != 1 {
|
||||
t.Fatalf("reaction upsert duplicated rows: got %d", len(reactions))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedMediaFromRealExport(t *testing.T) {
|
||||
seedDir := os.Getenv("TELESRV_REAL_STICKER_SEED_DIR")
|
||||
if seedDir == "" {
|
||||
t.Skip("TELESRV_REAL_STICKER_SEED_DIR not set")
|
||||
}
|
||||
if _, err := os.Stat(seedDir); err != nil {
|
||||
t.Skipf("seed dir %s not present: %v", seedDir, err)
|
||||
}
|
||||
media := newFakeMediaStore()
|
||||
blobs, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("local fs: %v", err)
|
||||
}
|
||||
svc := NewService(media, blobs, 2)
|
||||
stats, err := svc.SeedMedia(context.Background(), seedDir, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("seed media: %v", err)
|
||||
}
|
||||
t.Logf("seed stats: reactions=%d sets=%d docs=%d blobs=%d", stats.Reactions, stats.StickerSets, stats.Documents, stats.Blobs)
|
||||
if stats.Reactions == 0 {
|
||||
t.Error("expected reactions imported")
|
||||
}
|
||||
if stats.StickerSets == 0 {
|
||||
t.Error("expected sticker sets imported")
|
||||
}
|
||||
if stats.Documents == 0 {
|
||||
t.Error("expected documents imported")
|
||||
}
|
||||
if stats.Blobs == 0 {
|
||||
t.Error("expected blobs imported")
|
||||
}
|
||||
|
||||
// reaction 引用的文档应能被解析回真实 document（带 sticker 属性 + 主体 blob）。
|
||||
reactions, _ := media.ListAvailableReactions(context.Background())
|
||||
if len(reactions) == 0 {
|
||||
t.Fatal("no reactions stored")
|
||||
}
|
||||
first := reactions[0]
|
||||
if first.Reaction == "" {
|
||||
t.Error("reaction emoticon empty")
|
||||
}
|
||||
if first.StaticIconID == 0 || first.SelectAnimationID == 0 {
|
||||
t.Error("reaction missing document ids")
|
||||
}
|
||||
if d, ok, _ := media.GetDocument(context.Background(), first.SelectAnimationID); !ok {
|
||||
t.Error("reaction select animation document missing")
|
||||
} else {
|
||||
if d.ID > seedExternalDocumentIDOffset {
|
||||
t.Errorf("reaction document kept external source id: %d", d.ID)
|
||||
}
|
||||
if d.DCID != 2 {
|
||||
t.Errorf("document dc_id not rewritten: %d", d.DCID)
|
||||
}
|
||||
if _, ok, _ := media.GetFileBlob(context.Background(), blobKeyDoc(d.ID)); !ok {
|
||||
t.Errorf("reaction document %d main blob missing", d.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// 一个常规贴纸集应有 documents 且能按 short_name 解析。
|
||||
for _, s := range media.sets {
|
||||
for _, thumb := range s.Thumbs {
|
||||
if thumb.Downloadable() {
|
||||
t.Fatalf("sticker set %s exposes downloadable cover thumb %q without a serviceable blob", s.ShortName, thumb.Type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var sample domain.StickerSet
|
||||
for _, s := range media.sets {
|
||||
if s.Kind == domain.StickerSetKindStickers && len(s.DocumentIDs) > 0 {
|
||||
sample = s
|
||||
break
|
||||
}
|
||||
}
|
||||
if sample.ID == 0 {
|
||||
t.Fatal("no regular sticker set with documents imported")
|
||||
}
|
||||
if got, ok, _ := media.GetStickerSetByShortName(context.Background(), sample.ShortName); !ok || got.ID != sample.ID {
|
||||
t.Error("sticker set not resolvable by short name")
|
||||
}
|
||||
if doc, ok, _ := media.GetDocument(context.Background(), sample.DocumentIDs[0]); !ok {
|
||||
t.Fatalf("sample sticker document %d missing", sample.DocumentIDs[0])
|
||||
} else {
|
||||
if doc.ID > seedExternalDocumentIDOffset {
|
||||
t.Fatalf("sample sticker kept external source id: %d", doc.ID)
|
||||
}
|
||||
thumb, ok := findCachedThumb(doc.Thumbs)
|
||||
if !ok {
|
||||
t.Fatalf("sample sticker document thumbs are not inline cached: %+v", doc.Thumbs)
|
||||
}
|
||||
blob, ok, err := media.GetFileBlob(context.Background(), blobKeyDoc(doc.ID)+":"+thumb.Type)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("sample sticker thumb blob ok=%v err=%v", ok, err)
|
||||
}
|
||||
if want := seedThumbMimeType(thumb.Bytes); blob.MimeType != want {
|
||||
t.Fatalf("sample sticker thumb mime = %q, want %q", blob.MimeType, want)
|
||||
}
|
||||
if hasPathThumb(doc.Thumbs) {
|
||||
t.Fatalf("sample sticker document still exposes path thumb together with raster: %+v", doc.Thumbs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedDocumentStorageIDNormalizesExternalIDs(t *testing.T) {
|
||||
const sourceID int64 = 5382305375846410902
|
||||
const want int64 = 1382305375846410902
|
||||
if got := seedDocumentStorageID(sourceID); got != want {
|
||||
t.Fatalf("seedDocumentStorageID(%d) = %d, want %d", sourceID, got, want)
|
||||
}
|
||||
if got := seedDocumentStorageID(2222222); got != 2222222 {
|
||||
t.Fatalf("small server id changed: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedStickerSetInstalledFlagExcludesSystemSets(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
kind domain.StickerSetKind
|
||||
want bool
|
||||
}{
|
||||
{name: "regular stickers", kind: domain.StickerSetKindStickers, want: true},
|
||||
{name: "custom emoji", kind: domain.StickerSetKindEmoji, want: true},
|
||||
{name: "masks", kind: domain.StickerSetKindMasks, want: true},
|
||||
{name: "system resources", kind: domain.StickerSetKindSystem, want: false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := seedStickerSetInstalled(tc.kind); got != tc.want {
|
||||
t.Fatalf("seedStickerSetInstalled(%q) = %v, want %v", tc.kind, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedInlineCachedDocumentThumb(t *testing.T) {
|
||||
input := domain.PhotoSize{Kind: domain.PhotoSizeKindDefault, Type: "m", W: 128, H: 128, Size: 6400}
|
||||
got := seedInlineCachedDocumentThumb(input, []byte("jpeg"))
|
||||
if got.Kind != domain.PhotoSizeKindCached {
|
||||
t.Fatalf("kind = %q, want cached", got.Kind)
|
||||
}
|
||||
if got.Size != 0 || string(got.Bytes) != "jpeg" {
|
||||
t.Fatalf("cached thumb = %+v, want inline bytes without downloadable size", got)
|
||||
}
|
||||
large := make([]byte, seedInlineCachedDocumentThumbMaxBytes+1)
|
||||
if got := seedInlineCachedDocumentThumb(input, large); got.Kind != domain.PhotoSizeKindDefault || got.Size != input.Size || len(got.Bytes) != 0 {
|
||||
t.Fatalf("large thumb = %+v, want unchanged downloadable thumb", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedThumbMimeType(t *testing.T) {
|
||||
webp := []byte{'R', 'I', 'F', 'F', 0, 0, 0, 0, 'W', 'E', 'B', 'P'}
|
||||
if got := seedThumbMimeType(webp); got != "image/webp" {
|
||||
t.Fatalf("webp mime = %q, want image/webp", got)
|
||||
}
|
||||
jpeg := []byte{0xFF, 0xD8, 0xFF}
|
||||
if got := seedThumbMimeType(jpeg); got != "image/jpeg" {
|
||||
t.Fatalf("jpeg mime = %q, want image/jpeg", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedPreferRasterDocumentThumbsDropsPathWhenRasterExists(t *testing.T) {
|
||||
sizes := []domain.PhotoSize{
|
||||
{Kind: domain.PhotoSizeKindPath, Type: "j", Bytes: []byte("path")},
|
||||
{Kind: domain.PhotoSizeKindCached, Type: "m", Bytes: []byte("webp")},
|
||||
}
|
||||
got := seedPreferRasterDocumentThumbs(sizes)
|
||||
if hasPathThumb(got) {
|
||||
t.Fatalf("path thumb should be dropped when raster exists: %+v", got)
|
||||
}
|
||||
if !hasCachedThumb(got) {
|
||||
t.Fatalf("cached thumb should be kept: %+v", got)
|
||||
}
|
||||
|
||||
onlyPath := []domain.PhotoSize{{Kind: domain.PhotoSizeKindPath, Type: "j", Bytes: []byte("path")}}
|
||||
if got := seedPreferRasterDocumentThumbs(onlyPath); !hasPathThumb(got) {
|
||||
t.Fatalf("path-only thumbs should be kept: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentsNeedInlineCachedThumbsDetectsStaleMime(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
media := newFakeMediaStore()
|
||||
webp := []byte{'R', 'I', 'F', 'F', 0, 0, 0, 0, 'W', 'E', 'B', 'P'}
|
||||
doc := domain.Document{
|
||||
ID: 100,
|
||||
Thumbs: []domain.PhotoSize{
|
||||
{Kind: domain.PhotoSizeKindCached, Type: "m", Bytes: webp},
|
||||
},
|
||||
}
|
||||
if err := media.PutDocument(ctx, doc); err != nil {
|
||||
t.Fatalf("put doc: %v", err)
|
||||
}
|
||||
if err := media.PutFileBlob(ctx, domain.FileBlob{LocationKey: "doc:100:m", MimeType: "image/jpeg"}); err != nil {
|
||||
t.Fatalf("put blob: %v", err)
|
||||
}
|
||||
svc := NewService(media, nil, 2)
|
||||
stale, err := svc.documentsNeedInlineCachedThumbs(ctx, []int64{doc.ID})
|
||||
if err != nil {
|
||||
t.Fatalf("documentsNeedInlineCachedThumbs: %v", err)
|
||||
}
|
||||
if !stale {
|
||||
t.Fatal("expected stale mime to require repair")
|
||||
}
|
||||
|
||||
if err := media.PutFileBlob(ctx, domain.FileBlob{LocationKey: "doc:100:m", MimeType: "image/webp"}); err != nil {
|
||||
t.Fatalf("put repaired blob: %v", err)
|
||||
}
|
||||
stale, err = svc.documentsNeedInlineCachedThumbs(ctx, []int64{doc.ID})
|
||||
if err != nil {
|
||||
t.Fatalf("documentsNeedInlineCachedThumbs after repair: %v", err)
|
||||
}
|
||||
if stale {
|
||||
t.Fatal("repaired mime should not require repair")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentsNeedInlineCachedThumbsDetectsPathWithRaster(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
media := newFakeMediaStore()
|
||||
doc := domain.Document{
|
||||
ID: 100,
|
||||
Thumbs: []domain.PhotoSize{
|
||||
{Kind: domain.PhotoSizeKindPath, Type: "j", Bytes: []byte("path")},
|
||||
{Kind: domain.PhotoSizeKindCached, Type: "m", Bytes: []byte("webp")},
|
||||
},
|
||||
}
|
||||
if err := media.PutDocument(ctx, doc); err != nil {
|
||||
t.Fatalf("put doc: %v", err)
|
||||
}
|
||||
svc := NewService(media, nil, 2)
|
||||
stale, err := svc.documentsNeedInlineCachedThumbs(ctx, []int64{doc.ID})
|
||||
if err != nil {
|
||||
t.Fatalf("documentsNeedInlineCachedThumbs: %v", err)
|
||||
}
|
||||
if !stale {
|
||||
t.Fatal("path thumb with raster should require repair")
|
||||
}
|
||||
}
|
||||
|
||||
func hasCachedThumb(sizes []domain.PhotoSize) bool {
|
||||
_, ok := findCachedThumb(sizes)
|
||||
return ok
|
||||
}
|
||||
|
||||
func findCachedThumb(sizes []domain.PhotoSize) (domain.PhotoSize, bool) {
|
||||
for _, size := range sizes {
|
||||
if size.Kind == domain.PhotoSizeKindCached && len(size.Bytes) > 0 {
|
||||
return size, true
|
||||
}
|
||||
}
|
||||
return domain.PhotoSize{}, false
|
||||
}
|
||||
|
||||
func hasPathThumb(sizes []domain.PhotoSize) bool {
|
||||
for _, size := range sizes {
|
||||
if size.Kind == domain.PhotoSizeKindPath && len(size.Bytes) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func blobKeyDoc(id int64) string {
|
||||
return "doc:" + itoa(id)
|
||||
}
|
||||
|
||||
func itoa(v int64) string {
|
||||
if v == 0 {
|
||||
return "0"
|
||||
}
|
||||
neg := v < 0
|
||||
if neg {
|
||||
v = -v
|
||||
}
|
||||
var buf [20]byte
|
||||
i := len(buf)
|
||||
for v > 0 {
|
||||
i--
|
||||
buf[i] = byte('0' + v%10)
|
||||
v /= 10
|
||||
}
|
||||
if neg {
|
||||
i--
|
||||
buf[i] = '-'
|
||||
}
|
||||
return string(buf[i:])
|
||||
}
|
||||
256
internal/app/files/service.go
Normal file
256
internal/app/files/service.go
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
"telesrv/internal/store"
|
||||
)
|
||||
|
||||
// 上传分片上限：与 Telegram 客户端约定一致（单片 ≤512KB；分片总数有上限防止 OOM）。
|
||||
const (
|
||||
MaxUploadPartBytes = 524288 // 512KB
|
||||
MaxUploadParts = 8000 // 512KB * 8000 ≈ 4GB 理论上限，足够主路径媒体
|
||||
)
|
||||
|
||||
// blobMetaCacheCapacity 是 location_key→FileBlob 元数据 LRU 容量（每项约百字节，约 13MB）。
|
||||
const blobMetaCacheCapacity = 1 << 16
|
||||
|
||||
// 小文件热缓存只覆盖 sticker/reaction/thumbnail 一类不可变小 blob；大媒体继续分段读。
|
||||
const (
|
||||
blobBytesCacheMaxEntryBytes = 256 << 10 // 256KB
|
||||
blobBytesCacheMaxBytes = 64 << 20 // 64MB
|
||||
)
|
||||
|
||||
// Service 实现 upload 分片累积、blob 落盘、getFile 下载，并把上传文件组装成 Photo / Document。
|
||||
type Service struct {
|
||||
media store.MediaStore
|
||||
blobs BlobBackend
|
||||
dc int
|
||||
blobCache *blobMetaCache
|
||||
byteCache *blobBytesCache
|
||||
stickerSetCache *stickerSetFullCache
|
||||
}
|
||||
|
||||
// NewService 创建 files 服务。dc 是本 server 的 DC id，写入新建 document/photo 的 dc_id。
|
||||
func NewService(media store.MediaStore, blobs BlobBackend, dc int) *Service {
|
||||
return &Service{
|
||||
media: media,
|
||||
blobs: blobs,
|
||||
dc: dc,
|
||||
blobCache: newBlobMetaCache(blobMetaCacheCapacity),
|
||||
byteCache: newBlobBytesCache(blobBytesCacheMaxBytes),
|
||||
stickerSetCache: newStickerSetFullCache(),
|
||||
}
|
||||
}
|
||||
|
||||
// SaveFilePart 累积一个 small file 分片。
|
||||
func (s *Service) SaveFilePart(ctx context.Context, ownerUserID, fileID int64, part int, bytes []byte) (bool, error) {
|
||||
if err := validatePart(part, len(bytes)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := s.media.SaveFilePart(ctx, domain.UploadPart{
|
||||
OwnerUserID: ownerUserID,
|
||||
FileID: fileID,
|
||||
Part: part,
|
||||
Bytes: bytes,
|
||||
}); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// SaveBigFilePart 累积一个 big file 分片（带已知总分片数）。
|
||||
func (s *Service) SaveBigFilePart(ctx context.Context, ownerUserID, fileID int64, part, totalParts int, bytes []byte) (bool, error) {
|
||||
if err := validatePart(part, len(bytes)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if totalParts <= 0 || totalParts > MaxUploadParts {
|
||||
return false, domain.ErrFilePartsInvalid
|
||||
}
|
||||
if err := s.media.SaveFilePart(ctx, domain.UploadPart{
|
||||
OwnerUserID: ownerUserID,
|
||||
FileID: fileID,
|
||||
Part: part,
|
||||
TotalParts: totalParts,
|
||||
Big: true,
|
||||
Bytes: bytes,
|
||||
}); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// GetFile 按 location_key 取一段 blob 内容。found=false 表示该 location 无对应 blob。
|
||||
// 元数据走进程内 LRU（消除每 chunk 一次 PG 查）；小 blob 全量字节进 LRU，供 sticker /
|
||||
// reaction / thumbnail 热路径直接内存切片；大 blob 仍按 offset/limit 段读。
|
||||
func (s *Service) GetFile(ctx context.Context, req domain.FileDownloadRequest) (domain.FileChunk, bool, error) {
|
||||
blob, ok := s.blobCache.get(req.LocationKey)
|
||||
if !ok {
|
||||
var (
|
||||
found bool
|
||||
err error
|
||||
)
|
||||
blob, found, err = s.media.GetFileBlob(ctx, req.LocationKey)
|
||||
if err != nil {
|
||||
return domain.FileChunk{}, false, err
|
||||
}
|
||||
if !found {
|
||||
return domain.FileChunk{}, false, nil
|
||||
}
|
||||
s.blobCache.put(req.LocationKey, blob)
|
||||
}
|
||||
if blob.Size > 0 && blob.Size <= blobBytesCacheMaxEntryBytes {
|
||||
if data, ok := s.byteCache.get(blob.ObjectKey); ok {
|
||||
return domain.FileChunk{
|
||||
Bytes: sliceBlobBytes(data, req.Offset, int64(req.Limit)),
|
||||
MimeType: blob.MimeType,
|
||||
Total: int64(len(data)),
|
||||
}, true, nil
|
||||
}
|
||||
data, total, err := s.blobs.GetRange(ctx, blob.ObjectKey, 0, blobBytesCacheMaxEntryBytes+1)
|
||||
if err != nil {
|
||||
return domain.FileChunk{}, false, fmt.Errorf("read blob %q: %w", blob.LocationKey, err)
|
||||
}
|
||||
if total <= blobBytesCacheMaxEntryBytes && int64(len(data)) == total {
|
||||
s.byteCache.put(blob.ObjectKey, data)
|
||||
return domain.FileChunk{
|
||||
Bytes: sliceBlobBytes(data, req.Offset, int64(req.Limit)),
|
||||
MimeType: blob.MimeType,
|
||||
Total: total,
|
||||
}, true, nil
|
||||
}
|
||||
}
|
||||
data, total, err := s.blobs.GetRange(ctx, blob.ObjectKey, req.Offset, int64(req.Limit))
|
||||
if err != nil {
|
||||
return domain.FileChunk{}, false, fmt.Errorf("read blob %q: %w", blob.LocationKey, err)
|
||||
}
|
||||
return domain.FileChunk{
|
||||
Bytes: data,
|
||||
MimeType: blob.MimeType,
|
||||
Total: total,
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func sliceBlobBytes(data []byte, offset, limit int64) []byte {
|
||||
total := int64(len(data))
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
if offset >= total {
|
||||
return []byte{}
|
||||
}
|
||||
end := total
|
||||
if limit > 0 && offset+limit < end {
|
||||
end = offset + limit
|
||||
}
|
||||
return append([]byte(nil), data[offset:end]...)
|
||||
}
|
||||
|
||||
// ---- 资源读取（reaction / sticker / document）----
|
||||
|
||||
// ListAvailableReactions 返回可用 reaction 目录（带真实文档 id）。
|
||||
func (s *Service) ListAvailableReactions(ctx context.Context) ([]domain.AvailableReaction, error) {
|
||||
return s.media.ListAvailableReactions(ctx)
|
||||
}
|
||||
|
||||
// GetDocuments 按 id 批量加载文档（自定义 emoji / 贴纸）。
|
||||
func (s *Service) GetDocuments(ctx context.Context, ids []int64) ([]domain.Document, error) {
|
||||
return s.media.GetDocuments(ctx, ids)
|
||||
}
|
||||
|
||||
// ListStickerSets 列出某类贴纸集（用于 getAllStickers 等）。
|
||||
func (s *Service) ListStickerSets(ctx context.Context, kind domain.StickerSetKind) ([]domain.StickerSet, error) {
|
||||
return s.media.ListStickerSets(ctx, kind)
|
||||
}
|
||||
|
||||
// ResolveStickerSet 按 ref 解析贴纸集，并按 DocumentIDs 顺序加载其文档。
|
||||
func (s *Service) ResolveStickerSet(ctx context.Context, ref domain.StickerSetRef) (domain.StickerSet, []domain.Document, bool, error) {
|
||||
if set, docs, ok := s.stickerSetCache.get(ref); ok {
|
||||
return set, docs, true, nil
|
||||
}
|
||||
var (
|
||||
set domain.StickerSet
|
||||
found bool
|
||||
err error
|
||||
)
|
||||
switch ref.Kind {
|
||||
case domain.StickerSetRefByID:
|
||||
set, found, err = s.media.GetStickerSetByID(ctx, ref.ID)
|
||||
case domain.StickerSetRefByShortName:
|
||||
set, found, err = s.media.GetStickerSetByShortName(ctx, ref.ShortName)
|
||||
case domain.StickerSetRefBySystem:
|
||||
set, found, err = s.media.GetStickerSetBySystemKey(ctx, ref.SystemKey)
|
||||
default:
|
||||
return domain.StickerSet{}, nil, false, nil
|
||||
}
|
||||
if err != nil || !found {
|
||||
return domain.StickerSet{}, nil, found, err
|
||||
}
|
||||
docs, err := s.media.GetDocuments(ctx, set.DocumentIDs)
|
||||
if err != nil {
|
||||
return domain.StickerSet{}, nil, false, err
|
||||
}
|
||||
ordered := orderDocuments(docs, set.DocumentIDs)
|
||||
s.stickerSetCache.put(set, ordered)
|
||||
return set, ordered, true, nil
|
||||
}
|
||||
|
||||
// orderDocuments 把无序的文档按 ids 顺序重排（GetDocuments 用 ANY 查询不保证顺序）。
|
||||
func orderDocuments(docs []domain.Document, ids []int64) []domain.Document {
|
||||
byID := make(map[int64]domain.Document, len(docs))
|
||||
for _, d := range docs {
|
||||
byID[d.ID] = d
|
||||
}
|
||||
out := make([]domain.Document, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
if d, ok := byID[id]; ok {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// assembleUpload 把已上传分片按 part 顺序拼成完整字节，并清理分片。
|
||||
// expectedParts>0 时校验分片连续且齐全。
|
||||
func (s *Service) assembleUpload(ctx context.Context, ownerUserID, fileID int64, expectedParts int) ([]byte, error) {
|
||||
parts, err := s.media.LoadFileParts(ctx, ownerUserID, fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return nil, domain.ErrFilePartsInvalid
|
||||
}
|
||||
if expectedParts > 0 && len(parts) != expectedParts {
|
||||
return nil, domain.ErrFilePartsInvalid
|
||||
}
|
||||
total := 0
|
||||
for i, p := range parts {
|
||||
if p.Part != i {
|
||||
return nil, domain.ErrFilePartsInvalid // 缺片或乱序
|
||||
}
|
||||
total += len(p.Bytes)
|
||||
}
|
||||
buf := make([]byte, 0, total)
|
||||
for _, p := range parts {
|
||||
buf = append(buf, p.Bytes...)
|
||||
}
|
||||
if err := s.media.DeleteFileParts(ctx, ownerUserID, fileID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
func validatePart(part, size int) error {
|
||||
if part < 0 || part >= MaxUploadParts {
|
||||
return domain.ErrFilePartInvalid
|
||||
}
|
||||
if size == 0 {
|
||||
return domain.ErrFilePartInvalid
|
||||
}
|
||||
if size > MaxUploadPartBytes {
|
||||
return domain.ErrFilePartTooBig
|
||||
}
|
||||
return nil
|
||||
}
|
||||
136
internal/app/files/warm.go
Normal file
136
internal/app/files/warm.go
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// WarmStats 汇报一次启动资源缓存预热结果。
|
||||
type WarmStats struct {
|
||||
StickerSets int
|
||||
Documents int
|
||||
Blobs int
|
||||
}
|
||||
|
||||
// WarmCaches 从已持久化的 sticker/reaction 元数据预热小 blob 字节缓存与完整 sticker set 缓存。
|
||||
// SeedMedia 在已有数据时会跳过导入；该方法保证普通 server 重启后历史 sticker 首次渲染也不是冷缓存。
|
||||
func (s *Service) WarmCaches(ctx context.Context) (WarmStats, error) {
|
||||
var stats WarmStats
|
||||
seenDocs := make(map[int64]struct{})
|
||||
for _, kind := range []domain.StickerSetKind{
|
||||
domain.StickerSetKindStickers,
|
||||
domain.StickerSetKindEmoji,
|
||||
domain.StickerSetKindMasks,
|
||||
domain.StickerSetKindSystem,
|
||||
} {
|
||||
sets, err := s.media.ListStickerSets(ctx, kind)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
for _, set := range sets {
|
||||
docs, err := s.media.GetDocuments(ctx, set.DocumentIDs)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
ordered := orderDocuments(docs, set.DocumentIDs)
|
||||
s.stickerSetCache.put(set, ordered)
|
||||
stats.StickerSets++
|
||||
for _, doc := range ordered {
|
||||
if _, ok := seenDocs[doc.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seenDocs[doc.ID] = struct{}{}
|
||||
stats.Documents++
|
||||
warmed, err := s.prewarmDocumentBlobs(ctx, doc)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.Blobs += warmed
|
||||
}
|
||||
}
|
||||
}
|
||||
reactions, err := s.media.ListAvailableReactions(ctx)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
reactionIDs := make([]int64, 0, len(reactions)*4)
|
||||
for _, reaction := range reactions {
|
||||
reactionIDs = append(reactionIDs, reaction.DocumentIDs()...)
|
||||
}
|
||||
docs, err := s.media.GetDocuments(ctx, reactionIDs)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
for _, doc := range docs {
|
||||
if _, ok := seenDocs[doc.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seenDocs[doc.ID] = struct{}{}
|
||||
stats.Documents++
|
||||
warmed, err := s.prewarmDocumentBlobs(ctx, doc)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.Blobs += warmed
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (s *Service) prewarmDocumentBlobs(ctx context.Context, doc domain.Document) (int, error) {
|
||||
if doc.ID == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
warmed := 0
|
||||
ok, err := s.prewarmLocationKey(ctx, fmt.Sprintf("doc:%d", doc.ID))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if ok {
|
||||
warmed++
|
||||
}
|
||||
for _, thumb := range doc.Thumbs {
|
||||
if !thumb.Downloadable() {
|
||||
continue
|
||||
}
|
||||
ok, err := s.prewarmLocationKey(ctx, fmt.Sprintf("doc:%d:%s", doc.ID, thumb.Type))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if ok {
|
||||
warmed++
|
||||
}
|
||||
}
|
||||
return warmed, nil
|
||||
}
|
||||
|
||||
func (s *Service) prewarmLocationKey(ctx context.Context, locationKey string) (bool, error) {
|
||||
blob, ok := s.blobCache.get(locationKey)
|
||||
if !ok {
|
||||
var (
|
||||
found bool
|
||||
err error
|
||||
)
|
||||
blob, found, err = s.media.GetFileBlob(ctx, locationKey)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !found {
|
||||
return false, nil
|
||||
}
|
||||
s.blobCache.put(locationKey, blob)
|
||||
}
|
||||
if blob.Size <= 0 || blob.Size > blobBytesCacheMaxEntryBytes || s.byteCache.has(blob.ObjectKey) {
|
||||
return false, nil
|
||||
}
|
||||
data, total, err := s.blobs.GetRange(ctx, blob.ObjectKey, 0, blobBytesCacheMaxEntryBytes+1)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read blob %q: %w", blob.LocationKey, err)
|
||||
}
|
||||
if total <= blobBytesCacheMaxEntryBytes && int64(len(data)) == total {
|
||||
s.byteCache.put(blob.ObjectKey, data)
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue