feat: sync multilayer td integration

This commit is contained in:
A 2026-07-15 13:32:06 +08:00
parent 20a310f6ca
commit 766c5db992
491 changed files with 26235 additions and 35340 deletions

View file

@ -10,7 +10,7 @@ import (
"image"
"image/color"
stddraw "image/draw"
_ "image/jpeg" // 注册 jpeg DecodeConfig用于读取上传头像/图片尺寸
"image/jpeg"
"image/png"
"io"
"math"
@ -24,8 +24,8 @@ import (
_ "golang.org/x/image/webp" // 注册 webp Decode用于 custom emoji / sticker 静态缩略图合成
)
// 头像与图片消息共用的尺寸 type'a' 小图≤160'c' 大图,'x' 通用下载尺寸。
// 同一份上传字节在多个 location_key 下建 blob不做实际缩放dev 主路径足够)
// 头像使用真实的 's'(≤150)/'a'(≤160)/'c'(原图) rendition图片消息使用
// 'm' 缩略与 'x' 大图。每个头像 location_key 的元数据尺寸必须与实际 blob 一致
// UploadProfilePhoto 把已上传文件组装成头像 Photo落 blob/photos/profile_photos并设为当前头像。
func (s *Service) UploadProfilePhoto(ctx context.Context, ownerType domain.PeerType, ownerID int64, file domain.UploadedFileRef, date int) (domain.Photo, error) {
@ -103,8 +103,8 @@ func (s *Service) GetDocument(ctx context.Context, id int64) (domain.Document, b
return s.media.GetDocument(ctx, id)
}
// CreateAvatarFromUpload 把已上传文件组装成头像 Photo'a'/'c' 尺寸,匹配 InputPeerPhotoFileLocation
// big/small 与 channelFull 合成尺寸的下载路径),不绑定 profile_photos。用于频道 editPhoto。
// CreateAvatarFromUpload 把已上传文件组装成头像 Photo's'/'a'/'c' 尺寸,'a'/'c' 匹配
// InputPeerPhotoFileLocation big/small 与 channelFull 下载路径),不绑定 profile_photos。用于频道 editPhoto。
func (s *Service) CreateAvatarFromUpload(ctx context.Context, file domain.UploadedFileRef) (domain.Photo, error) {
data, err := s.assembleUpload(ctx, file.OwnerUserID, file.FileID, file.Parts)
if err != nil {
@ -113,7 +113,7 @@ func (s *Service) CreateAvatarFromUpload(ctx context.Context, file domain.Upload
if len(data) == 0 {
return domain.Photo{}, domain.ErrPhotoInvalid
}
return s.createPhoto(ctx, data, photoSizeSpecsForAvatar(data))
return s.createAvatarPhoto(ctx, data)
}
// CreateAvatarVideoFromUpload stores an animated profile video as photo.video_sizes.
@ -151,7 +151,7 @@ func (s *Service) createAvatarVideoFromUpload(ctx context.Context, file domain.U
}
s.blobCache.put(blob.LocationKey, blob)
stillBytes := s.avatarVideoStill(ctx, body, extraSizes)
sizes, err := s.putPhotoStaticSizes(ctx, photoID, stillBytes, photoSizeSpecsForAvatar(stillBytes))
sizes, err := s.putAvatarStaticSizes(ctx, photoID, stillBytes, photoSizeSpecsForAvatar(stillBytes))
if err != nil {
return domain.Photo{}, err
}
@ -192,7 +192,7 @@ func (s *Service) CreateAvatarMarkup(ctx context.Context, size domain.PhotoSize)
}
photoID := randomID()
stillBytes := s.generatedAvatarStill(ctx, size)
sizes, err := s.putPhotoStaticSizes(ctx, photoID, stillBytes, photoSizeSpecsForAvatar(stillBytes))
sizes, err := s.putAvatarStaticSizes(ctx, photoID, stillBytes, photoSizeSpecsForAvatar(stillBytes))
if err != nil {
return domain.Photo{}, err
}
@ -605,6 +605,26 @@ func (s *Service) createPhoto(ctx context.Context, data []byte, specs []photoSiz
return photo, nil
}
func (s *Service) createAvatarPhoto(ctx context.Context, data []byte) (domain.Photo, error) {
photoID := randomID()
sizes, err := s.putAvatarStaticSizes(ctx, photoID, data, photoSizeSpecsForAvatar(data))
if err != nil {
return domain.Photo{}, err
}
photo := domain.Photo{
ID: photoID,
AccessHash: randomID(),
FileReference: randomFileReference(),
Date: int(time.Now().Unix()),
DCID: s.dc,
Sizes: sizes,
}
if err := s.media.PutPhoto(ctx, photo); err != nil {
return domain.Photo{}, err
}
return photo, nil
}
func (s *Service) putPhotoStaticSizes(ctx context.Context, photoID int64, data []byte, specs []photoSizeSpec) ([]domain.PhotoSize, error) {
objectKey, err := s.blobs.Put(ctx, data)
if err != nil {
@ -630,6 +650,87 @@ func (s *Service) putPhotoStaticSizes(ctx context.Context, photoID int64, data [
return sizes, nil
}
// putAvatarStaticSizes stores independently rendered avatar sizes. DrKLO uses the
// photos.uploadProfilePhoto response's closest 150px size as an immediate local
// location, while UserProfilePhoto updates synthesize the canonical 'a' location.
// Keeping a real 's' rendition therefore gives those two states distinct keys and,
// more importantly, keeps every advertised size backed by matching image bytes.
func (s *Service) putAvatarStaticSizes(ctx context.Context, photoID int64, data []byte, specs []photoSizeSpec) ([]domain.PhotoSize, error) {
src, format, err := image.Decode(bytes.NewReader(data))
if err != nil || src.Bounds().Dx() <= 0 || src.Bounds().Dy() <= 0 {
return nil, domain.ErrPhotoInvalid
}
sizes := make([]domain.PhotoSize, 0, len(specs))
for _, spec := range specs {
rendition, err := avatarRendition(data, src, format, spec)
if err != nil {
return nil, err
}
objectKey, err := s.blobs.Put(ctx, rendition)
if err != nil {
return nil, err
}
blob := domain.FileBlob{
LocationKey: fmt.Sprintf("photo:%d:%s", photoID, spec.Type),
Backend: domain.MediaBackend(s.blobs.Name()),
ObjectKey: objectKey,
Size: int64(len(rendition)),
MimeType: imageMimeType(rendition),
}
if err := s.media.PutFileBlob(ctx, blob); err != nil {
return nil, err
}
s.blobCache.put(blob.LocationKey, blob)
s.prewarmSmallBlob(objectKey, rendition)
sizes = append(sizes, domain.PhotoSize{
Kind: domain.PhotoSizeKindDefault,
Type: spec.Type,
W: spec.W,
H: spec.H,
Size: len(rendition),
})
}
return sizes, nil
}
func avatarRendition(original []byte, src image.Image, format string, spec photoSizeSpec) ([]byte, error) {
bounds := src.Bounds()
if bounds.Dx() == spec.W && bounds.Dy() == spec.H {
return append([]byte(nil), original...), nil
}
if spec.W <= 0 || spec.H <= 0 {
return nil, domain.ErrPhotoInvalid
}
dst := image.NewRGBA(image.Rect(0, 0, spec.W, spec.H))
srcRect := centerCropRect(bounds, spec.W, spec.H)
xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, srcRect, xdraw.Src, nil)
var buf bytes.Buffer
if format == "jpeg" {
if err := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: 90}); err != nil {
return nil, err
}
} else if err := png.Encode(&buf, dst); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func centerCropRect(bounds image.Rectangle, targetW, targetH int) image.Rectangle {
sourceW, sourceH := bounds.Dx(), bounds.Dy()
if int64(sourceW)*int64(targetH) > int64(sourceH)*int64(targetW) {
cropW := maxInt(1, sourceH*targetW/targetH)
x := bounds.Min.X + (sourceW-cropW)/2
return image.Rect(x, bounds.Min.Y, x+cropW, bounds.Max.Y)
}
if int64(sourceW)*int64(targetH) < int64(sourceH)*int64(targetW) {
cropH := maxInt(1, sourceW*targetH/targetW)
y := bounds.Min.Y + (sourceH-cropH)/2
return image.Rect(bounds.Min.X, y, bounds.Max.X, y+cropH)
}
return bounds
}
func (s *Service) putDocumentThumb(ctx context.Context, docID int64, thumbData []byte) (domain.PhotoSize, error) {
if len(thumbData) == 0 {
return domain.PhotoSize{}, fmt.Errorf("empty document thumbnail")
@ -723,12 +824,15 @@ type photoSizeSpec struct {
func photoSizeSpecsForAvatar(data []byte) []photoSizeSpec {
w, h := imageDimensions(data, 640, 640)
small := 160
if w < small {
small = w
shortSide := w
if h < shortSide {
shortSide = h
}
sSize := minInt(shortSide, 150)
aSize := minInt(shortSide, 160)
return []photoSizeSpec{
{Type: "a", W: small, H: small},
{Type: "s", W: sSize, H: sSize},
{Type: "a", W: aSize, H: aSize},
{Type: "c", W: w, H: h},
}
}
@ -767,10 +871,14 @@ const (
avatarMarkupMaxSourceBytes = 2 << 20 // emoji/sticker thumb 小对象保护线。
)
// avatarVideoStill 生成动画头像的静态尺寸字节:优先抽取上传视频首帧——动画头像
// emoji/sticker 构造器或自选视频)的首帧就是用户在客户端看到的真实画面(彩色
// emoji、圆角、布局都一致抽帧不可用时回退到按 markup 服务端合成
// avatarVideoStill 生成动画头像的静态尺寸字节。emoji/sticker markup 能解析到
// 服务端缩略图时优先合成DrKLO 生成的 MP4 第一帧可能只有背景渐变,直接抽第一帧
// 会让静态头像永久缺少 emoji。普通视频或 markup 资源不可用时才回退 ffmpeg 首帧
func (s *Service) avatarVideoStill(ctx context.Context, body assembledUploadBlob, extraSizes []domain.PhotoSize) []byte {
markup := avatarStillMarkup(extraSizes)
if still, ok := s.generatedAvatarMarkupStill(ctx, markup); ok {
return still
}
if s.thumbs != nil && body.Size > 0 && body.Size <= videoThumbnailMaxInputBytes {
data, total, err := s.blobs.GetRange(ctx, body.ObjectKey, 0, body.Size)
if err == nil && int64(len(data)) == total && total == body.Size {
@ -789,17 +897,30 @@ func (s *Service) avatarVideoStill(ctx context.Context, body assembledUploadBlob
zap.Error(err))
}
}
return s.generatedAvatarStill(ctx, avatarStillMarkup(extraSizes))
return s.generatedAvatarStill(ctx, markup)
}
func (s *Service) generatedAvatarStill(ctx context.Context, markup domain.PhotoSize) []byte {
img := generatedAvatarBackground(markup.BackgroundColors)
if overlay, tintWhite, ok := s.avatarMarkupOverlay(ctx, markup); ok {
drawAvatarMarkup(img, overlay, tintWhite)
if still, ok := s.generatedAvatarMarkupStillOnBackground(ctx, markup, img); ok {
return still
}
return encodeAvatarPNG(img)
}
func (s *Service) generatedAvatarMarkupStill(ctx context.Context, markup domain.PhotoSize) ([]byte, bool) {
return s.generatedAvatarMarkupStillOnBackground(ctx, markup, generatedAvatarBackground(markup.BackgroundColors))
}
func (s *Service) generatedAvatarMarkupStillOnBackground(ctx context.Context, markup domain.PhotoSize, img *image.RGBA) ([]byte, bool) {
overlay, tintWhite, ok := s.avatarMarkupOverlay(ctx, markup)
if !ok {
return nil, false
}
drawAvatarMarkup(img, overlay, tintWhite)
return encodeAvatarPNG(img), true
}
func generatedAvatarBackground(colors []int) *image.RGBA {
if len(colors) == 0 {
colors = []int{0x5b8def, 0x53c6a4}
@ -865,6 +986,11 @@ func (s *Service) avatarMarkupOverlay(ctx context.Context, markup domain.PhotoSi
zap.Error(err))
return nil, false, false
}
// Seed 的 1x1 透明图只是“没有可用静态资源”的显式占位,不是可合成
// 内容。把它视为 unavailable交给调用方回退到 ffmpeg 视频首帧。
if img.Bounds().Dx() <= 1 || img.Bounds().Dy() <= 1 {
return nil, false, false
}
return img, documentIsTextColorEmoji(doc), true
}

View file

@ -223,6 +223,57 @@ func TestCreatePhotoFromBytesStoresDownloadableMessageSizes(t *testing.T) {
}
}
func TestCreateAvatarFromUploadStoresRealSizedRenditions(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
blobs, err := NewLocalFS(t.TempDir())
if err != nil {
t.Fatalf("NewLocalFS: %v", err)
}
svc := NewService(media, blobs, 2)
data := testJPEG(t, 640, 480)
if _, err := svc.SaveFilePart(ctx, 10, 301, 0, data); err != nil {
t.Fatalf("SaveFilePart: %v", err)
}
photo, err := svc.CreateAvatarFromUpload(ctx, domain.UploadedFileRef{
OwnerUserID: 10,
FileID: 301,
Parts: 1,
Name: "avatar.jpg",
})
if err != nil {
t.Fatalf("CreateAvatarFromUpload: %v", err)
}
wants := map[string]image.Point{
"s": {X: 150, Y: 150},
"a": {X: 160, Y: 160},
"c": {X: 640, Y: 480},
}
if len(photo.Sizes) != len(wants) {
t.Fatalf("avatar sizes = %+v, want s/a/c", photo.Sizes)
}
objectKeys := map[string]struct{}{}
for _, size := range photo.Sizes {
want, ok := wants[size.Type]
if !ok || size.W != want.X || size.H != want.Y {
t.Fatalf("avatar size = %+v, want one of %v", size, wants)
}
assertAvatarImageSize(t, svc, photo.ID, size.Type, want.X, want.Y, "image/jpeg")
blob, found, err := media.GetFileBlob(ctx, fmt.Sprintf("photo:%d:%s", photo.ID, size.Type))
if err != nil || !found {
t.Fatalf("avatar %s blob found=%v err=%v", size.Type, found, err)
}
if blob.Size != int64(size.Size) {
t.Fatalf("avatar %s blob size=%d metadata size=%d", size.Type, blob.Size, size.Size)
}
objectKeys[blob.ObjectKey] = struct{}{}
}
if len(objectKeys) != 3 {
t.Fatalf("avatar object keys = %v, want distinct s/a/c renditions", objectKeys)
}
}
func TestCreateDocumentFromBytesStoresBodyAndAttributes(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
@ -299,8 +350,9 @@ func TestCreateAvatarMarkupGeneratesDownloadableStaticSizes(t *testing.T) {
if !domain.PhotoHasVideo(photo.Sizes) {
t.Fatalf("avatar markup photo sizes = %+v, want video markup", photo.Sizes)
}
assertDownloadableAvatarSize(t, svc, photo.ID, "a")
assertDownloadableAvatarSize(t, svc, photo.ID, "c")
assertDownloadableAvatarSize(t, svc, photo.ID, "s", 150)
assertDownloadableAvatarSize(t, svc, photo.ID, "a", 160)
assertDownloadableAvatarSize(t, svc, photo.ID, "c", 640)
}
// TestCreateAvatarMarkupComposesEmojiThumbIntoStaticSizes 守护两个行为:
@ -433,8 +485,9 @@ func TestCreateAvatarVideoMarkupGeneratesDownloadableStaticSizes(t *testing.T) {
if err != nil {
t.Fatalf("CreateAvatarVideoMarkupFromUpload: %v", err)
}
assertDownloadableAvatarSize(t, svc, photo.ID, "a")
assertDownloadableAvatarSize(t, svc, photo.ID, "c")
assertDownloadableAvatarSize(t, svc, photo.ID, "s", 150)
assertDownloadableAvatarSize(t, svc, photo.ID, "a", 160)
assertDownloadableAvatarSize(t, svc, photo.ID, "c", 640)
chunk, found, err := svc.GetFile(ctx, domain.FileDownloadRequest{
LocationKey: fmt.Sprintf("photo:%d:u", photo.ID),
Offset: 0,
@ -448,9 +501,9 @@ func TestCreateAvatarVideoMarkupGeneratesDownloadableStaticSizes(t *testing.T) {
}
}
// TestCreateAvatarVideoMarkupStillUsesVideoFirstFrame 守护动画头像静态尺寸优先取
// 上传视频首帧(客户端真实渲染画面),而不是服务端合成的近似 still
func TestCreateAvatarVideoMarkupStillUsesVideoFirstFrame(t *testing.T) {
// TestCreateAvatarVideoMarkupFallsBackToVideoFirstFrame 守护 markup document/thumb
// 不可用时仍可从上传视频抽帧,不能让普通动画头像失去静态尺寸
func TestCreateAvatarVideoMarkupFallsBackToVideoFirstFrame(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
blobs, err := NewLocalFS(t.TempDir())
@ -479,7 +532,7 @@ func TestCreateAvatarVideoMarkupStillUsesVideoFirstFrame(t *testing.T) {
t.Fatalf("thumbnailer calls = %d, want 1", thumbnailer.calls)
}
chunk, found, err := svc.GetFile(ctx, domain.FileDownloadRequest{
LocationKey: fmt.Sprintf("photo:%d:a", photo.ID),
LocationKey: fmt.Sprintf("photo:%d:c", photo.ID),
Offset: 0,
Limit: 1 << 20,
})
@ -492,9 +545,112 @@ func TestCreateAvatarVideoMarkupStillUsesVideoFirstFrame(t *testing.T) {
if chunk.MimeType != "image/jpeg" {
t.Fatalf("avatar still mime = %q, want image/jpeg from extracted frame", chunk.MimeType)
}
assertAvatarImageSize(t, svc, photo.ID, "s", 150, 150, "image/jpeg")
assertAvatarImageSize(t, svc, photo.ID, "a", 160, 160, "image/jpeg")
}
func assertDownloadableAvatarSize(t *testing.T, svc *Service, photoID int64, sizeType string) {
func TestCreateAvatarVideoMarkupRejectsSyntheticPreviewAndFallsBackToVideo(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
blobs, err := NewLocalFS(t.TempDir())
if err != nil {
t.Fatalf("NewLocalFS: %v", err)
}
const emojiID = int64(78)
if err := media.PutDocument(ctx, domain.Document{
ID: emojiID,
MimeType: "application/x-tgsticker",
Thumbs: []domain.PhotoSize{{
Kind: domain.PhotoSizeKindCached, Type: "m", W: 1, H: 1,
Bytes: append([]byte(nil), seedSyntheticTGStickerPreviewThumbPNG...),
}},
}); err != nil {
t.Fatalf("PutDocument: %v", err)
}
frame := testJPEG(t, 640, 640)
thumbnailer := &fakeVideoThumbnailer{thumb: frame}
svc := NewService(media, blobs, 2, WithVideoThumbnailer(thumbnailer))
if _, err := svc.SaveFilePart(ctx, 10, 503, 0, []byte("profile-video-without-server-preview")); err != nil {
t.Fatalf("SaveFilePart: %v", err)
}
photo, err := svc.CreateAvatarVideoMarkupFromUpload(ctx,
domain.UploadedFileRef{OwnerUserID: 10, FileID: 503, Parts: 1, Name: "avatar.mp4"},
0,
domain.PhotoSize{Kind: domain.PhotoSizeKindVideoEmojiMarkup, EmojiID: emojiID, BackgroundColors: []int{0x112233}})
if err != nil {
t.Fatalf("CreateAvatarVideoMarkupFromUpload: %v", err)
}
if thumbnailer.calls != 1 {
t.Fatalf("thumbnailer calls = %d, want synthetic preview rejected and video fallback used", thumbnailer.calls)
}
chunk, found, err := svc.GetFile(ctx, domain.FileDownloadRequest{LocationKey: fmt.Sprintf("photo:%d:c", photo.ID), Limit: 1 << 20})
if err != nil || !found {
t.Fatalf("avatar c blob found=%v err=%v", found, err)
}
if !bytes.Equal(chunk.Bytes, frame) {
t.Fatal("avatar still did not use extracted video frame after rejecting synthetic preview")
}
}
// TestCreateAvatarVideoMarkupPrefersComposedStill 守护 DrKLO emoji 构造器边界:
// 客户端生成 MP4 的第一帧可能只有渐变背景;只要 markup thumb 可解析,静态头像
// 必须用服务端合成结果,确保 emoji 在当前 session 回显和冷启动中都可见。
func TestCreateAvatarVideoMarkupPrefersComposedStill(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
blobs, err := NewLocalFS(t.TempDir())
if err != nil {
t.Fatalf("NewLocalFS: %v", err)
}
const emojiID = int64(501)
if err := media.PutDocument(ctx, domain.Document{
ID: emojiID,
MimeType: "application/x-tgsticker",
Thumbs: []domain.PhotoSize{{
Kind: domain.PhotoSizeKindCached,
Type: "m",
W: 64,
H: 64,
Bytes: testTransparentThumbPNG(t),
}},
}); err != nil {
t.Fatalf("PutDocument: %v", err)
}
thumbnailer := &fakeVideoThumbnailer{thumb: testJPEG(t, 320, 320)}
svc := NewService(media, blobs, 2, WithVideoThumbnailer(thumbnailer))
if _, err := svc.SaveFilePart(ctx, 10, 502, 0, []byte("background-only-profile-video")); err != nil {
t.Fatalf("SaveFilePart: %v", err)
}
photo, err := svc.CreateAvatarVideoMarkupFromUpload(ctx,
domain.UploadedFileRef{OwnerUserID: 10, FileID: 502, Parts: 1, Name: "avatar.mp4"},
0,
domain.PhotoSize{
Kind: domain.PhotoSizeKindVideoEmojiMarkup,
EmojiID: emojiID,
BackgroundColors: []int{0x112233, 0x445566},
})
if err != nil {
t.Fatalf("CreateAvatarVideoMarkupFromUpload: %v", err)
}
if thumbnailer.calls != 0 {
t.Fatalf("thumbnailer calls = %d, want 0 when markup still is available", thumbnailer.calls)
}
r, g, b, _ := avatarStillCenterPixel(t, svc, photo.ID)
if r < 200 || g > 90 || b > 90 {
t.Fatalf("composed center pixel rgb=(%d,%d,%d), want visible red emoji overlay", r, g, b)
}
assertDownloadableAvatarSize(t, svc, photo.ID, "s", 150)
assertDownloadableAvatarSize(t, svc, photo.ID, "a", 160)
assertDownloadableAvatarSize(t, svc, photo.ID, "c", 640)
}
func assertDownloadableAvatarSize(t *testing.T, svc *Service, photoID int64, sizeType string, side int) {
t.Helper()
assertAvatarImageSize(t, svc, photoID, sizeType, side, side, "image/png")
}
func assertAvatarImageSize(t *testing.T, svc *Service, photoID int64, sizeType string, wantW, wantH int, wantMime string) {
t.Helper()
chunk, found, err := svc.GetFile(context.Background(), domain.FileDownloadRequest{
LocationKey: fmt.Sprintf("photo:%d:%s", photoID, sizeType),
@ -504,8 +660,15 @@ func assertDownloadableAvatarSize(t *testing.T, svc *Service, photoID int64, siz
if err != nil || !found {
t.Fatalf("avatar %s blob found=%v err=%v", sizeType, found, err)
}
if len(chunk.Bytes) == 0 || chunk.MimeType != "image/png" {
t.Fatalf("avatar %s chunk mime=%q bytes=%d, want image/png bytes", sizeType, chunk.MimeType, len(chunk.Bytes))
if len(chunk.Bytes) == 0 || chunk.MimeType != wantMime {
t.Fatalf("avatar %s chunk mime=%q bytes=%d, want %s bytes", sizeType, chunk.MimeType, len(chunk.Bytes), wantMime)
}
img, _, err := image.Decode(bytes.NewReader(chunk.Bytes))
if err != nil {
t.Fatalf("decode avatar %s: %v", sizeType, err)
}
if gotW, gotH := img.Bounds().Dx(), img.Bounds().Dy(); gotW != wantW || gotH != wantH {
t.Fatalf("avatar %s pixels=%dx%d, want %dx%d", sizeType, gotW, gotH, wantW, wantH)
}
}

View file

@ -81,6 +81,7 @@ func (s *Service) SeedMedia(ctx context.Context, root string, maxRegularSets int
// 这样向已部署(非空 store)的 data/sticker-seed 丢新集后重启即可生效,无需清库重 seed。
// 仅当检测到旧版缩略图/可渲染预览元数据缺失时 force=true 全量重导修复。
forceSticker := false
previewState := fmt.Sprintf("%s:dc=%d", seedStickerPreviewStateVersion, s.dc)
if n, err := s.media.CountStickerSets(ctx); err != nil {
return stats, err
} else if n > 0 {
@ -88,11 +89,18 @@ func (s *Service) SeedMedia(ctx context.Context, root string, maxRegularSets int
if err != nil {
return stats, err
}
forceSticker = stale
migrated, err := s.seedStateMatches(ctx, seedStickerPreviewStateKey, previewState)
if err != nil {
return stats, err
}
forceSticker = stale || !migrated
}
if err := s.seedStickerSets(ctx, root, maxRegularSets, forceSticker, &stats); err != nil {
return stats, fmt.Errorf("seed sticker sets: %w", err)
}
if err := s.putSeedState(ctx, seedStickerPreviewStateKey, previewState); err != nil {
return stats, fmt.Errorf("record sticker preview seed state: %w", err)
}
s.logSeedPhase("sticker_sets", phaseStarted, phaseBefore, stats)
phaseStarted = time.Now()
@ -379,6 +387,10 @@ func (s *Service) importDocument(ctx context.Context, dj seedDocumentJSON, binDi
if dj.ID == 0 {
return domain.Document{}, nil
}
existing, existingFound, err := s.media.GetDocument(ctx, dj.ID)
if err != nil {
return domain.Document{}, err
}
ref, _ := hex.DecodeString(dj.FileReference)
doc := domain.Document{
ID: dj.ID,
@ -436,27 +448,40 @@ func (s *Service) importDocument(ctx context.Context, dj seedDocumentJSON, binDi
if err != nil {
return domain.Document{}, err
}
objectKey, err := s.blobs.Put(ctx, data)
if err != nil {
return domain.Document{}, err
}
ps.Size = len(data)
if err := s.media.PutFileBlob(ctx, domain.FileBlob{
LocationKey: fmt.Sprintf("doc:%d:%s", doc.ID, ps.Type),
Backend: domain.MediaBackend(s.blobs.Name()),
ObjectKey: objectKey,
Size: int64(len(data)),
MimeType: seedThumbMimeType(data),
}); err != nil {
return domain.Document{}, err
}
ps = seedInlineCachedDocumentThumb(ps, data)
s.prewarmSmallBlob(objectKey, data)
stats.Blobs++
// A duplicate document can be present in several catalogs. Do not replace a
// better already-persisted preview (and its shared location key) with a lower
// quality rendition from the catalog imported later.
if prior, ok := seedDocumentThumbByType(existing.Thumbs, ps.Type); existingFound && ok && seedPhotoSizeBetter(prior, ps) {
ps = prior
} else {
objectKey, err := s.blobs.Put(ctx, data)
if err != nil {
return domain.Document{}, err
}
if err := s.media.PutFileBlob(ctx, domain.FileBlob{
LocationKey: fmt.Sprintf("doc:%d:%s", doc.ID, ps.Type),
Backend: domain.MediaBackend(s.blobs.Name()),
ObjectKey: objectKey,
Size: int64(len(data)),
MimeType: seedThumbMimeType(data),
}); err != nil {
return domain.Document{}, err
}
s.prewarmSmallBlob(objectKey, data)
stats.Blobs++
}
}
thumbs = append(thumbs, ps)
}
doc.Thumbs = thumbs
if existingFound {
doc.Thumbs = mergeSeedDocumentThumbs(existing.Thumbs, doc.Thumbs)
}
if err := s.ensureSeedCachedThumbBlobs(ctx, doc, stats); err != nil {
return domain.Document{}, err
}
if err := s.ensureTGStickerPreviewThumb(ctx, &doc, stats); err != nil {
return domain.Document{}, err
@ -700,6 +725,138 @@ func seedInlineCachedDocumentThumb(ps domain.PhotoSize, data []byte) domain.Phot
return ps
}
// mergeSeedDocumentThumbs makes duplicate seed imports monotonic for preview quality.
// PhotoSize.Type is also the blob location suffix, so only one winner per type may be
// advertised. Incoming metadata wins ties; a richer existing preview wins downgrades.
func mergeSeedDocumentThumbs(existing, incoming []domain.PhotoSize) []domain.PhotoSize {
out := append([]domain.PhotoSize(nil), incoming...)
byType := make(map[string]int, len(out))
for i, thumb := range out {
if thumb.Type != "" {
byType[thumb.Type] = i
}
}
for _, thumb := range existing {
if thumb.Type != "" {
if i, ok := byType[thumb.Type]; ok {
if seedPhotoSizeBetter(thumb, out[i]) {
out[i] = thumb
}
continue
}
byType[thumb.Type] = len(out)
}
out = append(out, thumb)
}
hasRealPreview := false
for _, thumb := range out {
if !seedSyntheticTGStickerPreviewThumb(thumb) && seedPhotoSizePreviewTier(thumb) > 1 {
hasRealPreview = true
break
}
}
if !hasRealPreview {
return out
}
filtered := out[:0]
for _, thumb := range out {
if !seedSyntheticTGStickerPreviewThumb(thumb) {
filtered = append(filtered, thumb)
}
}
return filtered
}
func seedDocumentThumbByType(thumbs []domain.PhotoSize, typ string) (domain.PhotoSize, bool) {
for _, thumb := range thumbs {
if thumb.Type == typ {
return thumb, true
}
}
return domain.PhotoSize{}, false
}
func seedPhotoSizeBetter(a, b domain.PhotoSize) bool {
aTier, bTier := seedPhotoSizePreviewTier(a), seedPhotoSizePreviewTier(b)
if aTier != bTier {
return aTier > bTier
}
aArea, bArea := int64(a.W)*int64(a.H), int64(b.W)*int64(b.H)
if aArea != bArea {
return aArea > bArea
}
aPayload, bPayload := len(a.Bytes)+a.Size, len(b.Bytes)+b.Size
return aPayload > bPayload
}
func seedPhotoSizePreviewTier(thumb domain.PhotoSize) int {
if seedSyntheticTGStickerPreviewThumb(thumb) {
return 0
}
switch thumb.Kind {
case domain.PhotoSizeKindCached:
if len(thumb.Bytes) > 0 && thumb.W > 0 && thumb.H > 0 {
return 4
}
case domain.PhotoSizeKindDefault, domain.PhotoSizeKindProgressive:
if thumb.Size > 0 && thumb.W > 0 && thumb.H > 0 {
return 4
}
case domain.PhotoSizeKindPath, domain.PhotoSizeKindStripped:
if len(thumb.Bytes) > 0 {
return 3
}
}
return 1
}
func seedSyntheticTGStickerPreviewThumb(thumb domain.PhotoSize) bool {
return thumb.Kind == domain.PhotoSizeKindCached &&
thumb.Type == seedSyntheticDocumentThumbType &&
thumb.W == 1 && thumb.H == 1 &&
bytes.Equal(thumb.Bytes, seedSyntheticTGStickerPreviewThumbPNG)
}
// ensureSeedCachedThumbBlobs keeps the RPC conversion invariant: document cached
// previews are exposed as downloadable PhotoSize entries, so every advertised type
// must have a matching blob even when the source JSON carried the bytes inline.
func (s *Service) ensureSeedCachedThumbBlobs(ctx context.Context, doc domain.Document, stats *SeedStats) error {
for _, thumb := range doc.Thumbs {
if thumb.Kind != domain.PhotoSizeKindCached || thumb.Type == "" || len(thumb.Bytes) == 0 {
continue
}
locationKey := fmt.Sprintf("doc:%d:%s", doc.ID, thumb.Type)
mimeType := seedThumbMimeType(thumb.Bytes)
stored, found, err := s.media.GetFileBlob(ctx, locationKey)
if err != nil {
return err
}
if found && stored.Size == int64(len(thumb.Bytes)) && stored.MimeType == mimeType {
continue
}
if s.blobs == nil {
return fmt.Errorf("blob backend not configured for cached document thumb %s", locationKey)
}
objectKey, err := s.blobs.Put(ctx, thumb.Bytes)
if err != nil {
return err
}
if err := s.media.PutFileBlob(ctx, domain.FileBlob{
LocationKey: locationKey,
Backend: domain.MediaBackend(s.blobs.Name()),
ObjectKey: objectKey,
Size: int64(len(thumb.Bytes)),
MimeType: mimeType,
}); err != nil {
return err
}
s.prewarmSmallBlob(objectKey, thumb.Bytes)
stats.Blobs++
}
return nil
}
func (s *Service) ensureTGStickerPreviewThumb(ctx context.Context, doc *domain.Document, stats *SeedStats) error {
if !seedDocumentNeedsSyntheticTGStickerPreviewThumb(*doc) {
return nil
@ -826,11 +983,12 @@ func (s *Service) documentsNeedSeedRepair(ctx context.Context, ids []int64) (boo
if err != nil {
return false, err
}
if ok {
want := seedThumbMimeType(thumb.Bytes)
if want != "application/octet-stream" && blob.MimeType != want {
return true, nil
}
if !ok {
return true, nil
}
want := seedThumbMimeType(thumb.Bytes)
if blob.Size != int64(len(thumb.Bytes)) || (want != "application/octet-stream" && blob.MimeType != want) {
return true, nil
}
}
}

View file

@ -10,13 +10,20 @@ import (
"os"
"path/filepath"
"sort"
"telesrv/internal/domain"
)
const (
seedEffectsStateKey = "files.effects"
seedEffectsStateVersion = "effects-v2"
seedAppearanceStateKey = "files.appearance"
seedAppearanceStateVersion = "appearance-v1"
seedStickerPreviewStateKey = "files.sticker_previews"
// v2 explicitly rebuilds sticker documents written before duplicate seed imports
// became monotonic. Those databases may contain an effects-generated transparent
// 1x1 preview where the sticker catalog has a real static thumbnail.
seedStickerPreviewStateVersion = "sticker-previews-v2-monotonic"
seedAppearanceStateKey = "files.appearance"
seedAppearanceStateVersion = "appearance-v1"
)
func (s *Service) seedStateMatches(ctx context.Context, key, want string) (bool, error) {
@ -88,16 +95,17 @@ func seedDocumentJSONLocationKeys(dj seedDocumentJSON, index seedDirIndex) []str
}
for _, tj := range dj.Thumbs {
ps, downloadable := seedPhotoSize(tj)
if !downloadable || ps.Type == "" {
if ps.Type == "" {
continue
}
if _, ok := index.thumb[dj.ID][ps.Type]; ok {
if downloadable {
if _, ok := index.thumb[dj.ID][ps.Type]; ok {
keys = append(keys, fmt.Sprintf("doc:%d:%s", dj.ID, ps.Type))
}
} else if ps.Kind == domain.PhotoSizeKindCached && len(ps.Bytes) > 0 && ps.W > 0 && ps.H > 0 {
keys = append(keys, fmt.Sprintf("doc:%d:%s", dj.ID, ps.Type))
}
}
if seedDocumentJSONNeedsSyntheticTGStickerPreviewThumb(dj) {
keys = append(keys, fmt.Sprintf("doc:%d:%s", dj.ID, seedSyntheticDocumentThumbType))
}
return keys
}
@ -144,6 +152,28 @@ func (s *Service) seedDocumentJSONsReady(ctx context.Context, docs []seedDocumen
if doc.DCID != s.dc || doc.MimeType != dj.MimeType || doc.Size != dj.Size {
return false, nil
}
// A catalog without its own thumbnail may share this document with a richer
// catalog. Readiness follows the preview that is actually stored instead of
// demanding the synthetic "m" key and repeatedly downgrading that richer
// document on every import.
if seedDocumentJSONNeedsSyntheticTGStickerPreviewThumb(dj) {
if len(doc.Thumbs) == 0 {
return false, nil
}
for _, thumb := range doc.Thumbs {
switch thumb.Kind {
case domain.PhotoSizeKindDefault, domain.PhotoSizeKindProgressive, domain.PhotoSizeKindCached:
if thumb.Type == "" {
return false, nil
}
key := fmt.Sprintf("doc:%d:%s", doc.ID, thumb.Type)
if _, seen := seenLocationKeys[key]; !seen {
seenLocationKeys[key] = struct{}{}
locationKeys = append(locationKeys, key)
}
}
}
}
delete(expected, doc.ID)
}
if len(expected) > 0 {

View file

@ -1,6 +1,7 @@
package files
import (
"bytes"
"context"
"fmt"
"os"
@ -463,8 +464,8 @@ func TestSeedMediaRepairsPartialReactionBlobs(t *testing.T) {
if err != nil {
t.Fatalf("repair seed: %v", err)
}
if stats.Reactions != 1 || stats.Blobs != 3 || stats.Skipped {
t.Fatalf("repair stats = %+v, want repair import", stats)
if stats.Reactions != 1 || stats.Blobs != 2 || stats.Skipped {
t.Fatalf("repair stats = %+v, want two missing/revalidated main blobs without rewriting intact preview", stats)
}
if _, ok, _ := media.GetFileBlob(context.Background(), "doc:2222222"); !ok {
t.Fatal("missing reaction blob was not repaired")
@ -609,8 +610,113 @@ func TestSeedMediaSkipsUnchangedEffectsDocuments(t *testing.T) {
if err != nil {
t.Fatalf("repair seed: %v", err)
}
if repaired.Effects != 1 || repaired.Documents != 1 || repaired.Blobs != 2 {
t.Fatalf("repair stats = %+v, want missing blob to force reimport", repaired)
if repaired.Effects != 1 || repaired.Documents != 1 || repaired.Blobs != 1 {
t.Fatalf("repair stats = %+v, want missing main blob repaired without rewriting intact preview", repaired)
}
}
func TestSeedEffectsDoesNotDowngradeSharedStickerPreview(t *testing.T) {
ctx := context.Background()
seedDir := t.TempDir()
const sourceID int64 = 7777777
realThumb := writeStatusPackWithThumbSeed(t, seedDir, sourceID, 29)
writeEffectsSeed(t, seedDir, sourceID)
media := newFakeMediaStore()
blobs, err := NewLocalFS(t.TempDir())
if err != nil {
t.Fatalf("local fs: %v", err)
}
svc := NewService(media, blobs, 2)
first, err := svc.SeedMedia(ctx, seedDir, 0)
if err != nil {
t.Fatalf("first seed: %v", err)
}
if first.StickerSets != 1 || first.Effects != 1 {
t.Fatalf("first stats = %+v, want shared sticker and effect catalogs", first)
}
doc, ok, err := media.GetDocument(ctx, sourceID)
if err != nil || !ok {
t.Fatalf("shared document ok=%v err=%v", ok, err)
}
thumb, ok := findCachedThumb(doc.Thumbs)
if !ok {
t.Fatalf("shared document thumbs = %+v, want real cached preview", doc.Thumbs)
}
if thumb.W != 128 || thumb.H != 128 || !bytes.Equal(thumb.Bytes, realThumb) || seedSyntheticTGStickerPreviewThumb(thumb) {
t.Fatalf("shared preview = %+v, want original 128x128 catalog thumbnail", thumb)
}
blob, ok, err := media.GetFileBlob(ctx, fmt.Sprintf("doc:%d:m", sourceID))
if err != nil || !ok {
t.Fatalf("shared preview blob ok=%v err=%v", ok, err)
}
if blob.MimeType != "image/jpeg" || blob.Size != int64(len(realThumb)) {
t.Fatalf("shared preview blob = %+v, want real JPEG metadata", blob)
}
chunk, ok, err := svc.GetFile(ctx, domain.FileDownloadRequest{LocationKey: fmt.Sprintf("doc:%d:m", sourceID), Limit: 1024})
if err != nil || !ok {
t.Fatalf("get shared preview ok=%v err=%v", ok, err)
}
if !bytes.Equal(chunk.Bytes, realThumb) {
t.Fatalf("downloaded shared preview = %x, want %x", chunk.Bytes, realThumb)
}
second, err := svc.SeedMedia(ctx, seedDir, 0)
if err != nil {
t.Fatalf("second seed: %v", err)
}
if second.Documents != 0 || second.Blobs != 0 {
t.Fatalf("second stats = %+v, want shared rich preview to satisfy effects readiness", second)
}
}
func TestSeedMediaMigratesSyntheticStickerPreviewToExportedThumbnail(t *testing.T) {
ctx := context.Background()
seedDir := t.TempDir()
const sourceID int64 = 8888888
realThumb := writeStatusPackWithThumbSeed(t, seedDir, sourceID, 31)
media := newFakeMediaStore()
if err := media.PutDocument(ctx, domain.Document{
ID: sourceID,
MimeType: "application/x-tgsticker",
Thumbs: []domain.PhotoSize{{
Kind: domain.PhotoSizeKindCached, Type: seedSyntheticDocumentThumbType,
W: 1, H: 1, Bytes: append([]byte(nil), seedSyntheticTGStickerPreviewThumbPNG...),
}},
}); err != nil {
t.Fatalf("put stale document: %v", err)
}
if err := media.PutStickerSet(ctx, domain.StickerSet{
ID: 773947703670341676, AccessHash: 1, ShortName: "StatusPack", Title: "Status Pack",
Hash: 31, Kind: domain.StickerSetKindEmoji, Emojis: true, DocumentIDs: []int64{sourceID},
}); err != nil {
t.Fatalf("put stale sticker set: %v", err)
}
blobs, err := NewLocalFS(t.TempDir())
if err != nil {
t.Fatalf("local fs: %v", err)
}
svc := NewService(media, blobs, 2)
stats, err := svc.SeedMedia(ctx, seedDir, 0)
if err != nil {
t.Fatalf("migration seed: %v", err)
}
if stats.StickerSets != 1 || stats.Documents != 1 {
t.Fatalf("migration stats = %+v, want forced sticker document rebuild", stats)
}
doc, ok, err := media.GetDocument(ctx, sourceID)
if err != nil || !ok {
t.Fatalf("migrated document ok=%v err=%v", ok, err)
}
thumb, ok := findCachedThumb(doc.Thumbs)
if !ok || thumb.W != 128 || thumb.H != 128 || !bytes.Equal(thumb.Bytes, realThumb) {
t.Fatalf("migrated thumbs = %+v, want exported 128x128 preview", doc.Thumbs)
}
if state, ok, err := media.GetSeedState(ctx, seedStickerPreviewStateKey); err != nil || !ok || state == "" {
t.Fatalf("preview migration state = %q ok=%v err=%v", state, ok, err)
}
}
@ -735,6 +841,28 @@ func writeStatusPackWithoutThumbSeed(t *testing.T, seedDir string, sourceID int6
}
}
func writeStatusPackWithThumbSeed(t *testing.T, seedDir string, sourceID int64, setHash int) []byte {
t.Helper()
setDir := filepath.Join(seedDir, "telegram_emoji_export", "StatusPack_773947703670341676")
stickersDir := filepath.Join(setDir, "stickers")
if err := os.MkdirAll(stickersDir, 0o755); err != nil {
t.Fatal(err)
}
realThumb := []byte{0xff, 0xd8, 0xff, 0xdb, 0, 4, 0xff, 0xd9}
raw := fmt.Sprintf(`{"result":{"set":{"id":773947703670341676,"access_hash":1,"title":"Status Pack","short_name":"StatusPack","count":1,"hash":%d,"emojis":true,"packs":[{"emoticon":"👋","documents":[%d]}]},"packs":[{"emoticon":"👋","documents":[%d]}],"documents":[{"id":%d,"access_hash":2,"file_reference":"","date":"2026-06-29T00:00:00Z","mime_type":"application/x-tgsticker","size":4,"dc_id":4,"attributes":[{"_":"DocumentAttributeImageSize","w":512,"h":512},{"_":"DocumentAttributeCustomEmoji","alt":"👋","text_color":true,"stickerset":{"id":773947703670341676,"access_hash":1}},{"_":"DocumentAttributeFilename","file_name":"AnimatedSticker.tgs"}],"thumbs":[{"_":"PhotoPathSize","type":"j","bytes":"01"},{"_":"PhotoSize","type":"m","w":128,"h":128,"size":%d}]}]}}`, setHash, sourceID, sourceID, sourceID, len(realThumb))
if err := os.WriteFile(filepath.Join(setDir, "set_info.json"), []byte(raw), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(stickersDir, fmt.Sprintf("status_%d.tgs", sourceID)), []byte("tgs!"), 0o644); err != nil {
t.Fatal(err)
}
thumbName := fmt.Sprintf("status_%d_thumb1_PhotoSize_typem_128x128.jpg", sourceID)
if err := os.WriteFile(filepath.Join(stickersDir, thumbName), realThumb, 0o644); err != nil {
t.Fatal(err)
}
return realThumb
}
func writeEffectsSeed(t *testing.T, seedDir string, sourceID int64) {
t.Helper()
docsDir := filepath.Join(seedDir, "telegram_effects_export", "documents")
@ -992,7 +1120,7 @@ func TestDocumentsNeedInlineCachedThumbsDetectsStaleMime(t *testing.T) {
if err := media.PutDocument(ctx, doc); err != nil {
t.Fatalf("put doc: %v", err)
}
if err := media.PutFileBlob(ctx, domain.FileBlob{LocationKey: "doc:100:m", MimeType: "image/jpeg"}); err != nil {
if err := media.PutFileBlob(ctx, domain.FileBlob{LocationKey: "doc:100:m", Size: int64(len(webp)), MimeType: "image/jpeg"}); err != nil {
t.Fatalf("put blob: %v", err)
}
svc := NewService(media, nil, 2)
@ -1004,7 +1132,7 @@ func TestDocumentsNeedInlineCachedThumbsDetectsStaleMime(t *testing.T) {
t.Fatal("expected stale mime to require repair")
}
if err := media.PutFileBlob(ctx, domain.FileBlob{LocationKey: "doc:100:m", MimeType: "image/webp"}); err != nil {
if err := media.PutFileBlob(ctx, domain.FileBlob{LocationKey: "doc:100:m", Size: int64(len(webp)), MimeType: "image/webp"}); err != nil {
t.Fatalf("put repaired blob: %v", err)
}
stale, err = svc.documentsNeedInlineCachedThumbs(ctx, []int64{doc.ID})