files for previous commit
This commit is contained in:
parent
8ef2b58bf9
commit
863ae2e990
7 changed files with 78 additions and 0 deletions
|
|
@ -954,6 +954,7 @@ func run(logger *zap.Logger) error {
|
||||||
filesapp.WithMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes),
|
filesapp.WithMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes),
|
||||||
filesapp.WithStorageRetentionAges(cfg.StorageRetentionMaxAge, cfg.StorageRetentionMaxAgeByCategory, cfg.StorageRetentionMaxAgeAvatar),
|
filesapp.WithStorageRetentionAges(cfg.StorageRetentionMaxAge, cfg.StorageRetentionMaxAgeByCategory, cfg.StorageRetentionMaxAgeAvatar),
|
||||||
filesapp.WithStorageMaxTotalBytes(cfg.StorageMaxTotalBytes),
|
filesapp.WithStorageMaxTotalBytes(cfg.StorageMaxTotalBytes),
|
||||||
|
filesapp.WithSecretChatDeleteFileAfterDownload(cfg.SecretChatDeleteFileAfterDownload),
|
||||||
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
|
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
|
||||||
externalMediaOption(cfg),
|
externalMediaOption(cfg),
|
||||||
webPagePreviewOption(cfg),
|
webPagePreviewOption(cfg),
|
||||||
|
|
@ -1689,6 +1690,7 @@ func run(logger *zap.Logger) error {
|
||||||
Photos: filesService,
|
Photos: filesService,
|
||||||
StickerSets: filesService,
|
StickerSets: filesService,
|
||||||
GifCatalog: filesService,
|
GifCatalog: filesService,
|
||||||
|
Storage: filesService,
|
||||||
Bots: botsService,
|
Bots: botsService,
|
||||||
Emoji: filesService,
|
Emoji: filesService,
|
||||||
Moderation: moderationService,
|
Moderation: moderationService,
|
||||||
|
|
|
||||||
|
|
@ -941,6 +941,10 @@ type FilesService interface {
|
||||||
GetFile(ctx context.Context, req domain.FileDownloadRequest) (domain.FileChunk, bool, error)
|
GetFile(ctx context.Context, req domain.FileDownloadRequest) (domain.FileChunk, bool, error)
|
||||||
// CreateEncryptedFileFromUpload 把密聊上传分片组装成盲 blob 并铸造 EncryptedFile 快照(P2)。
|
// CreateEncryptedFileFromUpload 把密聊上传分片组装成盲 blob 并铸造 EncryptedFile 快照(P2)。
|
||||||
CreateEncryptedFileFromUpload(ctx context.Context, file domain.UploadedFileRef, keyFingerprint int) (domain.EncryptedFileRef, error)
|
CreateEncryptedFileFromUpload(ctx context.Context, file domain.UploadedFileRef, keyFingerprint int) (domain.EncryptedFileRef, error)
|
||||||
|
// DeleteEncryptedFileBlob deletes a secret-chat encrypted file's blob
|
||||||
|
// bytes once fully downloaded (TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD,
|
||||||
|
// no-op otherwise) -- see internal/app/files.Service.DeleteEncryptedFileBlob.
|
||||||
|
DeleteEncryptedFileBlob(ctx context.Context, locationKey string) error
|
||||||
// GeoMapTile 渲染 geo 消息地图缩略占位图(upload.getWebFile),确定性、无外部依赖。
|
// GeoMapTile 渲染 geo 消息地图缩略占位图(upload.getWebFile),确定性、无外部依赖。
|
||||||
GeoMapTile(lat, long float64, w, h, zoom, scale int) ([]byte, string)
|
GeoMapTile(lat, long float64, w, h, zoom, scale int) ([]byte, string)
|
||||||
// 资源读取(reaction / sticker / document)。
|
// 资源读取(reaction / sticker / document)。
|
||||||
|
|
|
||||||
|
|
@ -73,6 +73,9 @@ func (f *fakeFiles) GetFile(_ context.Context, req domain.FileDownloadRequest) (
|
||||||
func (f *fakeFiles) CreateEncryptedFileFromUpload(context.Context, domain.UploadedFileRef, int) (domain.EncryptedFileRef, error) {
|
func (f *fakeFiles) CreateEncryptedFileFromUpload(context.Context, domain.UploadedFileRef, int) (domain.EncryptedFileRef, error) {
|
||||||
return domain.EncryptedFileRef{ID: 9001, AccessHash: 9002, Size: 16, DCID: 2, KeyFingerprint: 7}, nil
|
return domain.EncryptedFileRef{ID: 9001, AccessHash: 9002, Size: 16, DCID: 2, KeyFingerprint: 7}, nil
|
||||||
}
|
}
|
||||||
|
func (f *fakeFiles) DeleteEncryptedFileBlob(context.Context, string) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
func (f *fakeFiles) GeoMapTile(lat, long float64, w, h, zoom, scale int) ([]byte, string) {
|
func (f *fakeFiles) GeoMapTile(lat, long float64, w, h, zoom, scale int) ([]byte, string) {
|
||||||
return []byte{0x89, 'P', 'N', 'G', 0x0D, 0x0A, 0x1A, 0x0A, 1, 2, 3, 4}, "image/png"
|
return []byte{0x89, 'P', 'N', 'G', 0x0D, 0x0A, 0x1A, 0x0A, 1, 2, 3, 4}, "image/png"
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -104,6 +104,19 @@ func (r *Router) onUploadGetFile(ctx context.Context, req *tg.UploadGetFileReque
|
||||||
return nil, internalErr()
|
return nil, internalErr()
|
||||||
}
|
}
|
||||||
if found {
|
if found {
|
||||||
|
if isSecretChatFileFullyDownloaded(key, req.Offset, len(chunk.Bytes), chunk.Total) {
|
||||||
|
// The recipient has now received every byte of this secret-chat
|
||||||
|
// file. Best-effort, fire-and-forget: DeleteEncryptedFileBlob is
|
||||||
|
// itself a no-op unless TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD
|
||||||
|
// is enabled, and must never delay or fail this download response
|
||||||
|
// -- use a detached context since ctx may be canceled the moment
|
||||||
|
// this handler returns.
|
||||||
|
go func(locationKey string) {
|
||||||
|
if err := r.deps.Files.DeleteEncryptedFileBlob(context.Background(), locationKey); err != nil && r.log != nil {
|
||||||
|
r.log.Warn("delete secret chat file after download failed", zap.String("location_key", locationKey), zap.Error(err))
|
||||||
|
}
|
||||||
|
}(key)
|
||||||
|
}
|
||||||
return &tg.UploadFile{
|
return &tg.UploadFile{
|
||||||
Type: storageFileType(chunk.MimeType, chunk.Bytes),
|
Type: storageFileType(chunk.MimeType, chunk.Bytes),
|
||||||
Mtime: 0,
|
Mtime: 0,
|
||||||
|
|
@ -113,6 +126,19 @@ func (r *Router) onUploadGetFile(ctx context.Context, req *tg.UploadGetFileReque
|
||||||
return nil, locationInvalidErr()
|
return nil, locationInvalidErr()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isSecretChatFileFullyDownloaded reports whether a getFile response for
|
||||||
|
// location key key just delivered the last byte of a secret-chat encrypted
|
||||||
|
// file (location_key "enc:<id>") -- the signal onUploadGetFile uses to
|
||||||
|
// fire-and-forget DeleteEncryptedFileBlob. total<=0 (unknown size) never
|
||||||
|
// counts as fully downloaded -- there is nothing to compare against, so
|
||||||
|
// treating that as "done" could delete a file mid-transfer.
|
||||||
|
func isSecretChatFileFullyDownloaded(key string, offset int64, chunkLen int, total int64) bool {
|
||||||
|
if !strings.HasPrefix(key, "enc:") || total <= 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return offset+int64(chunkLen) >= total
|
||||||
|
}
|
||||||
|
|
||||||
// authorizedFileLocationKey applies the authorization/capability checks which cannot be
|
// authorizedFileLocationKey applies the authorization/capability checks which cannot be
|
||||||
// expressed by a plain location-key conversion. Secret-chat blobs are addressed internally by
|
// expressed by a plain location-key conversion. Secret-chat blobs are addressed internally by
|
||||||
// id, but the wire capability is the pair (id, access_hash); accepting id alone would let any
|
// id, but the wire capability is the pair (id, access_hash); accepting id alone would let any
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,37 @@ import (
|
||||||
"telesrv/internal/domain"
|
"telesrv/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// TestIsSecretChatFileFullyDownloaded covers the fire-and-forget
|
||||||
|
// delete-after-download trigger in onUploadGetFile: only an "enc:"-prefixed
|
||||||
|
// (secret-chat) location key, with a known total size, whose chunk reaches
|
||||||
|
// the end of the file, counts as fully downloaded.
|
||||||
|
func TestIsSecretChatFileFullyDownloaded(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
key string
|
||||||
|
offset int64
|
||||||
|
chunkLen int
|
||||||
|
total int64
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"secret chat, last chunk reaches exact end", "enc:123", 900, 100, 1000, true},
|
||||||
|
{"secret chat, last chunk overlaps past end", "enc:123", 950, 100, 1000, true},
|
||||||
|
{"secret chat, still mid-file", "enc:123", 0, 100, 1000, false},
|
||||||
|
{"secret chat, unknown total must never look done", "enc:123", 0, 100, 0, false},
|
||||||
|
{"secret chat, negative total must never look done", "enc:123", 0, 100, -1, false},
|
||||||
|
{"ordinary document key, even if it would otherwise look complete", "doc:123", 900, 100, 1000, false},
|
||||||
|
{"ordinary photo key", "photo:123:x", 0, 1000, 1000, false},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
if got := isSecretChatFileFullyDownloaded(c.key, c.offset, c.chunkLen, c.total); got != c.want {
|
||||||
|
t.Fatalf("isSecretChatFileFullyDownloaded(%q, %d, %d, %d) = %v, want %v",
|
||||||
|
c.key, c.offset, c.chunkLen, c.total, got, c.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFileSaveErrMapsStorageCapacityWithoutFloodWait(t *testing.T) {
|
func TestFileSaveErrMapsStorageCapacityWithoutFloodWait(t *testing.T) {
|
||||||
err := fileSaveErr(domain.ErrStorageFull)
|
err := fileSaveErr(domain.ErrStorageFull)
|
||||||
if !tgerr.Is(err, "STORAGE_FULL") {
|
if !tgerr.Is(err, "STORAGE_FULL") {
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,12 @@ type MediaStore interface {
|
||||||
// 供低磁盘空间守卫的周期性用量刷新使用(尤其是 s3 backend 的预算模式,没有
|
// 供低磁盘空间守卫的周期性用量刷新使用(尤其是 s3 backend 的预算模式,没有
|
||||||
// 操作系统级"剩余空间"概念,只能靠这个累计值和配置的预算比较)。
|
// 操作系统级"剩余空间"概念,只能靠这个累计值和配置的预算比较)。
|
||||||
SumFileBlobBytes(ctx context.Context) (int64, error)
|
SumFileBlobBytes(ctx context.Context) (int64, error)
|
||||||
|
// DeleteFileBlobRow deletes exactly one file_blobs row by its exact
|
||||||
|
// location_key -- no prefix matching (unlike the doc:/photo: sweep
|
||||||
|
// primitives, since callers of this method, e.g. secret-chat
|
||||||
|
// delete-after-download, already know the precise key and it never has
|
||||||
|
// sibling thumbnail/rendition rows to also clean up).
|
||||||
|
DeleteFileBlobRow(ctx context.Context, locationKey string) error
|
||||||
|
|
||||||
// seed 状态。只记录静态资源 catalog 的内容 hash,用于启动时跳过未变化的重复导入;
|
// seed 状态。只记录静态资源 catalog 的内容 hash,用于启动时跳过未变化的重复导入;
|
||||||
// 真实可服务性仍由 documents/file_blobs 校验保证,不能只相信这里的 hash。
|
// 真实可服务性仍由 documents/file_blobs 校验保证,不能只相信这里的 hash。
|
||||||
|
|
|
||||||
|
|
@ -238,6 +238,12 @@ func (s *MediaStore) PutFileBlob(ctx context.Context, blob domain.FileBlob) erro
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DeleteFileBlobRow deletes exactly one file_blobs row by its exact
|
||||||
|
// location_key -- see the store.MediaStore interface doc comment.
|
||||||
|
func (s *MediaStore) DeleteFileBlobRow(ctx context.Context, locationKey string) error {
|
||||||
|
return s.q.DeleteFileBlobRow(ctx, locationKey)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MediaStore) GetFileBlob(ctx context.Context, locationKey string) (domain.FileBlob, bool, error) {
|
func (s *MediaStore) GetFileBlob(ctx context.Context, locationKey string) (domain.FileBlob, bool, error) {
|
||||||
row, err := s.q.GetFileBlob(ctx, locationKey)
|
row, err := s.q.GetFileBlob(ctx, locationKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue