astra/owpengram-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

merged with fixes

Browse source
This commit is contained in:
onysd 2026-09-09 02:49:30 +03:00
parent a9e758b712
commit 2f1818d656
176 changed files with 9000 additions and 907 deletions
Download patch file Download diff file Expand all files Collapse all files

4
internal/app/files/blobcache.go
View file

@ -189,7 +189,9 @@ func (c *blobBytesCache) get(key string) ([]byte, bool) {
if el, ok := c.m[key]; ok {
c.ll.MoveToFront(el)
entry := el.Value.(*blobBytesEntry)
return append([]byte(nil), entry.bytes...), true
// Cache entries are immutable after publication. GetFile returns a
// capacity-clipped read-only view of the requested range.
return entry.bytes, true
}
return nil, false
}

42
internal/app/files/diskspace.go
View file

@ -1,6 +1,11 @@
package files
import "sync/atomic"
import (
"io"
"sync/atomic"
"telesrv/internal/domain"
)
// SpaceGuard bounds how much more may be written to the permanent blob
// backend. LocalDiskSpaceGuard checks real OS free disk bytes;
@ -24,6 +29,41 @@ type NoopSpaceGuard struct{}
func (NoopSpaceGuard) Allow(int64) (bool, error) { return true, nil }
func (NoopSpaceGuard) Usage() (int64, int64, bool) { return 0, 0, false }
// requireSpace maps a SpaceGuard rejection to domain.ErrStorageFull. A nil
// guard always allows the write.
func requireSpace(guard SpaceGuard, additional int64) error {
if guard == nil {
return nil
}
ok, err := guard.Allow(additional)
if err != nil {
return err
}
if !ok {
return domain.ErrStorageFull
}
return nil
}
// capacityReader stops a streaming permanent write before the backend can
// publish an object larger than the current capacity snapshot permits.
type capacityReader struct {
src io.Reader
guard SpaceGuard
total int64
}
func (r *capacityReader) Read(p []byte) (int, error) {
n, err := r.src.Read(p)
if n > 0 {
if guardErr := requireSpace(r.guard, r.total+int64(n)); guardErr != nil {
return 0, guardErr
}
r.total += int64(n)
}
return n, err
}
// LocalDiskSpaceGuard rejects writes once cached free disk bytes fall below
// minFreeBytes (<=0 disables the check). The free-bytes figure is
// refreshed by DiskUsageWorker, not recomputed per call, to avoid a statfs

72
internal/app/files/guarded_backend.go Normal file
View file

@ -0,0 +1,72 @@
package files
import (
"context"
"io"
"time"
)
// GuardedBlobBackend applies one capacity policy to every permanent write,
// including seeds and non-upload media paths, instead of relying on individual
// RPC handlers to remember a check.
type GuardedBlobBackend struct {
backend BlobBackend
guard SpaceGuard
}
func NewGuardedBlobBackend(backend BlobBackend, guard SpaceGuard) *GuardedBlobBackend {
return &GuardedBlobBackend{backend: backend, guard: guard}
}
func (g *GuardedBlobBackend) Name() string { return g.backend.Name() }
func (g *GuardedBlobBackend) Put(ctx context.Context, data []byte) (string, error) {
if err := requireSpace(g.guard, int64(len(data))); err != nil {
return "", err
}
return g.backend.Put(ctx, data)
}
func (g *GuardedBlobBackend) PutReader(ctx context.Context, r io.Reader) (string, int64, []byte, error) {
return g.backend.PutReader(ctx, &capacityReader{src: r, guard: g.guard})
}
func (g *GuardedBlobBackend) Get(ctx context.Context, key string) ([]byte, error) {
return g.backend.Get(ctx, key)
}
func (g *GuardedBlobBackend) GetRange(ctx context.Context, key string, offset, limit int64) ([]byte, int64, error) {
return g.backend.GetRange(ctx, key, offset, limit)
}
type GuardedUploadPartBackend struct {
backend UploadPartBackend
guard SpaceGuard
}
func NewGuardedUploadPartBackend(backend UploadPartBackend, guard SpaceGuard) *GuardedUploadPartBackend {
return &GuardedUploadPartBackend{backend: backend, guard: guard}
}
func (g *GuardedUploadPartBackend) PutUploadPart(ctx context.Context, ownerUserID, fileID int64, part int, data []byte) (uploadPartObject, error) {
if err := requireSpace(g.guard, int64(len(data))); err != nil {
return uploadPartObject{}, err
}
return g.backend.PutUploadPart(ctx, ownerUserID, fileID, part, data)
}
func (g *GuardedUploadPartBackend) GetUploadPart(ctx context.Context, key string) ([]byte, error) {
return g.backend.GetUploadPart(ctx, key)
}
func (g *GuardedUploadPartBackend) OpenUploadPart(ctx context.Context, key string) (io.ReadCloser, error) {
return g.backend.OpenUploadPart(ctx, key)
}
func (g *GuardedUploadPartBackend) DeleteUploadPart(ctx context.Context, key string) error {
return g.backend.DeleteUploadPart(ctx, key)
}
func (g *GuardedUploadPartBackend) DeleteExpiredUploadParts(ctx context.Context, before time.Time, limit int) (int64, error) {
return g.backend.DeleteExpiredUploadParts(ctx, before, limit)
}

49
internal/app/files/photos.go
View file

@ -188,11 +188,51 @@ func (s *Service) CreateAvatarVideoMarkupFromUpload(ctx context.Context, file do
return s.createAvatarVideoFromUpload(ctx, file, videoStartTs, []domain.PhotoSize{markup})
}
// CreateAvatarVideoFromBytes stores already-in-hand animated-video bytes as an
// avatar Photo, for callers that skip the chunked upload.saveFilePart
// transfer regular clients use (e.g. the admin console, which already has the
// full file from a browser upload) -- the video counterpart of
// CreateAvatarFromBytes.
func (s *Service) CreateAvatarVideoFromBytes(ctx context.Context, data []byte, ownerUserID int64, videoStartTs float64) (domain.Photo, error) {
if len(data) == 0 {
return domain.Photo{}, domain.ErrPhotoInvalid
}
objectKey, size, sha256sum, err := s.blobs.PutReader(ctx, bytes.NewReader(data))
if err != nil {
return domain.Photo{}, err
}
if size == 0 {
return domain.Photo{}, domain.ErrPhotoInvalid
}
body := assembledUploadBlob{ObjectKey: objectKey, Size: size, SHA256: sha256sum}
return s.createAvatarVideoFromBlob(ctx, body, ownerUserID, videoStartTs, nil)
}
func (s *Service) createAvatarVideoFromUpload(ctx context.Context, file domain.UploadedFileRef, videoStartTs float64, extraSizes []domain.PhotoSize) (domain.Photo, error) {
body, err := s.assembleUploadBlob(ctx, file.OwnerUserID, file.FileID, file.Parts)
if err != nil {
return domain.Photo{}, err
}
photo, err := s.createAvatarVideoFromBlob(ctx, body, file.OwnerUserID, videoStartTs, extraSizes)
if err != nil {
return domain.Photo{}, err
}
if err := s.cleanupUploadParts(ctx, file.OwnerUserID, file.FileID); err != nil {
s.log.Warn("cleanup assembled avatar video upload parts failed",
zap.Int64("owner_user_id", file.OwnerUserID),
zap.Int64("file_id", file.FileID),
zap.Int64("photo_id", photo.ID),
zap.Error(err))
}
return photo, nil
}
// createAvatarVideoFromBlob turns an already-durable video blob (from either
// the chunked-upload assembly path or a direct in-hand byte slice) into an
// avatar Photo. Shared by createAvatarVideoFromUpload and
// CreateAvatarVideoFromBytes so the still-frame extraction and photo/blob
// record construction stay in exactly one place.
func (s *Service) createAvatarVideoFromBlob(ctx context.Context, body assembledUploadBlob, ownerUserID int64, videoStartTs float64, extraSizes []domain.PhotoSize) (domain.Photo, error) {
if body.Size == 0 {
return domain.Photo{}, domain.ErrPhotoInvalid
}
@ -230,18 +270,11 @@ func (s *Service) createAvatarVideoFromUpload(ctx context.Context, file domain.U
Date: int(time.Now().Unix()),
DCID: s.dc,
Sizes: sizes,
OwnerUserID: file.OwnerUserID,
OwnerUserID: ownerUserID,
}
if err := s.media.PutPhoto(ctx, photo); err != nil {
return domain.Photo{}, err
}
if err := s.cleanupUploadParts(ctx, file.OwnerUserID, file.FileID); err != nil {
s.log.Warn("cleanup assembled avatar video upload parts failed",
zap.Int64("owner_user_id", file.OwnerUserID),
zap.Int64("file_id", file.FileID),
zap.Int64("photo_id", photoID),
zap.Error(err))
}
return photo, nil
}

124
internal/app/files/seed_test.go
View file

@ -15,6 +15,7 @@ import (
"time"
"telesrv/internal/domain"
"telesrv/internal/store"
)
// fakeMediaStore 是 store.MediaStore 的内存替身，用于在无 PG 时验证 seed 导入器。
@ -29,20 +30,27 @@ type fakeMediaStore struct {
webPages map[int64]domain.MessageWebPage
seedState map[string]string
receipts map[string]domain.UploadedMediaReceipt
// profilePhotos[ownerID|kind] 保存某 owner 当前 profile/fallback 照片引用。
profilePhotos map[string]domain.ProfilePhotoRef
}
func newFakeMediaStore() *fakeMediaStore {
return &fakeMediaStore{
blobs: map[string]domain.FileBlob{},
docs: map[int64]domain.Document{},
photos: map[int64]domain.Photo{},
sets: map[int64]domain.StickerSet{},
parts: map[string][]domain.UploadPart{},
seedState: map[string]string{},
receipts: map[string]domain.UploadedMediaReceipt{},
blobs: map[string]domain.FileBlob{},
docs: map[int64]domain.Document{},
photos: map[int64]domain.Photo{},
sets: map[int64]domain.StickerSet{},
parts: map[string][]domain.UploadPart{},
seedState: map[string]string{},
receipts: map[string]domain.UploadedMediaReceipt{},
profilePhotos: map[string]domain.ProfilePhotoRef{},
}
}
func fakeProfilePhotoKey(ownerType domain.PeerType, ownerID int64, kind domain.ProfilePhotoKind) string {
return fmt.Sprintf("%s:%d:%s", ownerType, ownerID, kind)
}
func fakeUploadReceiptKey(ownerUserID, fileID int64) string {
return fmt.Sprintf("%d/%d", ownerUserID, fileID)
}
@ -414,29 +422,101 @@ func (f *fakeMediaStore) CountAvailableReactions(_ context.Context) (int, error)
defer f.mu.Unlock()
return len(f.reactions), nil
}
func (f *fakeMediaStore) AddProfilePhotoKind(_ context.Context, _ domain.PeerType, _ int64, _ domain.ProfilePhotoKind, _ int64, _ int) error {
func (f *fakeMediaStore) AddProfilePhotoKind(_ context.Context, ownerType domain.PeerType, ownerID int64, kind domain.ProfilePhotoKind, photoID int64, date int) error {
f.mu.Lock()
defer f.mu.Unlock()
key := fakeProfilePhotoKey(ownerType, ownerID, kind)
existing := f.profilePhotos[key]
ref := domain.ProfilePhotoRef{PhotoID: photoID}
if p, ok := f.photos[photoID]; ok {
ref.DCID = p.DCID
ref.Stripped = domain.StrippedFromSizes(p.Sizes)
ref.HasVideo = domain.PhotoHasVideo(p.Sizes)
}
if existing.PhotoID != photoID {
f.profilePhotos[key] = ref
}
return nil
}
func (f *fakeMediaStore) CurrentProfilePhotoKind(_ context.Context, _ domain.PeerType, _ int64, _ domain.ProfilePhotoKind) (int64, bool, error) {
return 0, false, nil
func (f *fakeMediaStore) CurrentProfilePhotoKind(_ context.Context, ownerType domain.PeerType, ownerID int64, kind domain.ProfilePhotoKind) (int64, bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
ref, ok := f.profilePhotos[fakeProfilePhotoKey(ownerType, ownerID, kind)]
if !ok {
return 0, false, nil
}
return ref.PhotoID, true, nil
}
func (f *fakeMediaStore) CurrentProfilePhotos(_ context.Context, _ domain.PeerType, _ []int64) (map[int64]domain.ProfilePhotoRef, error) {
return map[int64]domain.ProfilePhotoRef{}, nil
func (f *fakeMediaStore) CurrentProfilePhotos(ctx context.Context, ownerType domain.PeerType, ids []int64) (map[int64]domain.ProfilePhotoRef, error) {
return f.CurrentProfilePhotosKind(ctx, ownerType, ids, domain.ProfilePhotoKindProfile)
}
func (f *fakeMediaStore) CurrentProfilePhotosKind(_ context.Context, _ domain.PeerType, _ []int64, _ domain.ProfilePhotoKind) (map[int64]domain.ProfilePhotoRef, error) {
return map[int64]domain.ProfilePhotoRef{}, nil
func (f *fakeMediaStore) CurrentProfilePhotosKind(_ context.Context, ownerType domain.PeerType, ids []int64, kind domain.ProfilePhotoKind) (map[int64]domain.ProfilePhotoRef, error) {
f.mu.Lock()
defer f.mu.Unlock()
out := make(map[int64]domain.ProfilePhotoRef, len(ids))
for _, id := range ids {
if ref, ok := f.profilePhotos[fakeProfilePhotoKey(ownerType, id, kind)]; ok {
out[id] = ref
}
}
return out, nil
}
func (f *fakeMediaStore) ListProfilePhotosKind(_ context.Context, _ domain.PeerType, _ int64, _ domain.ProfilePhotoKind, _, _ int, _ int64) ([]int64, int, error) {
return nil, 0, nil
func (f *fakeMediaStore) ListProfilePhotosKind(_ context.Context, ownerType domain.PeerType, ownerID int64, kind domain.ProfilePhotoKind, offset, limit int, maxID int64) ([]int64, int, error) {
f.mu.Lock()
defer f.mu.Unlock()
var ids []int64
if ref, ok := f.profilePhotos[fakeProfilePhotoKey(ownerType, ownerID, kind)]; ok {
ids = append(ids, ref.PhotoID)
}
return ids, len(ids), nil
}
func (f *fakeMediaStore) ListProfilePhotoDetailsKind(_ context.Context, _ domain.PeerType, _ int64, _ domain.ProfilePhotoKind, _, _ int, _ int64) ([]domain.Photo, int, error) {
return nil, 0, nil
func (f *fakeMediaStore) ListProfilePhotoDetailsKind(_ context.Context, ownerType domain.PeerType, ownerID int64, kind domain.ProfilePhotoKind, offset, limit int, maxID int64) ([]domain.Photo, int, error) {
f.mu.Lock()
defer f.mu.Unlock()
var out []domain.Photo
if ref, ok := f.profilePhotos[fakeProfilePhotoKey(ownerType, ownerID, kind)]; ok {
if p, ok := f.photos[ref.PhotoID]; ok {
out = append(out, p)
}
}
return out, len(out), nil
}
func (f *fakeMediaStore) DeleteProfilePhotos(_ context.Context, _ domain.PeerType, _ int64, _ []int64) ([]int64, error) {
return nil, nil
func (f *fakeMediaStore) DeleteProfilePhotos(_ context.Context, ownerType domain.PeerType, ownerID int64, photoIDs []int64) ([]int64, error) {
f.mu.Lock()
defer f.mu.Unlock()
var deleted []int64
key := fakeProfilePhotoKey(ownerType, ownerID, domain.ProfilePhotoKindProfile)
if ref, ok := f.profilePhotos[key]; ok {
for _, id := range photoIDs {
if id == ref.PhotoID {
deleted = append(deleted, id)
}
}
}
if len(deleted) > 0 {
delete(f.profilePhotos, key)
}
return deleted, nil
}
func (f *fakeMediaStore) DeleteProfilePhotosKind(_ context.Context, _ domain.PeerType, _ int64, _ domain.ProfilePhotoKind, _ []int64) ([]int64, error) {
return nil, nil
func (f *fakeMediaStore) DeleteProfilePhotosKind(_ context.Context, ownerType domain.PeerType, ownerID int64, kind domain.ProfilePhotoKind, photoIDs []int64) ([]int64, error) {
f.mu.Lock()
defer f.mu.Unlock()
var deleted []int64
key := fakeProfilePhotoKey(ownerType, ownerID, kind)
if ref, ok := f.profilePhotos[key]; ok {
for _, id := range photoIDs {
if id == ref.PhotoID {
deleted = append(deleted, id)
}
}
}
if len(deleted) > 0 {
delete(f.profilePhotos, key)
}
return deleted, nil
}
func (f *fakeMediaStore) WithTx(_ context.Context, fn func(ctx context.Context, txMedia store.MediaStore) error) error {
return fn(context.Background(), f)
}
func TestSeedMediaRepairsPartialReactionBlobs(t *testing.T) {

14
internal/app/files/upload_parts_test.go
View file

@ -256,7 +256,9 @@ func TestMaxUploadFileBytesUnlimitedByDefault(t *testing.T) {
type countingUploadPartBackend struct {
*LocalFS
getUploadPartCalls int
getUploadPartCalls int
putUploadPartCalls int
deleteUploadPartCalls int
}
func (c *countingUploadPartBackend) GetUploadPart(ctx context.Context, objectKey string) ([]byte, error) {
@ -264,6 +266,16 @@ func (c *countingUploadPartBackend) GetUploadPart(ctx context.Context, objectKey
return c.LocalFS.GetUploadPart(ctx, objectKey)
}
func (c *countingUploadPartBackend) PutUploadPart(ctx context.Context, ownerUserID, fileID int64, part int, data []byte) (uploadPartObject, error) {
c.putUploadPartCalls++
return c.LocalFS.PutUploadPart(ctx, ownerUserID, fileID, part, data)
}
func (c *countingUploadPartBackend) DeleteUploadPart(ctx context.Context, objectKey string) error {
c.deleteUploadPartCalls++
return c.LocalFS.DeleteUploadPart(ctx, objectKey)
}
func newUploadPartTestService(t *testing.T, media *fakeMediaStore, quota domain.UploadPartQuota) (*Service, *LocalFS) {
t.Helper()
blobs, err := NewLocalFS(t.TempDir())