added s3 support
This commit is contained in:
parent
03f10b66ee
commit
45f79148c2
12 changed files with 341 additions and 40 deletions
35
.env.example
35
.env.example
|
|
@ -242,20 +242,29 @@ TELESRV_RATING_ENABLED=true
|
|||
## Storage & Media -- Where uploaded media is stored, and how the server reacts to running low on space.
|
||||
|
||||
# Where uploaded media (photos, documents, stickers) is physically stored.
|
||||
# "localfs" writes to TELESRV_BLOB_DIR on this machine's disk. "s3" writes to
|
||||
# an S3-compatible object store -- self-hosted MinIO or AWS S3 -- configured
|
||||
# below. Switching only affects new uploads; existing files stay wherever
|
||||
# they were written and remain readable.
|
||||
TELESRV_BLOB_BACKEND=localfs
|
||||
# Only used when TELESRV_BLOB_BACKEND=s3.
|
||||
TELESRV_S3_ENDPOINT=
|
||||
# "s3" (the default) writes to an S3-compatible object store -- out of the
|
||||
# box that's the self-hosted MinIO container started by
|
||||
# deploy/docker-compose.yml (see its "minio" service), pre-configured below
|
||||
# to just work; point TELESRV_S3_* at AWS S3 instead if you'd rather not
|
||||
# self-host it. "localfs" writes to TELESRV_BLOB_DIR on this machine's disk
|
||||
# instead -- fully supported, just not the default. Switching backends only
|
||||
# affects new uploads: existing files stay wherever they were written and
|
||||
# remain readable/deletable as long as that backend's settings below stay
|
||||
# filled in (don't blank out the old backend's config right after switching
|
||||
# away from it, or its files become unreachable).
|
||||
TELESRV_BLOB_BACKEND=s3
|
||||
# Only used when TELESRV_BLOB_BACKEND=s3 (or after switching away from s3,
|
||||
# for as long as old s3-stored files still need to stay reachable).
|
||||
# Defaults below match deploy/docker-compose.yml's "minio" service exactly.
|
||||
TELESRV_S3_ENDPOINT=127.0.0.1:9000
|
||||
TELESRV_S3_REGION=us-east-1
|
||||
TELESRV_S3_BUCKET=
|
||||
TELESRV_S3_ACCESS_KEY_ID=
|
||||
TELESRV_S3_SECRET_ACCESS_KEY=
|
||||
TELESRV_S3_USE_SSL=true
|
||||
# MinIO typically needs this on (bucket in the URL path); AWS S3 does not.
|
||||
TELESRV_S3_PATH_STYLE=false
|
||||
TELESRV_S3_BUCKET=owpengram-media
|
||||
TELESRV_S3_ACCESS_KEY_ID=owpengram
|
||||
TELESRV_S3_SECRET_ACCESS_KEY=owpengram123
|
||||
# Local MinIO runs plain HTTP; set to true for AWS S3 or a MinIO behind TLS.
|
||||
TELESRV_S3_USE_SSL=false
|
||||
# MinIO needs this on (bucket in the URL path); AWS S3 does not.
|
||||
TELESRV_S3_PATH_STYLE=true
|
||||
# Reject new uploads once storage is nearly full, instead of letting the disk
|
||||
# fill up. Thresholds live in the Advanced section below.
|
||||
TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE=true
|
||||
|
|
|
|||
|
|
@ -2609,7 +2609,7 @@ SELECT count(DISTINCT owner_user_id)::bigint FROM (`+perOwnerMediaSizeSQL+`) x W
|
|||
}
|
||||
stats.BackendKind = strings.ToLower(strings.TrimSpace(os.Getenv("TELESRV_BLOB_BACKEND")))
|
||||
if stats.BackendKind == "" {
|
||||
stats.BackendKind = "localfs"
|
||||
stats.BackendKind = "s3"
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -718,14 +718,33 @@ func run(logger *zap.Logger) error {
|
|||
if err != nil {
|
||||
return fmt.Errorf("init local blob dir: %w", err)
|
||||
}
|
||||
// Construct the s3 backend whenever it's configured, even if it isn't
|
||||
// the active one right now: this is what keeps files readable/deletable
|
||||
// after switching TELESRV_BLOB_BACKEND away from s3, as long as the old
|
||||
// bucket/credentials are still reachable. Only a genuinely active s3
|
||||
// backend failing to initialize is fatal to boot.
|
||||
var s3Backend *filesapp.S3FS
|
||||
if cfg.S3Endpoint != "" && cfg.S3Bucket != "" {
|
||||
b, err := filesapp.NewS3FS(ctx, cfg.S3Endpoint, cfg.S3AccessKeyID, cfg.S3SecretAccessKey, cfg.S3Bucket, cfg.S3Region, cfg.S3UseSSL, cfg.S3PathStyle)
|
||||
switch {
|
||||
case err != nil && cfg.BlobBackendKind == "s3":
|
||||
return fmt.Errorf("init s3 blob backend: %w", err)
|
||||
case err != nil:
|
||||
logger.Warn("s3 blob backend configured but failed to initialize; files previously written to s3 will be unreadable until this is fixed",
|
||||
zap.String("endpoint", cfg.S3Endpoint), zap.String("bucket", cfg.S3Bucket), zap.Error(err))
|
||||
default:
|
||||
s3Backend = b
|
||||
}
|
||||
}
|
||||
var blobBackend filesapp.BlobBackend = localBlobFS
|
||||
var additionalBlobBackend filesapp.BlobBackend
|
||||
var spaceGuard filesapp.SpaceGuard = filesapp.NoopSpaceGuard{}
|
||||
if cfg.BlobBackendKind == "s3" {
|
||||
s3Backend, err := filesapp.NewS3FS(ctx, cfg.S3Endpoint, cfg.S3AccessKeyID, cfg.S3SecretAccessKey, cfg.S3Bucket, cfg.S3Region, cfg.S3UseSSL, cfg.S3PathStyle)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init s3 blob backend: %w", err)
|
||||
if s3Backend == nil {
|
||||
return fmt.Errorf("s3 blob backend not initialized (check TELESRV_S3_* config)")
|
||||
}
|
||||
blobBackend = s3Backend
|
||||
additionalBlobBackend = localBlobFS
|
||||
logger.Info("blob backend ready", zap.String("backend", "s3"), zap.String("bucket", cfg.S3Bucket), zap.String("endpoint", cfg.S3Endpoint))
|
||||
if cfg.StorageLowSpaceGuardEnable && cfg.StorageMaxTotalBytes > 0 {
|
||||
s3Guard := filesapp.NewS3BudgetSpaceGuard(cfg.StorageMaxTotalBytes)
|
||||
|
|
@ -734,6 +753,11 @@ func run(logger *zap.Logger) error {
|
|||
}
|
||||
} else {
|
||||
logger.Info("blob backend ready", zap.String("backend", "localfs"), zap.String("dir", cfg.BlobDir))
|
||||
if s3Backend != nil {
|
||||
additionalBlobBackend = s3Backend
|
||||
logger.Info("s3 blob backend also configured; kept reachable for files written while it was the active backend",
|
||||
zap.String("bucket", cfg.S3Bucket))
|
||||
}
|
||||
if cfg.StorageLowSpaceGuardEnable && cfg.StorageMinFreeBytes > 0 {
|
||||
localGuard := filesapp.NewLocalDiskSpaceGuard(cfg.StorageMinFreeBytes)
|
||||
spaceGuard = localGuard
|
||||
|
|
@ -748,6 +772,7 @@ func run(logger *zap.Logger) error {
|
|||
MaxFiles: cfg.UploadInFlightMaxFiles,
|
||||
}),
|
||||
filesapp.WithUploadPartBackend(localBlobFS),
|
||||
filesapp.WithAdditionalBlobBackend(additionalBlobBackend),
|
||||
filesapp.WithSpaceGuard(spaceGuard),
|
||||
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
|
||||
externalMediaOption(cfg),
|
||||
|
|
|
|||
|
|
@ -66,8 +66,32 @@ services:
|
|||
retries: 10
|
||||
restart: unless-stopped
|
||||
|
||||
# S3 兼容对象存储,默认 blob backend(见 .env.example 的 TELESRV_BLOB_BACKEND=s3)。
|
||||
# 桶由 telesrv 自己在启动时按需创建(internal/app/files/blobs3.go 的 NewS3FS),
|
||||
# 这里不需要额外的 mc 初始化容器。控制台(9001)仅用于本地调试查看已存对象。
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
container_name: ${TELESRV_DOCKER_PREFIX:-owpengram}-minio
|
||||
command: ["server", "/data", "--console-address", ":9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: owpengram
|
||||
MINIO_ROOT_PASSWORD: owpengram123
|
||||
ports:
|
||||
- "9000:9000" # S3 API(对应 .env.example 的 TELESRV_S3_ENDPOINT=localhost:9000)
|
||||
- "9001:9001" # Web 控制台,浏览器打开 http://localhost:9001 查看存储内容
|
||||
volumes:
|
||||
- miniodata:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
name: ${TELESRV_DOCKER_PREFIX:-owpengram}_pgdata
|
||||
redisdata:
|
||||
name: ${TELESRV_DOCKER_PREFIX:-owpengram}_redisdata
|
||||
miniodata:
|
||||
name: ${TELESRV_DOCKER_PREFIX:-owpengram}_miniodata
|
||||
|
|
|
|||
173
internal/app/files/backend_switch_test.go
Normal file
173
internal/app/files/backend_switch_test.go
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// fakeNamedBackend is a minimal BlobBackend double with a settable Name(),
|
||||
// used to prove Service.backendFor's routing logic without needing two real
|
||||
// backend implementations (LocalFS always reports "localfs").
|
||||
type fakeNamedBackend struct {
|
||||
BlobBackend
|
||||
name string
|
||||
}
|
||||
|
||||
func (f fakeNamedBackend) Name() string { return f.name }
|
||||
|
||||
// TestBackendForRoutesToTheBackendThatActuallyWroteTheBlob proves the core
|
||||
// safety property behind switching TELESRV_BLOB_BACKEND: a row whose
|
||||
// recorded backend differs from the currently active one must still
|
||||
// resolve to the backend that actually holds its bytes (via
|
||||
// WithAdditionalBlobBackend), not silently fall through to the active one
|
||||
// (which would 404 -- the row/bytes still exist, but the code would be
|
||||
// looking in the wrong place).
|
||||
func TestBackendForRoutesToTheBackendThatActuallyWroteTheBlob(t *testing.T) {
|
||||
active := fakeNamedBackend{name: "s3"}
|
||||
old := fakeNamedBackend{name: "localfs"}
|
||||
|
||||
s := NewService(nil, active, 2, WithAdditionalBlobBackend(old))
|
||||
|
||||
got, err := s.backendFor(domain.MediaBackendS3)
|
||||
if err != nil || got.Name() != "s3" {
|
||||
t.Fatalf("resolve active backend: got %v err=%v, want s3", got, err)
|
||||
}
|
||||
got, err = s.backendFor(domain.MediaBackendLocalFS)
|
||||
if err != nil || got.Name() != "localfs" {
|
||||
t.Fatalf("resolve registered old backend: got %v err=%v, want localfs", got, err)
|
||||
}
|
||||
// A row predating the owner/backend column (empty string) must not be
|
||||
// treated as unreachable -- it falls back to the active backend, which
|
||||
// is where every pre-migration row's bytes actually are.
|
||||
got, err = s.backendFor("")
|
||||
if err != nil || got.Name() != "s3" {
|
||||
t.Fatalf("resolve empty/legacy backend: got %v err=%v, want active s3", got, err)
|
||||
}
|
||||
// A backend that was never configured (e.g. its config was wiped after
|
||||
// switching away from it) must fail loudly, not silently read from the
|
||||
// wrong place.
|
||||
if _, err := s.backendFor("gcs"); err == nil {
|
||||
t.Fatal("expected an error resolving an unconfigured backend, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
// TestServiceGetFileReadsFromRegisteredOldBackendAfterSwitch is the
|
||||
// end-to-end version: a Service whose active backend is A must still serve
|
||||
// a file whose file_blobs row says it lives on B, as long as B was
|
||||
// registered via WithAdditionalBlobBackend -- exactly the "I switched
|
||||
// TELESRV_BLOB_BACKEND, did my old files disappear?" scenario.
|
||||
func TestServiceGetFileReadsFromRegisteredOldBackendAfterSwitch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
oldBackend, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("new old backend: %v", err)
|
||||
}
|
||||
newBackend, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("new active backend: %v", err)
|
||||
}
|
||||
// Give the "old" backend a distinct name so backendFor can tell them
|
||||
// apart, the way a real second BlobBackend implementation would.
|
||||
oldNamed := fakeNamedBackend{BlobBackend: oldBackend, name: "old-backend"}
|
||||
|
||||
data := []byte("file written before the backend switch")
|
||||
objectKey, err := oldBackend.Put(ctx, data)
|
||||
if err != nil {
|
||||
t.Fatalf("write to old backend: %v", err)
|
||||
}
|
||||
|
||||
media := newFakeMediaStore()
|
||||
if err := media.PutFileBlob(ctx, domain.FileBlob{
|
||||
LocationKey: "doc:1",
|
||||
Backend: "old-backend",
|
||||
ObjectKey: objectKey,
|
||||
Size: int64(len(data)),
|
||||
}); err != nil {
|
||||
t.Fatalf("put file blob metadata: %v", err)
|
||||
}
|
||||
|
||||
// Active backend is the NEW one; the old one is only reachable via the
|
||||
// registry -- mirrors production after a TELESRV_BLOB_BACKEND switch.
|
||||
s := NewService(media, newBackend, 2, WithAdditionalBlobBackend(oldNamed))
|
||||
|
||||
chunk, found, err := s.GetFile(ctx, domain.FileDownloadRequest{LocationKey: "doc:1", Limit: 1 << 20})
|
||||
if err != nil {
|
||||
t.Fatalf("get file after backend switch: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("file written before the switch was not found -- it would appear to have vanished")
|
||||
}
|
||||
if string(chunk.Bytes) != string(data) {
|
||||
t.Fatalf("chunk = %q, want %q", chunk.Bytes, data)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBackendSwitchRoundTripsThroughRealMinIO exercises the actual
|
||||
// scenario the user asked about: files written while s3 (MinIO) was active
|
||||
// stay readable after switching to localfs, and files written while
|
||||
// localfs was active stay readable after switching to s3 -- both directions,
|
||||
// against a real MinIO instance (deploy/docker-compose.yml's "minio"
|
||||
// service). Skips if MinIO isn't reachable at the default dev endpoint.
|
||||
func TestBackendSwitchRoundTripsThroughRealMinIO(t *testing.T) {
|
||||
endpoint := os.Getenv("TELESRV_TEST_S3_ENDPOINT")
|
||||
if endpoint == "" {
|
||||
endpoint = "localhost:9000"
|
||||
}
|
||||
ctx := context.Background()
|
||||
s3, err := NewS3FS(ctx, endpoint, "owpengram", "owpengram123", "telesrv-test-backend-switch", "us-east-1", false, true)
|
||||
if err != nil {
|
||||
t.Skipf("minio not reachable at %s (start deploy/docker-compose.yml's minio service to run this test): %v", endpoint, err)
|
||||
}
|
||||
local, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("new local backend: %v", err)
|
||||
}
|
||||
|
||||
t.Run("s3_active_localfs_switch", func(t *testing.T) {
|
||||
s3Data := []byte("written while s3 was the active backend")
|
||||
s3Key, err := s3.Put(ctx, s3Data)
|
||||
if err != nil {
|
||||
t.Fatalf("write to s3: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s3.Delete(ctx, s3Key) })
|
||||
|
||||
media := newFakeMediaStore()
|
||||
if err := media.PutFileBlob(ctx, domain.FileBlob{
|
||||
LocationKey: "doc:1", Backend: domain.MediaBackendS3, ObjectKey: s3Key, Size: int64(len(s3Data)),
|
||||
}); err != nil {
|
||||
t.Fatalf("put file blob: %v", err)
|
||||
}
|
||||
|
||||
// Switched: localfs is now active, s3 kept reachable as the old backend.
|
||||
s := NewService(media, local, 2, WithAdditionalBlobBackend(s3))
|
||||
chunk, found, err := s.GetFile(ctx, domain.FileDownloadRequest{LocationKey: "doc:1", Limit: 1 << 20})
|
||||
if err != nil || !found || string(chunk.Bytes) != string(s3Data) {
|
||||
t.Fatalf("read s3-written file after switching to localfs: found=%v err=%v bytes=%q, want %q", found, err, chunk.Bytes, s3Data)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("localfs_active_s3_switch", func(t *testing.T) {
|
||||
localData := []byte("written while localfs was the active backend")
|
||||
localKey, err := local.Put(ctx, localData)
|
||||
if err != nil {
|
||||
t.Fatalf("write to localfs: %v", err)
|
||||
}
|
||||
|
||||
media := newFakeMediaStore()
|
||||
if err := media.PutFileBlob(ctx, domain.FileBlob{
|
||||
LocationKey: "doc:2", Backend: domain.MediaBackendLocalFS, ObjectKey: localKey, Size: int64(len(localData)),
|
||||
}); err != nil {
|
||||
t.Fatalf("put file blob: %v", err)
|
||||
}
|
||||
|
||||
// Switched: s3 is now active, localfs kept reachable as the old backend.
|
||||
s := NewService(media, s3, 2, WithAdditionalBlobBackend(local))
|
||||
chunk, found, err := s.GetFile(ctx, domain.FileDownloadRequest{LocationKey: "doc:2", Limit: 1 << 20})
|
||||
if err != nil || !found || string(chunk.Bytes) != string(localData) {
|
||||
t.Fatalf("read localfs-written file after switching to s3: found=%v err=%v bytes=%q, want %q", found, err, chunk.Bytes, localData)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -34,7 +34,11 @@ func (s *Service) DocumentAnimationJSON(ctx context.Context, documentID int64) (
|
|||
if !found || blob.Size <= 0 || blob.Size > maxEmojiAnimationBytes {
|
||||
return nil, false, nil
|
||||
}
|
||||
data, total, err := s.blobs.GetRange(ctx, blob.ObjectKey, 0, blob.Size)
|
||||
backend, err := s.backendFor(blob.Backend)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
data, total, err := backend.GetRange(ctx, blob.ObjectKey, 0, blob.Size)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1096,7 +1096,12 @@ func (s *Service) readSmallBlob(ctx context.Context, locationKey string, expecte
|
|||
if !found || blob.Size <= 0 || blob.Size > avatarMarkupMaxSourceBytes {
|
||||
return nil, false
|
||||
}
|
||||
data, total, err := s.blobs.GetRange(ctx, blob.ObjectKey, 0, blob.Size)
|
||||
backend, err := s.backendFor(blob.Backend)
|
||||
if err != nil {
|
||||
s.log.Warn("resolve avatar markup blob backend failed", zap.String("location_key", locationKey), zap.Error(err))
|
||||
return nil, false
|
||||
}
|
||||
data, total, err := backend.GetRange(ctx, blob.ObjectKey, 0, blob.Size)
|
||||
if err != nil {
|
||||
s.log.Warn("read avatar markup blob failed",
|
||||
zap.String("location_key", locationKey),
|
||||
|
|
|
|||
|
|
@ -67,11 +67,13 @@ func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time,
|
|||
|
||||
// deleteOrphanedBlobs removes each blob from its backend once confirming
|
||||
// (via CountFileBlobRefs) no other file_blobs row still references
|
||||
// (backend, object_key). Only blobs on the currently active backend are
|
||||
// physically removed -- a blob left over from a previously active backend
|
||||
// (deployment switched TELESRV_BLOB_BACKEND at some point; switching back
|
||||
// isn't supported) is logged and skipped rather than silently dropped,
|
||||
// since there's no configured client to reach it right now anyway.
|
||||
// (backend, object_key). Resolves the correct backend per blob via
|
||||
// backendFor (not just the currently active one) -- a blob written before a
|
||||
// TELESRV_BLOB_BACKEND switch still needs deleting from wherever it
|
||||
// actually lives. If that backend is no longer configured (its credentials
|
||||
// were removed after switching away from it), the blob is logged and
|
||||
// skipped rather than silently dropped, since there's nothing reachable to
|
||||
// delete it from.
|
||||
func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionStore, blobs []domain.FileBlob) {
|
||||
for _, b := range blobs {
|
||||
refs, err := store.CountFileBlobRefs(ctx, string(b.Backend), b.ObjectKey)
|
||||
|
|
@ -82,12 +84,13 @@ func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionS
|
|||
if refs > 0 {
|
||||
continue
|
||||
}
|
||||
if string(b.Backend) != s.blobs.Name() {
|
||||
s.log.Warn("orphaned blob is on an inactive backend, skipping physical delete",
|
||||
zap.String("backend", string(b.Backend)), zap.String("object_key", b.ObjectKey))
|
||||
backend, err := s.backendFor(b.Backend)
|
||||
if err != nil {
|
||||
s.log.Warn("orphaned blob's backend is not configured, skipping physical delete",
|
||||
zap.String("backend", string(b.Backend)), zap.String("object_key", b.ObjectKey), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if err := s.blobs.Delete(ctx, b.ObjectKey); err != nil {
|
||||
if err := backend.Delete(ctx, b.ObjectKey); err != nil {
|
||||
s.log.Warn("delete orphaned blob failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,7 +48,14 @@ const (
|
|||
type Service struct {
|
||||
media store.MediaStore
|
||||
blobs BlobBackend
|
||||
uploadParts UploadPartBackend
|
||||
// otherBackends holds additional, non-active BlobBackend instances keyed
|
||||
// by Name() (e.g. "localfs" while s3 is active, or vice versa). Only
|
||||
// used for reading/deleting rows written before the deployment switched
|
||||
// TELESRV_BLOB_BACKEND -- new writes always go through blobs. Without
|
||||
// this, a backend switch would make every pre-switch file unreadable
|
||||
// (the row/bytes still exist, but nothing would know how to reach them).
|
||||
otherBackends map[string]BlobBackend
|
||||
uploadParts UploadPartBackend
|
||||
dc int
|
||||
log *zap.Logger
|
||||
thumbs VideoThumbnailer
|
||||
|
|
@ -127,6 +134,23 @@ func WithSpaceGuard(guard SpaceGuard) Option {
|
|||
}
|
||||
}
|
||||
|
||||
// WithAdditionalBlobBackend registers a non-active blob backend purely for
|
||||
// reading/deleting rows written while it used to be active. Register the
|
||||
// previous backend here whenever TELESRV_BLOB_BACKEND changes and the old
|
||||
// backend's storage/credentials are still reachable -- otherwise every file
|
||||
// written before the switch becomes unreadable (see Service.otherBackends).
|
||||
func WithAdditionalBlobBackend(backend BlobBackend) Option {
|
||||
return func(s *Service) {
|
||||
if backend == nil {
|
||||
return
|
||||
}
|
||||
if s.otherBackends == nil {
|
||||
s.otherBackends = make(map[string]BlobBackend, 1)
|
||||
}
|
||||
s.otherBackends[backend.Name()] = backend
|
||||
}
|
||||
}
|
||||
|
||||
// WithUploadPartBackend overrides where transient upload-part chunks are
|
||||
// staged before assembly, independent of the permanent blob backend passed
|
||||
// to NewService. Needed when the permanent backend is s3 (S3FS doesn't
|
||||
|
|
@ -190,6 +214,22 @@ func NewService(media store.MediaStore, blobs BlobBackend, dc int, opts ...Optio
|
|||
return s
|
||||
}
|
||||
|
||||
// backendFor resolves the BlobBackend that actually holds a given blob row,
|
||||
// which is not necessarily the currently active s.blobs -- a row written
|
||||
// before a TELESRV_BLOB_BACKEND switch stays on whichever backend wrote it.
|
||||
// Falls back to the active backend for an empty/unset name (older rows
|
||||
// predating this field) rather than failing a previously-working read.
|
||||
func (s *Service) backendFor(backend domain.MediaBackend) (BlobBackend, error) {
|
||||
name := string(backend)
|
||||
if name == "" || name == s.blobs.Name() {
|
||||
return s.blobs, nil
|
||||
}
|
||||
if b, ok := s.otherBackends[name]; ok {
|
||||
return b, nil
|
||||
}
|
||||
return nil, fmt.Errorf("blob backend %q is not configured (was TELESRV_BLOB_BACKEND changed without keeping the old backend reachable?)", name)
|
||||
}
|
||||
|
||||
// SaveFilePart 累积一个 small file 分片。
|
||||
func (s *Service) SaveFilePart(ctx context.Context, ownerUserID, fileID int64, part int, bytes []byte) (bool, error) {
|
||||
if err := validatePart(part, len(bytes)); err != nil {
|
||||
|
|
@ -390,6 +430,10 @@ func (s *Service) GetFile(ctx context.Context, req domain.FileDownloadRequest) (
|
|||
}
|
||||
blob = res.blob
|
||||
}
|
||||
backend, err := s.backendFor(blob.Backend)
|
||||
if err != nil {
|
||||
return domain.FileChunk{}, false, err
|
||||
}
|
||||
if blob.Size > 0 && blob.Size <= blobBytesCacheMaxEntryBytes {
|
||||
cacheLog.byteCacheEligible = true
|
||||
if data, ok := s.byteCache.get(blob.ObjectKey); ok {
|
||||
|
|
@ -406,7 +450,7 @@ func (s *Service) GetFile(ctx context.Context, req domain.FileDownloadRequest) (
|
|||
if cached, ok := s.byteCache.get(blob.ObjectKey); ok {
|
||||
return blobBytesResult{data: cached, total: int64(len(cached)), cacheable: true, cacheHit: true}, nil
|
||||
}
|
||||
data, total, err := s.blobs.GetRange(ctx, blob.ObjectKey, 0, blobBytesCacheMaxEntryBytes+1)
|
||||
data, total, err := backend.GetRange(ctx, blob.ObjectKey, 0, blobBytesCacheMaxEntryBytes+1)
|
||||
if err != nil {
|
||||
return blobBytesResult{}, err
|
||||
}
|
||||
|
|
@ -443,7 +487,7 @@ func (s *Service) GetFile(ctx context.Context, req domain.FileDownloadRequest) (
|
|||
if cacheLog.source == "unknown" {
|
||||
cacheLog.source = "backend_range"
|
||||
}
|
||||
data, total, err := s.blobs.GetRange(ctx, blob.ObjectKey, req.Offset, int64(req.Limit))
|
||||
data, total, err := backend.GetRange(ctx, blob.ObjectKey, req.Offset, int64(req.Limit))
|
||||
if err != nil {
|
||||
return domain.FileChunk{}, false, fmt.Errorf("read blob %q: %w", blob.LocationKey, err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -404,7 +404,11 @@ func (s *Service) readStickerMaterialBlob(ctx context.Context, doc domain.Docume
|
|||
if err != nil || !found || blob.Size <= 0 || blob.Size > domain.MaxStickerMaterialDocumentSize {
|
||||
return nil, false
|
||||
}
|
||||
data, total, err := s.blobs.GetRange(ctx, blob.ObjectKey, 0, blob.Size)
|
||||
backend, err := s.backendFor(blob.Backend)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
data, total, err := backend.GetRange(ctx, blob.ObjectKey, 0, blob.Size)
|
||||
if err != nil || int64(len(data)) != total || total != blob.Size {
|
||||
return nil, false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -121,7 +121,14 @@ func (s *Service) warmBlobBytes(ctx context.Context, blob domain.FileBlob) (bool
|
|||
if blob.Size <= 0 || blob.Size > blobBytesCacheMaxEntryBytes || s.byteCache.has(blob.ObjectKey) {
|
||||
return false, nil
|
||||
}
|
||||
data, total, err := s.blobs.GetRange(ctx, blob.ObjectKey, 0, blobBytesCacheMaxEntryBytes+1)
|
||||
backend, err := s.backendFor(blob.Backend)
|
||||
if err != nil {
|
||||
// Warmup is a pure optimization; a blob left on a backend that's no
|
||||
// longer configured just stays uncached here -- GetFile's own
|
||||
// backendFor call surfaces the real error if it's ever requested.
|
||||
return false, nil
|
||||
}
|
||||
data, total, err := backend.GetRange(ctx, blob.ObjectKey, 0, blobBytesCacheMaxEntryBytes+1)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read blob %q: %w", blob.LocationKey, err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -795,14 +795,17 @@ func Load() (Config, error) {
|
|||
OfficialGiftsDir: envOr("TELESRV_OFFICIAL_GIFTS_DIR", "data/official-gifts"),
|
||||
StarGiftTONStartingGrant: envInt64Or("TELESRV_STARGIFT_TON_STARTING_GRANT", 10_000_000_000),
|
||||
BlobDir: envOr("TELESRV_BLOB_DIR", "data/blobs"),
|
||||
BlobBackendKind: strings.ToLower(strings.TrimSpace(envOr("TELESRV_BLOB_BACKEND", "localfs"))),
|
||||
S3Endpoint: envOr("TELESRV_S3_ENDPOINT", ""),
|
||||
// s3 (MinIO by default, see deploy/docker-compose.yml's minio service) is
|
||||
// the default blob backend; localfs remains fully supported as an
|
||||
// explicit opt-in (TELESRV_BLOB_BACKEND=localfs).
|
||||
BlobBackendKind: strings.ToLower(strings.TrimSpace(envOr("TELESRV_BLOB_BACKEND", "s3"))),
|
||||
S3Endpoint: envOr("TELESRV_S3_ENDPOINT", "127.0.0.1:9000"), // 同理避开 localhost→IPv6 回退延迟
|
||||
S3Region: envOr("TELESRV_S3_REGION", "us-east-1"),
|
||||
S3Bucket: envOr("TELESRV_S3_BUCKET", ""),
|
||||
S3AccessKeyID: envOr("TELESRV_S3_ACCESS_KEY_ID", ""),
|
||||
S3SecretAccessKey: envOr("TELESRV_S3_SECRET_ACCESS_KEY", ""),
|
||||
S3UseSSL: envBoolOr("TELESRV_S3_USE_SSL", true),
|
||||
S3PathStyle: envBoolOr("TELESRV_S3_PATH_STYLE", false),
|
||||
S3Bucket: envOr("TELESRV_S3_BUCKET", "owpengram-media"),
|
||||
S3AccessKeyID: envOr("TELESRV_S3_ACCESS_KEY_ID", "owpengram"),
|
||||
S3SecretAccessKey: envOr("TELESRV_S3_SECRET_ACCESS_KEY", "owpengram123"),
|
||||
S3UseSSL: envBoolOr("TELESRV_S3_USE_SSL", false),
|
||||
S3PathStyle: envBoolOr("TELESRV_S3_PATH_STYLE", true),
|
||||
StorageLowSpaceGuardEnable: envBoolOr("TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE", true),
|
||||
StorageMinFreeBytes: envInt64Or("TELESRV_STORAGE_MIN_FREE_BYTES", 1<<30),
|
||||
StorageMaxTotalBytes: envInt64Or("TELESRV_STORAGE_MAX_TOTAL_BYTES", 0),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue