diff --git a/.env.example b/.env.example index d5b65f94..c2d07bff 100644 --- a/.env.example +++ b/.env.example @@ -404,6 +404,12 @@ TELESRV_STICKER_SEED_DIR=data/sticker-seed TELESRV_STICKER_SEED_MAX_SETS=300 # Sticker set auto-installed for every newly registered account; <=0 disables this. TELESRV_DEFAULT_STICKER_SET_ID=0 +# Drop plain .gif/.mp4 files here and they're imported into the admin-curated +# GIF catalog on startup (@gif serves it in the client's GIF picker) -- no +# export manifest needed, unlike TELESRV_STICKER_SEED_DIR above. Each file is +# imported once (matched by filename); renaming a file re-imports it as a new +# entry. Missing directory is skipped, not an error. +TELESRV_GIF_SEED_DIR=data/gifs # Storage low-space guard thresholds (master toggle is TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE above). # localfs: reject new uploads once real free disk bytes fall below this; <=0 disables. diff --git a/cmd/telesrv/main.go b/cmd/telesrv/main.go index 2490298e..87a705c7 100644 --- a/cmd/telesrv/main.go +++ b/cmd/telesrv/main.go @@ -798,6 +798,16 @@ func run(logger *zap.Logger) error { zap.Int("blobs", stats.Blobs), ) } + if stats, err := filesService.SeedGifs(ctx, cfg.GifSeedDir); err != nil { + return fmt.Errorf("seed gifs: %w", err) + } else if stats.Imported > 0 || stats.Failed > 0 { + logger.Info("gif catalog seed import complete", + zap.String("dir", cfg.GifSeedDir), + zap.Int("imported", stats.Imported), + zap.Int("skipped", stats.Skipped), + zap.Int("failed", stats.Failed), + ) + } if stats, err := filesService.SeedPremiumPromo(ctx, cfg.PremiumPromoSeedDir); err != nil { return fmt.Errorf("seed premium promo: %w", err) } else if !stats.Skipped { diff --git a/deploy/migrations/20260807140000_gif_catalog_seed_filename.down.sql b/deploy/migrations/20260807140000_gif_catalog_seed_filename.down.sql new file mode 100644 index 00000000..8b9924b8 --- /dev/null +++ b/deploy/migrations/20260807140000_gif_catalog_seed_filename.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS public.gif_catalog_source_filename_uniq; +ALTER TABLE public.gif_catalog DROP COLUMN IF EXISTS source_filename; diff --git a/deploy/migrations/20260807140000_gif_catalog_seed_filename.up.sql b/deploy/migrations/20260807140000_gif_catalog_seed_filename.up.sql new file mode 100644 index 00000000..88f52ee3 --- /dev/null +++ b/deploy/migrations/20260807140000_gif_catalog_seed_filename.up.sql @@ -0,0 +1,12 @@ +-- Tracks which gif_catalog entries came from a filesystem seed drop +-- (data/gifs/, see files.Service.SeedGifs) vs. an admin-panel upload, so a +-- restart can tell "already imported this file" from "brand new file" without +-- re-transcoding every GIF on every startup. Empty for panel-created entries. +ALTER TABLE public.gif_catalog ADD COLUMN source_filename text DEFAULT ''::text NOT NULL; + +-- Only seed-imported rows need uniqueness here: two panel uploads may +-- legitimately share nothing to compare, but two seed imports of the same +-- filename must collapse to one row on every restart. +CREATE UNIQUE INDEX gif_catalog_source_filename_uniq + ON public.gif_catalog (source_filename) + WHERE source_filename <> ''; diff --git a/internal/app/files/gif_admin.go b/internal/app/files/gif_admin.go index 047a5a0f..3a9a6731 100644 --- a/internal/app/files/gif_admin.go +++ b/internal/app/files/gif_admin.go @@ -111,6 +111,13 @@ func detectGifCatalogUploadMime(data []byte) (string, bool) { // AdminCreateGifCatalogEntry adds an already-materialized document (from // AdminUploadGifMaterial) to the catalog @gif serves. func (s *Service) AdminCreateGifCatalogEntry(ctx context.Context, title string, documentID int64) (domain.GifCatalogEntry, error) { + return s.createGifCatalogEntry(ctx, title, documentID, "") +} + +// createGifCatalogEntry is the shared insert path for both the admin-panel +// upload (AdminCreateGifCatalogEntry, sourceFilename="") and the filesystem +// seed import (SeedGifs, sourceFilename=the imported file's name). +func (s *Service) createGifCatalogEntry(ctx context.Context, title string, documentID int64, sourceFilename string) (domain.GifCatalogEntry, error) { if s.gifCatalog == nil { return domain.GifCatalogEntry{}, domain.ErrGifCatalogUnavailable } @@ -124,9 +131,10 @@ func (s *Service) AdminCreateGifCatalogEntry(ctx context.Context, title string, return domain.GifCatalogEntry{}, domain.ErrGifCatalogEntryInvalid } entry, err := s.gifCatalog.CreateGifCatalogEntry(ctx, domain.GifCatalogEntry{ - ID: randomID(), - Title: title, - DocumentID: documentID, + ID: randomID(), + Title: title, + DocumentID: documentID, + SourceFilename: sourceFilename, }) if err != nil { return domain.GifCatalogEntry{}, err diff --git a/internal/app/files/gif_seed.go b/internal/app/files/gif_seed.go new file mode 100644 index 00000000..41725f57 --- /dev/null +++ b/internal/app/files/gif_seed.go @@ -0,0 +1,141 @@ +package files + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "unicode" + + "go.uber.org/zap" + + "telesrv/internal/domain" +) + +// GifSeedStats reports one SeedGifs run's outcome. +type GifSeedStats struct { + Imported int + Skipped int + Failed int +} + +// SeedGifs imports every .gif/.mp4 file directly under root into the +// admin-curated GIF catalog @gif serves, so an operator can populate the +// catalog by just dropping files into a folder -- the same "drop it in +// data/, it shows up on next start" workflow SeedMedia already +// gives sticker/emoji packs, minus the export-manifest format that needs +// (regular sticker seeding carries fixed document ids/access hashes from an +// external export; there's no equivalent authority for a GIF an operator just +// found, so files here get a fresh app-generated id every import instead). +// +// A missing root is skipped, not an error -- same convention as SeedMedia and +// SeedPremiumPromo, since not every deployment wants a curated GIF catalog. +// Each file is imported at most once across restarts: entry.SourceFilename +// records the file's base name, and a file whose name already has a catalog +// row is skipped without re-reading or re-transcoding it. Renaming a file on +// disk therefore re-imports it as a new entry -- there's no content-hash +// dedup, only the filename-based one HasGifCatalogSourceFilename provides. +func (s *Service) SeedGifs(ctx context.Context, root string) (GifSeedStats, error) { + var stats GifSeedStats + if root == "" { + return stats, nil + } + entries, err := os.ReadDir(root) + if err != nil { + if os.IsNotExist(err) { + if s.log != nil { + s.log.Warn("gif seed directory does not exist, skipping gif catalog import (set TELESRV_GIF_SEED_DIR)", + zap.String("dir", root)) + } + return stats, nil + } + return stats, fmt.Errorf("read gif seed dir: %w", err) + } + if s.gifCatalog == nil { + if s.log != nil { + s.log.Warn("gif catalog store is not configured, skipping gif seed import", zap.String("dir", root)) + } + return stats, nil + } + + names := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + ext := strings.ToLower(filepath.Ext(entry.Name())) + if ext == ".gif" || ext == ".mp4" { + names = append(names, entry.Name()) + } + } + sort.Strings(names) // deterministic import/log order across runs + + for _, name := range names { + imported, err := s.seedOneGif(ctx, root, name) + switch { + case err != nil: + stats.Failed++ + if s.log != nil { + s.log.Warn("gif seed import failed", zap.String("file", name), zap.Error(err)) + } + case imported: + stats.Imported++ + default: + stats.Skipped++ + } + } + return stats, nil +} + +func (s *Service) seedOneGif(ctx context.Context, root, name string) (imported bool, err error) { + has, err := s.gifCatalog.HasGifCatalogSourceFilename(ctx, name) + if err != nil { + return false, err + } + if has { + return false, nil + } + data, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + return false, err + } + doc, err := s.AdminUploadGifMaterial(ctx, name, data) + if err != nil { + return false, err + } + if _, err := s.createGifCatalogEntry(ctx, gifTitleFromFilename(name), doc.ID, name); err != nil { + return false, err + } + return true, nil +} + +// gifTitleFromFilename derives a display title from a seed file's base name: +// strip the extension, replace separators with spaces, and title-case each +// word -- "cat_jumping-2.gif" becomes "Cat Jumping 2". +func gifTitleFromFilename(name string) string { + base := strings.TrimSuffix(name, filepath.Ext(name)) + base = strings.Map(func(r rune) rune { + if r == '_' || r == '-' { + return ' ' + } + return r + }, base) + words := strings.Fields(base) + for i, w := range words { + // []rune, not w[:1]/w[1:]: byte-slicing would corrupt any multi-byte + // first character (Cyrillic filenames are the expected case here, not + // an edge case). + r := []rune(w) + words[i] = string(unicode.ToUpper(r[0])) + string(r[1:]) + } + title := strings.Join(words, " ") + // MaxGifCatalogTitleLen is a byte budget (createGifCatalogEntry checks + // len(title) directly), so truncate by bytes too -- but back off to the + // last full rune so a multi-byte character never gets split in half. + for len(title) > domain.MaxGifCatalogTitleLen { + title = string([]rune(title)[:len([]rune(title))-1]) + } + return title +} diff --git a/internal/config/config.go b/internal/config/config.go index 71afc46a..b66df0fe 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -294,6 +294,11 @@ type Config struct { // PremiumPromoSeedDir 是 help.getPremiumPromo 视频与缩略图导出目录。 // 目录缺失时保留无视频兼容响应；目录存在但内容非法时启动失败。 PremiumPromoSeedDir string + // GifSeedDir is a directory of plain .gif/.mp4 files to import into the + // admin-curated GIF catalog on startup (files.Service.SeedGifs) -- unlike + // StickerSeedDir, this expects no export manifest, just raw files dropped + // in. Missing directory is skipped, not an error. + GifSeedDir string // BusinessAIProvider 控制服务端 Business automation 回复生成器。 // 空值/"echo" 回显触发私聊文本，用于跑通后续 AI provider 链路； // "template" 使用 quick reply 模板。 @@ -770,6 +775,7 @@ func Load() (Config, error) { StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"), StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300), PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"), + GifSeedDir: envOr("TELESRV_GIF_SEED_DIR", "data/gifs"), MapboxToken: envOr("TELESRV_MAPBOX_TOKEN", ""), MapTileCacheDir: envOr("TELESRV_MAPTILE_CACHE_DIR", "data/maptiles"), ExternalMediaEnable: envBoolOr("TELESRV_EXTERNAL_MEDIA_ENABLE", true), diff --git a/internal/domain/gif_catalog.go b/internal/domain/gif_catalog.go index e41031b2..08b23480 100644 --- a/internal/domain/gif_catalog.go +++ b/internal/domain/gif_catalog.go @@ -45,6 +45,12 @@ type GifCatalogEntry struct { Enabled bool SortOrder int CreatedBy string - CreatedAt time.Time - UpdatedAt time.Time + // SourceFilename is set only for entries files.Service.SeedGifs imported + // from the data/gifs/ drop directory -- empty for anything created through + // the admin panel. It exists purely so a restart can tell "this file was + // already imported" from "this is a new file" without re-transcoding + // every GIF on every startup; it carries no meaning once imported. + SourceFilename string + CreatedAt time.Time + UpdatedAt time.Time } diff --git a/internal/store/gif_catalog.go b/internal/store/gif_catalog.go index 1a959c73..424e361f 100644 --- a/internal/store/gif_catalog.go +++ b/internal/store/gif_catalog.go @@ -13,6 +13,12 @@ type GifCatalogStore interface { // by the caller (same convention as documents/photos elsewhere in this // codebase -- ids are app-generated, not database-serial). CreateGifCatalogEntry(ctx context.Context, entry domain.GifCatalogEntry) (domain.GifCatalogEntry, error) + // HasGifCatalogSourceFilename reports whether a seed-imported entry for + // this filename already exists, so files.Service.SeedGifs can skip + // re-transcoding a file it already imported on a previous startup. + // Always false for an empty filename (that's the panel-upload sentinel, + // never a real seed match). + HasGifCatalogSourceFilename(ctx context.Context, filename string) (bool, error) // ListGifCatalog returns every entry ordered by (sort_order, id). // onlyEnabled=true is what @gif serves; the admin panel lists everything. ListGifCatalog(ctx context.Context, onlyEnabled bool) ([]domain.GifCatalogEntry, error) diff --git a/internal/store/postgres/gif_catalog.go b/internal/store/postgres/gif_catalog.go index 69db215a..ea5b1f32 100644 --- a/internal/store/postgres/gif_catalog.go +++ b/internal/store/postgres/gif_catalog.go @@ -21,10 +21,10 @@ func (s *GifCatalogStore) CreateGifCatalogEntry(ctx context.Context, entry domai return domain.GifCatalogEntry{}, fmt.Errorf("create gif catalog entry: id and document_id are required") } row := s.db.QueryRow(ctx, ` -INSERT INTO gif_catalog (id, title, document_id, enabled, sort_order, created_by) -VALUES ($1, $2, $3, true, $4, $5) -RETURNING id, title, document_id, enabled, sort_order, created_by, created_at, updated_at`, - entry.ID, entry.Title, entry.DocumentID, entry.SortOrder, entry.CreatedBy) +INSERT INTO gif_catalog (id, title, document_id, enabled, sort_order, created_by, source_filename) +VALUES ($1, $2, $3, true, $4, $5, $6) +RETURNING id, title, document_id, enabled, sort_order, created_by, created_at, updated_at, source_filename`, + entry.ID, entry.Title, entry.DocumentID, entry.SortOrder, entry.CreatedBy, entry.SourceFilename) out, err := scanGifCatalogEntry(row.Scan) if err != nil { return domain.GifCatalogEntry{}, fmt.Errorf("create gif catalog entry: %w", err) @@ -32,9 +32,25 @@ RETURNING id, title, document_id, enabled, sort_order, created_by, created_at, u return out, nil } +// HasGifCatalogSourceFilename reports whether a seed-imported entry for this +// filename already exists. Deliberately not folded into CreateGifCatalogEntry +// as an ON CONFLICT DO NOTHING: SeedGifs needs to know *before* transcoding +// whether a file is new, not just fail silently after paying for ffmpeg work +// that turns out to be wasted. +func (s *GifCatalogStore) HasGifCatalogSourceFilename(ctx context.Context, filename string) (bool, error) { + if filename == "" { + return false, nil + } + var exists bool + if err := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM gif_catalog WHERE source_filename = $1)`, filename).Scan(&exists); err != nil { + return false, fmt.Errorf("check gif catalog source filename: %w", err) + } + return exists, nil +} + func (s *GifCatalogStore) ListGifCatalog(ctx context.Context, onlyEnabled bool) ([]domain.GifCatalogEntry, error) { rows, err := s.db.Query(ctx, ` -SELECT id, title, document_id, enabled, sort_order, created_by, created_at, updated_at +SELECT id, title, document_id, enabled, sort_order, created_by, created_at, updated_at, source_filename FROM gif_catalog WHERE NOT $1 OR enabled ORDER BY sort_order, id @@ -80,7 +96,7 @@ func (s *GifCatalogStore) DeleteGifCatalogEntry(ctx context.Context, id int64) ( func scanGifCatalogEntry(scan func(dest ...any) error) (domain.GifCatalogEntry, error) { var e domain.GifCatalogEntry - if err := scan(&e.ID, &e.Title, &e.DocumentID, &e.Enabled, &e.SortOrder, &e.CreatedBy, &e.CreatedAt, &e.UpdatedAt); err != nil { + if err := scan(&e.ID, &e.Title, &e.DocumentID, &e.Enabled, &e.SortOrder, &e.CreatedBy, &e.CreatedAt, &e.UpdatedAt, &e.SourceFilename); err != nil { return domain.GifCatalogEntry{}, err } return e, nil