added gifs scan at start

This commit is contained in:
onysd 2026-08-07 11:02:05 +03:00
parent 15f2d6e925
commit 4ac94f00de
10 changed files with 224 additions and 11 deletions

View file

@ -404,6 +404,12 @@ TELESRV_STICKER_SEED_DIR=data/sticker-seed
TELESRV_STICKER_SEED_MAX_SETS=300 TELESRV_STICKER_SEED_MAX_SETS=300
# Sticker set auto-installed for every newly registered account; <=0 disables this. # Sticker set auto-installed for every newly registered account; <=0 disables this.
TELESRV_DEFAULT_STICKER_SET_ID=0 TELESRV_DEFAULT_STICKER_SET_ID=0
# Drop plain .gif/.mp4 files here and they're imported into the admin-curated
# GIF catalog on startup (@gif serves it in the client's GIF picker) -- no
# export manifest needed, unlike TELESRV_STICKER_SEED_DIR above. Each file is
# imported once (matched by filename); renaming a file re-imports it as a new
# entry. Missing directory is skipped, not an error.
TELESRV_GIF_SEED_DIR=data/gifs
# Storage low-space guard thresholds (master toggle is TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE above). # Storage low-space guard thresholds (master toggle is TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE above).
# localfs: reject new uploads once real free disk bytes fall below this; <=0 disables. # localfs: reject new uploads once real free disk bytes fall below this; <=0 disables.

View file

@ -798,6 +798,16 @@ func run(logger *zap.Logger) error {
zap.Int("blobs", stats.Blobs), zap.Int("blobs", stats.Blobs),
) )
} }
if stats, err := filesService.SeedGifs(ctx, cfg.GifSeedDir); err != nil {
return fmt.Errorf("seed gifs: %w", err)
} else if stats.Imported > 0 || stats.Failed > 0 {
logger.Info("gif catalog seed import complete",
zap.String("dir", cfg.GifSeedDir),
zap.Int("imported", stats.Imported),
zap.Int("skipped", stats.Skipped),
zap.Int("failed", stats.Failed),
)
}
if stats, err := filesService.SeedPremiumPromo(ctx, cfg.PremiumPromoSeedDir); err != nil { if stats, err := filesService.SeedPremiumPromo(ctx, cfg.PremiumPromoSeedDir); err != nil {
return fmt.Errorf("seed premium promo: %w", err) return fmt.Errorf("seed premium promo: %w", err)
} else if !stats.Skipped { } else if !stats.Skipped {

View file

@ -0,0 +1,2 @@
DROP INDEX IF EXISTS public.gif_catalog_source_filename_uniq;
ALTER TABLE public.gif_catalog DROP COLUMN IF EXISTS source_filename;

View file

@ -0,0 +1,12 @@
-- Tracks which gif_catalog entries came from a filesystem seed drop
-- (data/gifs/, see files.Service.SeedGifs) vs. an admin-panel upload, so a
-- restart can tell "already imported this file" from "brand new file" without
-- re-transcoding every GIF on every startup. Empty for panel-created entries.
ALTER TABLE public.gif_catalog ADD COLUMN source_filename text DEFAULT ''::text NOT NULL;
-- Only seed-imported rows need uniqueness here: two panel uploads may
-- legitimately share nothing to compare, but two seed imports of the same
-- filename must collapse to one row on every restart.
CREATE UNIQUE INDEX gif_catalog_source_filename_uniq
ON public.gif_catalog (source_filename)
WHERE source_filename <> '';

View file

@ -111,6 +111,13 @@ func detectGifCatalogUploadMime(data []byte) (string, bool) {
// AdminCreateGifCatalogEntry adds an already-materialized document (from // AdminCreateGifCatalogEntry adds an already-materialized document (from
// AdminUploadGifMaterial) to the catalog @gif serves. // AdminUploadGifMaterial) to the catalog @gif serves.
func (s *Service) AdminCreateGifCatalogEntry(ctx context.Context, title string, documentID int64) (domain.GifCatalogEntry, error) { func (s *Service) AdminCreateGifCatalogEntry(ctx context.Context, title string, documentID int64) (domain.GifCatalogEntry, error) {
return s.createGifCatalogEntry(ctx, title, documentID, "")
}
// createGifCatalogEntry is the shared insert path for both the admin-panel
// upload (AdminCreateGifCatalogEntry, sourceFilename="") and the filesystem
// seed import (SeedGifs, sourceFilename=the imported file's name).
func (s *Service) createGifCatalogEntry(ctx context.Context, title string, documentID int64, sourceFilename string) (domain.GifCatalogEntry, error) {
if s.gifCatalog == nil { if s.gifCatalog == nil {
return domain.GifCatalogEntry{}, domain.ErrGifCatalogUnavailable return domain.GifCatalogEntry{}, domain.ErrGifCatalogUnavailable
} }
@ -124,9 +131,10 @@ func (s *Service) AdminCreateGifCatalogEntry(ctx context.Context, title string,
return domain.GifCatalogEntry{}, domain.ErrGifCatalogEntryInvalid return domain.GifCatalogEntry{}, domain.ErrGifCatalogEntryInvalid
} }
entry, err := s.gifCatalog.CreateGifCatalogEntry(ctx, domain.GifCatalogEntry{ entry, err := s.gifCatalog.CreateGifCatalogEntry(ctx, domain.GifCatalogEntry{
ID: randomID(), ID: randomID(),
Title: title, Title: title,
DocumentID: documentID, DocumentID: documentID,
SourceFilename: sourceFilename,
}) })
if err != nil { if err != nil {
return domain.GifCatalogEntry{}, err return domain.GifCatalogEntry{}, err

View file

@ -0,0 +1,141 @@
package files
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"unicode"
"go.uber.org/zap"
"telesrv/internal/domain"
)
// GifSeedStats reports one SeedGifs run's outcome.
type GifSeedStats struct {
Imported int
Skipped int
Failed int
}
// SeedGifs imports every .gif/.mp4 file directly under root into the
// admin-curated GIF catalog @gif serves, so an operator can populate the
// catalog by just dropping files into a folder -- the same "drop it in
// data/<seed dir>, it shows up on next start" workflow SeedMedia already
// gives sticker/emoji packs, minus the export-manifest format that needs
// (regular sticker seeding carries fixed document ids/access hashes from an
// external export; there's no equivalent authority for a GIF an operator just
// found, so files here get a fresh app-generated id every import instead).
//
// A missing root is skipped, not an error -- same convention as SeedMedia and
// SeedPremiumPromo, since not every deployment wants a curated GIF catalog.
// Each file is imported at most once across restarts: entry.SourceFilename
// records the file's base name, and a file whose name already has a catalog
// row is skipped without re-reading or re-transcoding it. Renaming a file on
// disk therefore re-imports it as a new entry -- there's no content-hash
// dedup, only the filename-based one HasGifCatalogSourceFilename provides.
func (s *Service) SeedGifs(ctx context.Context, root string) (GifSeedStats, error) {
var stats GifSeedStats
if root == "" {
return stats, nil
}
entries, err := os.ReadDir(root)
if err != nil {
if os.IsNotExist(err) {
if s.log != nil {
s.log.Warn("gif seed directory does not exist, skipping gif catalog import (set TELESRV_GIF_SEED_DIR)",
zap.String("dir", root))
}
return stats, nil
}
return stats, fmt.Errorf("read gif seed dir: %w", err)
}
if s.gifCatalog == nil {
if s.log != nil {
s.log.Warn("gif catalog store is not configured, skipping gif seed import", zap.String("dir", root))
}
return stats, nil
}
names := make([]string, 0, len(entries))
for _, entry := range entries {
if entry.IsDir() {
continue
}
ext := strings.ToLower(filepath.Ext(entry.Name()))
if ext == ".gif" || ext == ".mp4" {
names = append(names, entry.Name())
}
}
sort.Strings(names) // deterministic import/log order across runs
for _, name := range names {
imported, err := s.seedOneGif(ctx, root, name)
switch {
case err != nil:
stats.Failed++
if s.log != nil {
s.log.Warn("gif seed import failed", zap.String("file", name), zap.Error(err))
}
case imported:
stats.Imported++
default:
stats.Skipped++
}
}
return stats, nil
}
func (s *Service) seedOneGif(ctx context.Context, root, name string) (imported bool, err error) {
has, err := s.gifCatalog.HasGifCatalogSourceFilename(ctx, name)
if err != nil {
return false, err
}
if has {
return false, nil
}
data, err := os.ReadFile(filepath.Join(root, name))
if err != nil {
return false, err
}
doc, err := s.AdminUploadGifMaterial(ctx, name, data)
if err != nil {
return false, err
}
if _, err := s.createGifCatalogEntry(ctx, gifTitleFromFilename(name), doc.ID, name); err != nil {
return false, err
}
return true, nil
}
// gifTitleFromFilename derives a display title from a seed file's base name:
// strip the extension, replace separators with spaces, and title-case each
// word -- "cat_jumping-2.gif" becomes "Cat Jumping 2".
func gifTitleFromFilename(name string) string {
base := strings.TrimSuffix(name, filepath.Ext(name))
base = strings.Map(func(r rune) rune {
if r == '_' || r == '-' {
return ' '
}
return r
}, base)
words := strings.Fields(base)
for i, w := range words {
// []rune, not w[:1]/w[1:]: byte-slicing would corrupt any multi-byte
// first character (Cyrillic filenames are the expected case here, not
// an edge case).
r := []rune(w)
words[i] = string(unicode.ToUpper(r[0])) + string(r[1:])
}
title := strings.Join(words, " ")
// MaxGifCatalogTitleLen is a byte budget (createGifCatalogEntry checks
// len(title) directly), so truncate by bytes too -- but back off to the
// last full rune so a multi-byte character never gets split in half.
for len(title) > domain.MaxGifCatalogTitleLen {
title = string([]rune(title)[:len([]rune(title))-1])
}
return title
}

View file

@ -294,6 +294,11 @@ type Config struct {
// PremiumPromoSeedDir 是 help.getPremiumPromo 视频与缩略图导出目录。 // PremiumPromoSeedDir 是 help.getPremiumPromo 视频与缩略图导出目录。
// 目录缺失时保留无视频兼容响应;目录存在但内容非法时启动失败。 // 目录缺失时保留无视频兼容响应;目录存在但内容非法时启动失败。
PremiumPromoSeedDir string PremiumPromoSeedDir string
// GifSeedDir is a directory of plain .gif/.mp4 files to import into the
// admin-curated GIF catalog on startup (files.Service.SeedGifs) -- unlike
// StickerSeedDir, this expects no export manifest, just raw files dropped
// in. Missing directory is skipped, not an error.
GifSeedDir string
// BusinessAIProvider 控制服务端 Business automation 回复生成器。 // BusinessAIProvider 控制服务端 Business automation 回复生成器。
// 空值/"echo" 回显触发私聊文本,用于跑通后续 AI provider 链路; // 空值/"echo" 回显触发私聊文本,用于跑通后续 AI provider 链路;
// "template" 使用 quick reply 模板。 // "template" 使用 quick reply 模板。
@ -770,6 +775,7 @@ func Load() (Config, error) {
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"), StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300), StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"), PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"),
GifSeedDir: envOr("TELESRV_GIF_SEED_DIR", "data/gifs"),
MapboxToken: envOr("TELESRV_MAPBOX_TOKEN", ""), MapboxToken: envOr("TELESRV_MAPBOX_TOKEN", ""),
MapTileCacheDir: envOr("TELESRV_MAPTILE_CACHE_DIR", "data/maptiles"), MapTileCacheDir: envOr("TELESRV_MAPTILE_CACHE_DIR", "data/maptiles"),
ExternalMediaEnable: envBoolOr("TELESRV_EXTERNAL_MEDIA_ENABLE", true), ExternalMediaEnable: envBoolOr("TELESRV_EXTERNAL_MEDIA_ENABLE", true),

View file

@ -45,6 +45,12 @@ type GifCatalogEntry struct {
Enabled bool Enabled bool
SortOrder int SortOrder int
CreatedBy string CreatedBy string
CreatedAt time.Time // SourceFilename is set only for entries files.Service.SeedGifs imported
UpdatedAt time.Time // from the data/gifs/ drop directory -- empty for anything created through
// the admin panel. It exists purely so a restart can tell "this file was
// already imported" from "this is a new file" without re-transcoding
// every GIF on every startup; it carries no meaning once imported.
SourceFilename string
CreatedAt time.Time
UpdatedAt time.Time
} }

View file

@ -13,6 +13,12 @@ type GifCatalogStore interface {
// by the caller (same convention as documents/photos elsewhere in this // by the caller (same convention as documents/photos elsewhere in this
// codebase -- ids are app-generated, not database-serial). // codebase -- ids are app-generated, not database-serial).
CreateGifCatalogEntry(ctx context.Context, entry domain.GifCatalogEntry) (domain.GifCatalogEntry, error) CreateGifCatalogEntry(ctx context.Context, entry domain.GifCatalogEntry) (domain.GifCatalogEntry, error)
// HasGifCatalogSourceFilename reports whether a seed-imported entry for
// this filename already exists, so files.Service.SeedGifs can skip
// re-transcoding a file it already imported on a previous startup.
// Always false for an empty filename (that's the panel-upload sentinel,
// never a real seed match).
HasGifCatalogSourceFilename(ctx context.Context, filename string) (bool, error)
// ListGifCatalog returns every entry ordered by (sort_order, id). // ListGifCatalog returns every entry ordered by (sort_order, id).
// onlyEnabled=true is what @gif serves; the admin panel lists everything. // onlyEnabled=true is what @gif serves; the admin panel lists everything.
ListGifCatalog(ctx context.Context, onlyEnabled bool) ([]domain.GifCatalogEntry, error) ListGifCatalog(ctx context.Context, onlyEnabled bool) ([]domain.GifCatalogEntry, error)

View file

@ -21,10 +21,10 @@ func (s *GifCatalogStore) CreateGifCatalogEntry(ctx context.Context, entry domai
return domain.GifCatalogEntry{}, fmt.Errorf("create gif catalog entry: id and document_id are required") return domain.GifCatalogEntry{}, fmt.Errorf("create gif catalog entry: id and document_id are required")
} }
row := s.db.QueryRow(ctx, ` row := s.db.QueryRow(ctx, `
INSERT INTO gif_catalog (id, title, document_id, enabled, sort_order, created_by) INSERT INTO gif_catalog (id, title, document_id, enabled, sort_order, created_by, source_filename)
VALUES ($1, $2, $3, true, $4, $5) VALUES ($1, $2, $3, true, $4, $5, $6)
RETURNING id, title, document_id, enabled, sort_order, created_by, created_at, updated_at`, RETURNING id, title, document_id, enabled, sort_order, created_by, created_at, updated_at, source_filename`,
entry.ID, entry.Title, entry.DocumentID, entry.SortOrder, entry.CreatedBy) entry.ID, entry.Title, entry.DocumentID, entry.SortOrder, entry.CreatedBy, entry.SourceFilename)
out, err := scanGifCatalogEntry(row.Scan) out, err := scanGifCatalogEntry(row.Scan)
if err != nil { if err != nil {
return domain.GifCatalogEntry{}, fmt.Errorf("create gif catalog entry: %w", err) return domain.GifCatalogEntry{}, fmt.Errorf("create gif catalog entry: %w", err)
@ -32,9 +32,25 @@ RETURNING id, title, document_id, enabled, sort_order, created_by, created_at, u
return out, nil return out, nil
} }
// HasGifCatalogSourceFilename reports whether a seed-imported entry for this
// filename already exists. Deliberately not folded into CreateGifCatalogEntry
// as an ON CONFLICT DO NOTHING: SeedGifs needs to know *before* transcoding
// whether a file is new, not just fail silently after paying for ffmpeg work
// that turns out to be wasted.
func (s *GifCatalogStore) HasGifCatalogSourceFilename(ctx context.Context, filename string) (bool, error) {
if filename == "" {
return false, nil
}
var exists bool
if err := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM gif_catalog WHERE source_filename = $1)`, filename).Scan(&exists); err != nil {
return false, fmt.Errorf("check gif catalog source filename: %w", err)
}
return exists, nil
}
func (s *GifCatalogStore) ListGifCatalog(ctx context.Context, onlyEnabled bool) ([]domain.GifCatalogEntry, error) { func (s *GifCatalogStore) ListGifCatalog(ctx context.Context, onlyEnabled bool) ([]domain.GifCatalogEntry, error) {
rows, err := s.db.Query(ctx, ` rows, err := s.db.Query(ctx, `
SELECT id, title, document_id, enabled, sort_order, created_by, created_at, updated_at SELECT id, title, document_id, enabled, sort_order, created_by, created_at, updated_at, source_filename
FROM gif_catalog FROM gif_catalog
WHERE NOT $1 OR enabled WHERE NOT $1 OR enabled
ORDER BY sort_order, id ORDER BY sort_order, id
@ -80,7 +96,7 @@ func (s *GifCatalogStore) DeleteGifCatalogEntry(ctx context.Context, id int64) (
func scanGifCatalogEntry(scan func(dest ...any) error) (domain.GifCatalogEntry, error) { func scanGifCatalogEntry(scan func(dest ...any) error) (domain.GifCatalogEntry, error) {
var e domain.GifCatalogEntry var e domain.GifCatalogEntry
if err := scan(&e.ID, &e.Title, &e.DocumentID, &e.Enabled, &e.SortOrder, &e.CreatedBy, &e.CreatedAt, &e.UpdatedAt); err != nil { if err := scan(&e.ID, &e.Title, &e.DocumentID, &e.Enabled, &e.SortOrder, &e.CreatedBy, &e.CreatedAt, &e.UpdatedAt, &e.SourceFilename); err != nil {
return domain.GifCatalogEntry{}, err return domain.GifCatalogEntry{}, err
} }
return e, nil return e, nil