added gifs scan at start
This commit is contained in:
parent
15f2d6e925
commit
4ac94f00de
10 changed files with 224 additions and 11 deletions
|
|
@ -404,6 +404,12 @@ TELESRV_STICKER_SEED_DIR=data/sticker-seed
|
|||
TELESRV_STICKER_SEED_MAX_SETS=300
|
||||
# Sticker set auto-installed for every newly registered account; <=0 disables this.
|
||||
TELESRV_DEFAULT_STICKER_SET_ID=0
|
||||
# Drop plain .gif/.mp4 files here and they're imported into the admin-curated
|
||||
# GIF catalog on startup (@gif serves it in the client's GIF picker) -- no
|
||||
# export manifest needed, unlike TELESRV_STICKER_SEED_DIR above. Each file is
|
||||
# imported once (matched by filename); renaming a file re-imports it as a new
|
||||
# entry. Missing directory is skipped, not an error.
|
||||
TELESRV_GIF_SEED_DIR=data/gifs
|
||||
|
||||
# Storage low-space guard thresholds (master toggle is TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE above).
|
||||
# localfs: reject new uploads once real free disk bytes fall below this; <=0 disables.
|
||||
|
|
|
|||
|
|
@ -798,6 +798,16 @@ func run(logger *zap.Logger) error {
|
|||
zap.Int("blobs", stats.Blobs),
|
||||
)
|
||||
}
|
||||
if stats, err := filesService.SeedGifs(ctx, cfg.GifSeedDir); err != nil {
|
||||
return fmt.Errorf("seed gifs: %w", err)
|
||||
} else if stats.Imported > 0 || stats.Failed > 0 {
|
||||
logger.Info("gif catalog seed import complete",
|
||||
zap.String("dir", cfg.GifSeedDir),
|
||||
zap.Int("imported", stats.Imported),
|
||||
zap.Int("skipped", stats.Skipped),
|
||||
zap.Int("failed", stats.Failed),
|
||||
)
|
||||
}
|
||||
if stats, err := filesService.SeedPremiumPromo(ctx, cfg.PremiumPromoSeedDir); err != nil {
|
||||
return fmt.Errorf("seed premium promo: %w", err)
|
||||
} else if !stats.Skipped {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
DROP INDEX IF EXISTS public.gif_catalog_source_filename_uniq;
|
||||
ALTER TABLE public.gif_catalog DROP COLUMN IF EXISTS source_filename;
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
-- Tracks which gif_catalog entries came from a filesystem seed drop
|
||||
-- (data/gifs/, see files.Service.SeedGifs) vs. an admin-panel upload, so a
|
||||
-- restart can tell "already imported this file" from "brand new file" without
|
||||
-- re-transcoding every GIF on every startup. Empty for panel-created entries.
|
||||
ALTER TABLE public.gif_catalog ADD COLUMN source_filename text DEFAULT ''::text NOT NULL;
|
||||
|
||||
-- Only seed-imported rows need uniqueness here: two panel uploads may
|
||||
-- legitimately share nothing to compare, but two seed imports of the same
|
||||
-- filename must collapse to one row on every restart.
|
||||
CREATE UNIQUE INDEX gif_catalog_source_filename_uniq
|
||||
ON public.gif_catalog (source_filename)
|
||||
WHERE source_filename <> '';
|
||||
|
|
@ -111,6 +111,13 @@ func detectGifCatalogUploadMime(data []byte) (string, bool) {
|
|||
// AdminCreateGifCatalogEntry adds an already-materialized document (from
|
||||
// AdminUploadGifMaterial) to the catalog @gif serves.
|
||||
func (s *Service) AdminCreateGifCatalogEntry(ctx context.Context, title string, documentID int64) (domain.GifCatalogEntry, error) {
|
||||
return s.createGifCatalogEntry(ctx, title, documentID, "")
|
||||
}
|
||||
|
||||
// createGifCatalogEntry is the shared insert path for both the admin-panel
|
||||
// upload (AdminCreateGifCatalogEntry, sourceFilename="") and the filesystem
|
||||
// seed import (SeedGifs, sourceFilename=the imported file's name).
|
||||
func (s *Service) createGifCatalogEntry(ctx context.Context, title string, documentID int64, sourceFilename string) (domain.GifCatalogEntry, error) {
|
||||
if s.gifCatalog == nil {
|
||||
return domain.GifCatalogEntry{}, domain.ErrGifCatalogUnavailable
|
||||
}
|
||||
|
|
@ -124,9 +131,10 @@ func (s *Service) AdminCreateGifCatalogEntry(ctx context.Context, title string,
|
|||
return domain.GifCatalogEntry{}, domain.ErrGifCatalogEntryInvalid
|
||||
}
|
||||
entry, err := s.gifCatalog.CreateGifCatalogEntry(ctx, domain.GifCatalogEntry{
|
||||
ID: randomID(),
|
||||
Title: title,
|
||||
DocumentID: documentID,
|
||||
ID: randomID(),
|
||||
Title: title,
|
||||
DocumentID: documentID,
|
||||
SourceFilename: sourceFilename,
|
||||
})
|
||||
if err != nil {
|
||||
return domain.GifCatalogEntry{}, err
|
||||
|
|
|
|||
141
internal/app/files/gif_seed.go
Normal file
141
internal/app/files/gif_seed.go
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// GifSeedStats reports one SeedGifs run's outcome.
|
||||
type GifSeedStats struct {
|
||||
Imported int
|
||||
Skipped int
|
||||
Failed int
|
||||
}
|
||||
|
||||
// SeedGifs imports every .gif/.mp4 file directly under root into the
|
||||
// admin-curated GIF catalog @gif serves, so an operator can populate the
|
||||
// catalog by just dropping files into a folder -- the same "drop it in
|
||||
// data/<seed dir>, it shows up on next start" workflow SeedMedia already
|
||||
// gives sticker/emoji packs, minus the export-manifest format that needs
|
||||
// (regular sticker seeding carries fixed document ids/access hashes from an
|
||||
// external export; there's no equivalent authority for a GIF an operator just
|
||||
// found, so files here get a fresh app-generated id every import instead).
|
||||
//
|
||||
// A missing root is skipped, not an error -- same convention as SeedMedia and
|
||||
// SeedPremiumPromo, since not every deployment wants a curated GIF catalog.
|
||||
// Each file is imported at most once across restarts: entry.SourceFilename
|
||||
// records the file's base name, and a file whose name already has a catalog
|
||||
// row is skipped without re-reading or re-transcoding it. Renaming a file on
|
||||
// disk therefore re-imports it as a new entry -- there's no content-hash
|
||||
// dedup, only the filename-based one HasGifCatalogSourceFilename provides.
|
||||
func (s *Service) SeedGifs(ctx context.Context, root string) (GifSeedStats, error) {
|
||||
var stats GifSeedStats
|
||||
if root == "" {
|
||||
return stats, nil
|
||||
}
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
if s.log != nil {
|
||||
s.log.Warn("gif seed directory does not exist, skipping gif catalog import (set TELESRV_GIF_SEED_DIR)",
|
||||
zap.String("dir", root))
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
return stats, fmt.Errorf("read gif seed dir: %w", err)
|
||||
}
|
||||
if s.gifCatalog == nil {
|
||||
if s.log != nil {
|
||||
s.log.Warn("gif catalog store is not configured, skipping gif seed import", zap.String("dir", root))
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(entry.Name()))
|
||||
if ext == ".gif" || ext == ".mp4" {
|
||||
names = append(names, entry.Name())
|
||||
}
|
||||
}
|
||||
sort.Strings(names) // deterministic import/log order across runs
|
||||
|
||||
for _, name := range names {
|
||||
imported, err := s.seedOneGif(ctx, root, name)
|
||||
switch {
|
||||
case err != nil:
|
||||
stats.Failed++
|
||||
if s.log != nil {
|
||||
s.log.Warn("gif seed import failed", zap.String("file", name), zap.Error(err))
|
||||
}
|
||||
case imported:
|
||||
stats.Imported++
|
||||
default:
|
||||
stats.Skipped++
|
||||
}
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (s *Service) seedOneGif(ctx context.Context, root, name string) (imported bool, err error) {
|
||||
has, err := s.gifCatalog.HasGifCatalogSourceFilename(ctx, name)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if has {
|
||||
return false, nil
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(root, name))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
doc, err := s.AdminUploadGifMaterial(ctx, name, data)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err := s.createGifCatalogEntry(ctx, gifTitleFromFilename(name), doc.ID, name); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// gifTitleFromFilename derives a display title from a seed file's base name:
|
||||
// strip the extension, replace separators with spaces, and title-case each
|
||||
// word -- "cat_jumping-2.gif" becomes "Cat Jumping 2".
|
||||
func gifTitleFromFilename(name string) string {
|
||||
base := strings.TrimSuffix(name, filepath.Ext(name))
|
||||
base = strings.Map(func(r rune) rune {
|
||||
if r == '_' || r == '-' {
|
||||
return ' '
|
||||
}
|
||||
return r
|
||||
}, base)
|
||||
words := strings.Fields(base)
|
||||
for i, w := range words {
|
||||
// []rune, not w[:1]/w[1:]: byte-slicing would corrupt any multi-byte
|
||||
// first character (Cyrillic filenames are the expected case here, not
|
||||
// an edge case).
|
||||
r := []rune(w)
|
||||
words[i] = string(unicode.ToUpper(r[0])) + string(r[1:])
|
||||
}
|
||||
title := strings.Join(words, " ")
|
||||
// MaxGifCatalogTitleLen is a byte budget (createGifCatalogEntry checks
|
||||
// len(title) directly), so truncate by bytes too -- but back off to the
|
||||
// last full rune so a multi-byte character never gets split in half.
|
||||
for len(title) > domain.MaxGifCatalogTitleLen {
|
||||
title = string([]rune(title)[:len([]rune(title))-1])
|
||||
}
|
||||
return title
|
||||
}
|
||||
|
|
@ -294,6 +294,11 @@ type Config struct {
|
|||
// PremiumPromoSeedDir 是 help.getPremiumPromo 视频与缩略图导出目录。
|
||||
// 目录缺失时保留无视频兼容响应;目录存在但内容非法时启动失败。
|
||||
PremiumPromoSeedDir string
|
||||
// GifSeedDir is a directory of plain .gif/.mp4 files to import into the
|
||||
// admin-curated GIF catalog on startup (files.Service.SeedGifs) -- unlike
|
||||
// StickerSeedDir, this expects no export manifest, just raw files dropped
|
||||
// in. Missing directory is skipped, not an error.
|
||||
GifSeedDir string
|
||||
// BusinessAIProvider 控制服务端 Business automation 回复生成器。
|
||||
// 空值/"echo" 回显触发私聊文本,用于跑通后续 AI provider 链路;
|
||||
// "template" 使用 quick reply 模板。
|
||||
|
|
@ -770,6 +775,7 @@ func Load() (Config, error) {
|
|||
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
|
||||
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
|
||||
PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"),
|
||||
GifSeedDir: envOr("TELESRV_GIF_SEED_DIR", "data/gifs"),
|
||||
MapboxToken: envOr("TELESRV_MAPBOX_TOKEN", ""),
|
||||
MapTileCacheDir: envOr("TELESRV_MAPTILE_CACHE_DIR", "data/maptiles"),
|
||||
ExternalMediaEnable: envBoolOr("TELESRV_EXTERNAL_MEDIA_ENABLE", true),
|
||||
|
|
|
|||
|
|
@ -45,6 +45,12 @@ type GifCatalogEntry struct {
|
|||
Enabled bool
|
||||
SortOrder int
|
||||
CreatedBy string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
// SourceFilename is set only for entries files.Service.SeedGifs imported
|
||||
// from the data/gifs/ drop directory -- empty for anything created through
|
||||
// the admin panel. It exists purely so a restart can tell "this file was
|
||||
// already imported" from "this is a new file" without re-transcoding
|
||||
// every GIF on every startup; it carries no meaning once imported.
|
||||
SourceFilename string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,12 @@ type GifCatalogStore interface {
|
|||
// by the caller (same convention as documents/photos elsewhere in this
|
||||
// codebase -- ids are app-generated, not database-serial).
|
||||
CreateGifCatalogEntry(ctx context.Context, entry domain.GifCatalogEntry) (domain.GifCatalogEntry, error)
|
||||
// HasGifCatalogSourceFilename reports whether a seed-imported entry for
|
||||
// this filename already exists, so files.Service.SeedGifs can skip
|
||||
// re-transcoding a file it already imported on a previous startup.
|
||||
// Always false for an empty filename (that's the panel-upload sentinel,
|
||||
// never a real seed match).
|
||||
HasGifCatalogSourceFilename(ctx context.Context, filename string) (bool, error)
|
||||
// ListGifCatalog returns every entry ordered by (sort_order, id).
|
||||
// onlyEnabled=true is what @gif serves; the admin panel lists everything.
|
||||
ListGifCatalog(ctx context.Context, onlyEnabled bool) ([]domain.GifCatalogEntry, error)
|
||||
|
|
|
|||
|
|
@ -21,10 +21,10 @@ func (s *GifCatalogStore) CreateGifCatalogEntry(ctx context.Context, entry domai
|
|||
return domain.GifCatalogEntry{}, fmt.Errorf("create gif catalog entry: id and document_id are required")
|
||||
}
|
||||
row := s.db.QueryRow(ctx, `
|
||||
INSERT INTO gif_catalog (id, title, document_id, enabled, sort_order, created_by)
|
||||
VALUES ($1, $2, $3, true, $4, $5)
|
||||
RETURNING id, title, document_id, enabled, sort_order, created_by, created_at, updated_at`,
|
||||
entry.ID, entry.Title, entry.DocumentID, entry.SortOrder, entry.CreatedBy)
|
||||
INSERT INTO gif_catalog (id, title, document_id, enabled, sort_order, created_by, source_filename)
|
||||
VALUES ($1, $2, $3, true, $4, $5, $6)
|
||||
RETURNING id, title, document_id, enabled, sort_order, created_by, created_at, updated_at, source_filename`,
|
||||
entry.ID, entry.Title, entry.DocumentID, entry.SortOrder, entry.CreatedBy, entry.SourceFilename)
|
||||
out, err := scanGifCatalogEntry(row.Scan)
|
||||
if err != nil {
|
||||
return domain.GifCatalogEntry{}, fmt.Errorf("create gif catalog entry: %w", err)
|
||||
|
|
@ -32,9 +32,25 @@ RETURNING id, title, document_id, enabled, sort_order, created_by, created_at, u
|
|||
return out, nil
|
||||
}
|
||||
|
||||
// HasGifCatalogSourceFilename reports whether a seed-imported entry for this
|
||||
// filename already exists. Deliberately not folded into CreateGifCatalogEntry
|
||||
// as an ON CONFLICT DO NOTHING: SeedGifs needs to know *before* transcoding
|
||||
// whether a file is new, not just fail silently after paying for ffmpeg work
|
||||
// that turns out to be wasted.
|
||||
func (s *GifCatalogStore) HasGifCatalogSourceFilename(ctx context.Context, filename string) (bool, error) {
|
||||
if filename == "" {
|
||||
return false, nil
|
||||
}
|
||||
var exists bool
|
||||
if err := s.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM gif_catalog WHERE source_filename = $1)`, filename).Scan(&exists); err != nil {
|
||||
return false, fmt.Errorf("check gif catalog source filename: %w", err)
|
||||
}
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (s *GifCatalogStore) ListGifCatalog(ctx context.Context, onlyEnabled bool) ([]domain.GifCatalogEntry, error) {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT id, title, document_id, enabled, sort_order, created_by, created_at, updated_at
|
||||
SELECT id, title, document_id, enabled, sort_order, created_by, created_at, updated_at, source_filename
|
||||
FROM gif_catalog
|
||||
WHERE NOT $1 OR enabled
|
||||
ORDER BY sort_order, id
|
||||
|
|
@ -80,7 +96,7 @@ func (s *GifCatalogStore) DeleteGifCatalogEntry(ctx context.Context, id int64) (
|
|||
|
||||
func scanGifCatalogEntry(scan func(dest ...any) error) (domain.GifCatalogEntry, error) {
|
||||
var e domain.GifCatalogEntry
|
||||
if err := scan(&e.ID, &e.Title, &e.DocumentID, &e.Enabled, &e.SortOrder, &e.CreatedBy, &e.CreatedAt, &e.UpdatedAt); err != nil {
|
||||
if err := scan(&e.ID, &e.Title, &e.DocumentID, &e.Enabled, &e.SortOrder, &e.CreatedBy, &e.CreatedAt, &e.UpdatedAt, &e.SourceFilename); err != nil {
|
||||
return domain.GifCatalogEntry{}, err
|
||||
}
|
||||
return e, nil
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue