diff --git a/.gitignore b/.gitignore index 994cca39..b7c5a824 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,11 @@ coverage.* node_modules/ +# Python caches +__pycache__/ +*.py[cod] +*$py.class + # 本地环境 / 密钥（server RSA private key 必须持久化，但禁止入库） .env .env.* diff --git a/README.md b/README.md index a225ca8c..6e747e32 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ codebase. | ✅ | Media and files | Upload, download, local blob storage, photos, documents, thumbnails, external media fetch, web page previews, map tile cache hooks, profile/channel photos. | | ✅ | Stickers and reactions | Sticker/reaction catalog, seed support, recent reactions, top reactions, default reactions, and moderation-oriented reaction paths. | | ✅ | Gifts and stars | Star gifts and local stars ledger foundations for compatibility and future feature work. | -| ✅ | Bots and mini apps | Bot service foundations, callbacks, inline helpers, webview/mini-app paths, minimal Bot API gateway, and demo tools. | +| ✅ | Bots and mini apps | Bot service foundations, callbacks, inline helpers, webview/mini-app paths, a minimal Bot API gateway for libraries such as `python-telegram-bot`, persistent `getUpdates` delivery, and demo tools. | | ✅ | Calls and live streams | Private call signaling foundations, group call state, RTMP live streaming, scheduled video chats, channel `join_as`, SFU/TURN building blocks, liveness, and expiry workers. | | ✅ | Admin and operations | Admin API/UI backend, PostgreSQL migrations, Redis volatile state, retention workers, pprof/debug hooks, and load-test helpers. | | ✅ | Desktop, Android, and Web focus | Telegram Desktop is the primary target, with Android and Web compatibility paths actively covered by the same server. | @@ -115,6 +115,8 @@ Useful local environment variables: | `TELESRV_BLOB_DIR` | `data/blobs` | local media blob directory | | `TELESRV_STICKER_SEED_DIR` | `data/sticker-seed` | optional sticker/reaction seed directory | | `TELESRV_PUBLIC_LINK_WEB_ADDR` | empty | optional public link landing endpoint for sticker and chatlist links | +| `TELESRV_BOT_API_ADDR` | empty | optional HTTP Bot API gateway listen address, for example `127.0.0.1:8081` | +| `TELESRV_BOT_API_UPDATE_RETENTION` | `24h` | retention window for unconfirmed Bot API `getUpdates` queue entries | | `TELESRV_AI_ENABLED` | `true` | enable AI compose entry points | | `TELESRV_AI_PROVIDERS` | `local` | ordered AI provider chain, such as `local` or `kimi,local` | | `TELESRV_AI_TIMEOUT` | `15s` | per AI provider call timeout | diff --git a/README.zh-CN.md b/README.zh-CN.md index ea9bdeca..58e271fa 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -49,7 +49,7 @@ https://github.com/user-attachments/assets/25e651dc-a022-4d60-8b9b-ca3e8bfe216c | ✅ | 媒体与文件 | upload、download、本地 blob 存储、照片、文档、缩略图、外链媒体抓取、网页预览、地图缩略图缓存、用户/频道头像。 | | ✅ | Stickers 与 Reactions | sticker/reaction catalog、seed 支持、recent reactions、top reactions、default reactions、reaction moderation 相关路径。 | | ✅ | Gifts 与 Stars | star gifts、本地 stars ledger 基础，用于兼容和后续功能扩展。 | -| ✅ | Bots 与 Mini Apps | bot 服务基础、callbacks、inline helpers、webview/mini-app 路径、最小 Bot API gateway、demo 工具。 | +| ✅ | Bots 与 Mini Apps | bot 服务基础、callbacks、inline helpers、webview/mini-app 路径、适配 `python-telegram-bot` 等库的最小 Bot API gateway、持久化 `getUpdates` 投递队列和 demo 工具。 | | ✅ | 通话与直播 | 私聊通话信令基础、group call 状态、RTMP live stream、定时视频通话、频道 `join_as` 身份、SFU/TURN building blocks、liveness 与 expiry worker。 | | ✅ | 管理与运维 | Admin API/UI backend、PostgreSQL migrations、Redis 易失态、retention workers、pprof/debug hooks、load-test helpers。 | | ✅ | Desktop、Android 与 Web 兼容 | Telegram Desktop 是第一目标，Android 与 Web 兼容路径也由同一套 server 持续覆盖。 | @@ -107,6 +107,8 @@ go build -o bin/gramsrv ./cmd/telesrv | `TELESRV_BLOB_DIR` | `data/blobs` | 本地媒体 blob 目录 | | `TELESRV_STICKER_SEED_DIR` | `data/sticker-seed` | 可选 sticker/reaction 种子目录 | | `TELESRV_PUBLIC_LINK_WEB_ADDR` | 空 | 可选的 sticker/chatlist 公开链接落地页监听地址 | +| `TELESRV_BOT_API_ADDR` | 空 | 可选 HTTP Bot API gateway 监听地址，例如 `127.0.0.1:8081` | +| `TELESRV_BOT_API_UPDATE_RETENTION` | `24h` | 未确认 Bot API `getUpdates` 队列记录的保留窗口 | | `TELESRV_AI_ENABLED` | `true` | 启用 AI compose 入口 | | `TELESRV_AI_PROVIDERS` | `local` | AI provider 调用链，例如 `local` 或 `kimi,local` | | `TELESRV_AI_TIMEOUT` | `15s` | 单次 AI provider 调用超时 | diff --git a/cmd/bots/ptbecho/README.md b/cmd/bots/ptbecho/README.md new file mode 100644 index 00000000..72c8eb9a --- /dev/null +++ b/cmd/bots/ptbecho/README.md @@ -0,0 +1,49 @@ +# python-telegram-bot echo demo + +This demo uses the normal `python-telegram-bot` API and only swaps the Bot API +base URLs to telesrv. + +```powershell +python -m pip install python-telegram-bot +$env:TELESRV_BOT_TOKEN = ":" +python .\cmd\bots\ptbecho\echo.py ` + --base-url http://127.0.0.1:8081/bot ` + --base-file-url http://127.0.0.1:8081/file/bot +``` + +In a group with BotFather privacy enabled, send a command such as: + +```text +/ping hello from group +``` + +主动发送一条消息并退出： + +```powershell +python .\cmd\bots\ptbecho\echo.py ` + --send-only ` + --send-chat-id -1000000000002 ` + --send-text "hello from python-telegram-bot" +``` + +长轮询 echo 启动后立即主动发送一条消息： + +```powershell +python .\cmd\bots\ptbecho\echo.py ` + --send-chat-id -1000000000002 ` + --send-text "ptbecho is online" +``` + +可选参数： + +- `--send-count N`：连续主动发送 N 条。 +- `--send-interval SEC`：连续发送之间的间隔。 +- `TELESRV_BOT_DEMO_CHAT_ID` / `TELESRV_BOT_DEMO_SEND_TEXT`：主动发送参数的环境变量形式。 + +本地超级群 chat id 使用 Bot API 形式 `-100`；例如 channel id 为 +`2` 时是 `-1000000000002`。 + +Implemented telesrv Bot API surface for this demo: `getMe`, `getUpdates`, +`deleteWebhook`, `sendMessage`, and file URL configuration. The wider gateway +also has basic `sendPhoto`, `sendDocument`, `editMessageText`, `deleteMessage`, +`answerCallbackQuery`, `getFile`, and `/file/bot...` support. diff --git a/cmd/bots/ptbecho/echo.py b/cmd/bots/ptbecho/echo.py new file mode 100644 index 00000000..6e3d5a84 --- /dev/null +++ b/cmd/bots/ptbecho/echo.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""python-telegram-bot echo demo for telesrv Bot API. + +This is a normal python-telegram-bot program. The only telesrv-specific part is +the custom base_url/base_file_url pair. + +Example: + + python cmd/bots/ptbecho/echo.py \ + --token "1780243224:..." \ + --base-url http://127.0.0.1:8081/bot \ + --base-file-url http://127.0.0.1:8081/file/bot + +With BotFather privacy enabled, supergroup bots only receive commands, replies +to the bot, mentions, or messages otherwise visible to bots. In a group, send: + + /ping hello + +The same program can also send proactive messages: + + python cmd/bots/ptbecho/echo.py \ + --token "1780243224:..." \ + --send-only \ + --send-chat-id -1000000000002 \ + --send-text "hello from python-telegram-bot" +""" + +import argparse +import asyncio +import logging +import os +import signal +from typing import Iterable + +from telegram import Bot, Update +from telegram.constants import ChatAction +from telegram.ext import ( + Application, + ApplicationBuilder, + CommandHandler, + ContextTypes, + MessageHandler, + filters, +) + + +LOG = logging.getLogger("ptbecho") + + +def env_int(name: str) -> int | None: + raw = os.getenv(name) + if raw is None or raw == "": + return None + try: + return int(raw) + except ValueError as exc: + raise SystemExit(f"{name} must be an integer, got {raw!r}") from exc + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Echo bot demo using python-telegram-bot against telesrv.") + parser.add_argument("--token", default=os.getenv("TELESRV_BOT_TOKEN"), help="Bot token, defaults to TELESRV_BOT_TOKEN") + parser.add_argument("--base-url", default=os.getenv("TELESRV_BOT_API_BASE_URL", "http://127.0.0.1:8081/bot")) + parser.add_argument("--base-file-url", default=os.getenv("TELESRV_BOT_API_BASE_FILE_URL", "http://127.0.0.1:8081/file/bot")) + parser.add_argument("--prefix", default="echo: ") + parser.add_argument("--drop-pending", action="store_true", help="Drop pending updates before polling") + parser.add_argument("--timeout", type=int, default=30, help="getUpdates long-poll timeout seconds") + parser.add_argument( + "--send-chat-id", + type=int, + default=env_int("TELESRV_BOT_DEMO_CHAT_ID"), + help="Chat id for proactive sendMessage, defaults to TELESRV_BOT_DEMO_CHAT_ID", + ) + parser.add_argument( + "--send-text", + default=os.getenv("TELESRV_BOT_DEMO_SEND_TEXT", ""), + help="Text for proactive sendMessage, defaults to TELESRV_BOT_DEMO_SEND_TEXT", + ) + parser.add_argument("--send-count", type=int, default=1, help="Number of proactive messages to send") + parser.add_argument("--send-interval", type=float, default=1.0, help="Seconds between proactive sends") + parser.add_argument("--send-only", action="store_true", help="Send proactive messages and exit without polling") + parser.add_argument("--log-level", default="INFO") + args = parser.parse_args() + if not args.token: + parser.error("missing --token or TELESRV_BOT_TOKEN") + if args.send_count < 1: + parser.error("--send-count must be >= 1") + if args.send_interval < 0: + parser.error("--send-interval must be >= 0") + wants_send = args.send_only or bool(args.send_text) + if wants_send and args.send_chat_id is None: + parser.error("--send-chat-id is required when --send-text or --send-only is used") + if args.send_only and not args.send_text: + parser.error("--send-only requires --send-text") + return args + + +async def start(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + if update.effective_message is None: + return + await update.effective_message.reply_text("send /ping in a group, or any text in private chat") + + +async def ping(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + await echo(update, context) + + +async def echo(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + if update.effective_message is None or update.effective_chat is None: + return + text = update.effective_message.text or update.effective_message.caption or "" + if not text: + return + prefix = context.application.bot_data.get("prefix", "echo: ") + await context.bot.send_chat_action(chat_id=update.effective_chat.id, action=ChatAction.TYPING) + sent = await update.effective_message.reply_text(prefix + text) + LOG.info( + "echoed update_id=%s chat_id=%s message_id=%s sent_message_id=%s text=%r", + update.update_id, + update.effective_chat.id, + update.effective_message.message_id, + sent.message_id, + text, + ) + + +async def send_active_messages(bot: Bot, chat_id: int, text: str, count: int, interval: float) -> None: + for index in range(count): + sent = await bot.send_message(chat_id=chat_id, text=text) + LOG.info( + "sent proactive index=%s/%s chat_id=%s message_id=%s text=%r", + index + 1, + count, + chat_id, + sent.message_id, + text, + ) + if index + 1 < count: + await asyncio.sleep(interval) + + +async def send_on_startup(app: Application) -> None: + chat_id = app.bot_data.get("send_chat_id") + text = app.bot_data.get("send_text") + if chat_id is None or not text: + return + await send_active_messages( + app.bot, + chat_id=chat_id, + text=text, + count=int(app.bot_data.get("send_count", 1)), + interval=float(app.bot_data.get("send_interval", 1.0)), + ) + + +async def post_init(app: Application) -> None: + me = await app.bot.get_me() + LOG.info("listening as @%s (%s), bot_api=%s", me.username or me.id, me.id, app.bot_data["base_url"]) + if app.bot_data.get("send_chat_id") is not None and app.bot_data.get("send_text"): + app.create_task(send_on_startup(app), name="ptbecho-proactive-send") + + +def build_app(args: argparse.Namespace) -> Application: + app = ( + ApplicationBuilder() + .token(args.token) + .base_url(args.base_url) + .base_file_url(args.base_file_url) + .post_init(post_init) + .build() + ) + app.bot_data["prefix"] = args.prefix + app.bot_data["base_url"] = args.base_url + app.bot_data["send_chat_id"] = args.send_chat_id + app.bot_data["send_text"] = args.send_text + app.bot_data["send_count"] = args.send_count + app.bot_data["send_interval"] = args.send_interval + app.add_handler(CommandHandler("start", start)) + app.add_handler(CommandHandler("ping", ping)) + app.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, echo)) + return app + + +async def run_send_only(args: argparse.Namespace) -> None: + bot = Bot(token=args.token, base_url=args.base_url, base_file_url=args.base_file_url) + me = await bot.get_me() + LOG.info("authenticated as @%s (%s), bot_api=%s", me.username or me.id, me.id, args.base_url) + await send_active_messages( + bot, + chat_id=args.send_chat_id, + text=args.send_text, + count=args.send_count, + interval=args.send_interval, + ) + + +def stop_signals() -> Iterable[int]: + if os.name == "nt": + return (signal.SIGINT, signal.SIGTERM) + return (signal.SIGINT, signal.SIGTERM, signal.SIGHUP) + + +def main() -> int: + args = parse_args() + logging.basicConfig( + level=getattr(logging, args.log_level.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + logging.getLogger("httpx").setLevel(logging.WARNING) + logging.getLogger("httpcore").setLevel(logging.WARNING) + if args.send_only: + asyncio.run(run_send_only(args)) + return 0 + + app = build_app(args) + app.run_polling( + allowed_updates=["message", "edited_message"], + drop_pending_updates=args.drop_pending, + poll_interval=0.0, + timeout=args.timeout, + stop_signals=stop_signals(), + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/cmd/telesrv/main.go b/cmd/telesrv/main.go index 94568ae7..8d8359a8 100644 --- a/cmd/telesrv/main.go +++ b/cmd/telesrv/main.go @@ -318,6 +318,7 @@ func run(logger *zap.Logger) error { readModelVersionStore := storepkg.NewCachedReadModelVersionStore(postgres.NewReadModelVersionStore(pool), 0, 0) dispatchOutboxStore := postgres.NewDispatchOutboxStore(pool, postgres.WithLeaseTimeout(cfg.OutboxLeaseTimeout)) bootstrapUpdateStore := postgres.NewBootstrapUpdateJobStore(pool) + botAPIUpdateStore := postgres.NewBotAPIUpdateStore(pool) boxIDAllocator := redisstore.NewBoxIDAllocator(rdb, postgres.NewMessageBoxCounterSource(pool)) channelIDAllocator := redisstore.NewChannelIDAllocator(rdb, postgres.NewChannelIDCounterSource(pool)) channelMessageIDAllocator := redisstore.NewChannelMessageIDAllocator(rdb, postgres.NewChannelMessageIDCounterSource(pool)) @@ -432,7 +433,7 @@ func run(logger *zap.Logger) error { cfg.UpdateEventRetention, cfg.RetentionInterval, cfg.RetentionBatch, - ).Run(ctx) + ).WithBotAPIUpdateRetention(botAPIUpdateStore, cfg.BotAPIUpdateRetention).Run(ctx) go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"), cfg.UploadPartTTL, cfg.UploadPartGCInterval, @@ -672,6 +673,7 @@ func run(logger *zap.Logger) error { Users: usersService, Updates: updatesService, BootstrapUpdates: bootstrapUpdateStore, + BotAPIUpdates: botAPIUpdateStore, Contacts: contactsService, Dialogs: dialogsService, Chatlists: chatlistsService, @@ -708,6 +710,7 @@ func run(logger *zap.Logger) error { ProfilePhotos: cachedPhotos, Stories: router, ChannelFullBots: router, + ChannelBotMembers: channelsService, ChannelMediaCounts: channelsService, PrivateMediaCounts: messagesService, RPCProjections: router, @@ -744,11 +747,12 @@ func run(logger *zap.Logger) error { go rpc.NewPhoneExpiryDispatcher(router, logger.Named("rpc").Named("phone-expiry"), cfg.CallExpiryInterval).Run(ctx) go rpc.NewGroupCallSweepDispatcher(router, logger.Named("rpc").Named("groupcall-sweep"), cfg.GroupCallSweepInterval, cfg.GroupCallCheckTTL).Run(ctx) go router.RunChannelFanout(ctx) + go router.RunBotAPIEnqueue(ctx) go router.RunPresenceSweeper(ctx, time.Minute) go activeSessions.RunPendingSweeper(ctx, time.Minute) go router.RunPremiumSweeper(ctx, cfg.PremiumSweepInterval, cfg.PremiumSweepBatch) go router.RunInlineBotPushSubscriber(ctx) - if _, err := botapi.Start(ctx, cfg.BotAPIAddr, botsService, usersService, router, logger.Named("botapi")); err != nil { + if _, err := botapi.Start(ctx, cfg.BotAPIAddr, botsService, usersService, router, router, logger.Named("botapi")); err != nil { return fmt.Errorf("start bot api: %w", err) } if _, err := adminapi.Start(ctx, adminapi.Config{Addr: cfg.AdminAPIAddr, Token: cfg.AdminAPIToken}, adminService, logger.Named("adminapi")); err != nil { @@ -757,6 +761,7 @@ func run(logger *zap.Logger) error { if _, err := stickerlinks.Start(ctx, stickerlinks.Config{ Addr: cfg.PublicLinkWebAddr, PublicBaseURL: cfg.PublicBaseURL, + Users: userStore, }, filesService, logger.Named("stickerlinks")); err != nil { return fmt.Errorf("start sticker links: %w", err) } diff --git a/deploy/migrations/0060_bot_api_updates.down.sql b/deploy/migrations/0060_bot_api_updates.down.sql new file mode 100644 index 00000000..d4f93e0d --- /dev/null +++ b/deploy/migrations/0060_bot_api_updates.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS public.bot_api_update_states; +DROP TABLE IF EXISTS public.bot_api_updates; diff --git a/deploy/migrations/0060_bot_api_updates.up.sql b/deploy/migrations/0060_bot_api_updates.up.sql new file mode 100644 index 00000000..7427f176 --- /dev/null +++ b/deploy/migrations/0060_bot_api_updates.up.sql @@ -0,0 +1,29 @@ +CREATE TABLE public.bot_api_updates ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + bot_user_id bigint NOT NULL, + update_kind varchar(32) NOT NULL, + peer_type varchar(16) NOT NULL, + peer_id bigint NOT NULL, + message_id integer NOT NULL, + source_pts integer NOT NULL DEFAULT 0, + date integer NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT bot_api_updates_kind_check CHECK ((update_kind)::text = ANY (ARRAY['message'::text, 'edited_message'::text])), + CONSTRAINT bot_api_updates_peer_type_check CHECK ((peer_type)::text = ANY (ARRAY['user'::text, 'channel'::text])), + CONSTRAINT bot_api_updates_peer_id_check CHECK (peer_id > 0), + CONSTRAINT bot_api_updates_message_id_check CHECK (message_id > 0), + CONSTRAINT bot_api_updates_source_pts_check CHECK (source_pts >= 0), + CONSTRAINT bot_api_updates_source_unique UNIQUE (bot_user_id, update_kind, peer_type, peer_id, message_id, source_pts), + CONSTRAINT bot_api_updates_bot_user_id_fkey FOREIGN KEY (bot_user_id) REFERENCES public.bots(bot_user_id) ON DELETE CASCADE +); + +CREATE INDEX bot_api_updates_bot_scan_idx ON public.bot_api_updates (bot_user_id, id); +CREATE INDEX bot_api_updates_retention_idx ON public.bot_api_updates (date, id); + +CREATE TABLE public.bot_api_update_states ( + bot_user_id bigint PRIMARY KEY, + confirmed_update_id bigint NOT NULL DEFAULT 0, + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT bot_api_update_states_confirmed_check CHECK (confirmed_update_id >= 0), + CONSTRAINT bot_api_update_states_bot_user_id_fkey FOREIGN KEY (bot_user_id) REFERENCES public.bots(bot_user_id) ON DELETE CASCADE +); diff --git a/internal/app/channels/bot_member_ids_cache.go b/internal/app/channels/bot_member_ids_cache.go new file mode 100644 index 00000000..6eff1902 --- /dev/null +++ b/internal/app/channels/bot_member_ids_cache.go @@ -0,0 +1,95 @@ +package channels + +import ( + "context" + "encoding/binary" + "hash/fnv" + + "telesrv/internal/app/readmodel" + "telesrv/internal/domain" + "telesrv/internal/readmodelcache" + "telesrv/internal/store" +) + +const ( + activeBotMemberIDsMaxEntries = 8192 +) + +type activeBotMemberIDsCacheKey struct { + viewerUserID int64 + channelID int64 + limit int +} + +type activeBotMemberIDsCache struct { + cache *readmodelcache.Cache[activeBotMemberIDsCacheKey, []int64] +} + +func newActiveBotMemberIDsCache() *activeBotMemberIDsCache { + return &activeBotMemberIDsCache{ + cache: readmodelcache.New[activeBotMemberIDsCacheKey, []int64](readmodelcache.Config[activeBotMemberIDsCacheKey, []int64]{ + MaxEntries: activeBotMemberIDsMaxEntries, + Clone: cloneInt64s, + }), + } +} + +func (c *activeBotMemberIDsCache) getOrLoad(ctx context.Context, key activeBotMemberIDsCacheKey, load func() ([]int64, error)) ([]int64, error) { + if c == nil { + return load() + } + return c.cache.GetOrLoad(ctx, key, load) +} + +func (c *activeBotMemberIDsCache) getOrLoadVersioned(ctx context.Context, key activeBotMemberIDsCacheKey, hash int64, load func() ([]int64, error)) ([]int64, error) { + if c == nil { + return load() + } + return c.cache.GetOrLoadVersioned(ctx, key, hash, load) +} + +func (c *activeBotMemberIDsCache) invalidateChannel(channelID int64) { + if c == nil || channelID == 0 { + return + } + c.cache.InvalidateWhere(func(key activeBotMemberIDsCacheKey) bool { + return key.channelID == channelID + }) +} + +func (c *activeBotMemberIDsCache) flush() { + if c == nil { + return + } + c.cache.Flush() +} + +func (s *Service) channelBotMemberIDsHash(ctx context.Context, viewerUserID, channelID int64, key activeBotMemberIDsCacheKey) (int64, error) { + keys := []store.ReadModelKey{ + {Model: readmodel.ModelChannelBase, OwnerUserID: 0, PeerType: domain.PeerTypeChannel, PeerID: channelID}, + {Model: readmodel.ModelChannelParticipants, OwnerUserID: 0, PeerType: domain.PeerTypeChannel, PeerID: channelID}, + {Model: readmodel.ModelChannelMember, OwnerUserID: viewerUserID, PeerType: domain.PeerTypeChannel, PeerID: channelID}, + } + rows, err := s.versions.ReadModelHashes(ctx, keys) + if err != nil { + return 0, err + } + base := rows[keys[0]] + participants := rows[keys[1]] + if base == 0 || participants == 0 { + return 0, nil + } + return readmodel.MixHashes(base, participants, rows[keys[2]], botMemberIDsKeyHash(key)), nil +} + +func botMemberIDsKeyHash(key activeBotMemberIDsCacheKey) int64 { + h := fnv.New64a() + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], uint64(key.limit)) + _, _ = h.Write(buf[:]) + sum := int64(h.Sum64() & 0x7fffffffffffffff) + if sum == 0 { + return 1 + } + return sum +} diff --git a/internal/app/channels/bot_policy.go b/internal/app/channels/bot_policy.go index 5ecd78c3..878e55fb 100644 --- a/internal/app/channels/bot_policy.go +++ b/internal/app/channels/bot_policy.go @@ -106,20 +106,72 @@ func (s *Service) skippedBotDeliveryUserIDs(ctx context.Context, req domain.Send if s.bots == nil || req.ChannelID == 0 || req.UserID == 0 { return nil, nil } - if lister, ok := s.channels.(activeChannelBotMemberIDLister); ok { - memberIDs, err := lister.ListActiveChannelBotMemberIDs(ctx, req.UserID, req.ChannelID, domain.MaxSynchronousChannelDialogFanout) - if err != nil { - return nil, err - } - return s.skippedBotDeliveryUserIDsForIDs(ctx, req, memberIDs) - } - memberIDs, err := s.channels.ListActiveChannelMemberIDs(ctx, req.UserID, req.ChannelID, domain.MaxSynchronousChannelDialogFanout) + memberIDs, err := s.loadActiveBotMemberIDs(ctx, req.UserID, req.ChannelID, domain.MaxSynchronousChannelDialogFanout) if err != nil { return nil, err } return s.skippedBotDeliveryUserIDsForIDs(ctx, req, memberIDs) } +// ActiveBotMemberIDs returns active bot members for non-privacy-critical producers +// such as Bot API update queue fanout. Privacy delivery decisions use +// loadActiveBotMemberIDs directly to avoid stale-cache leaks. +func (s *Service) ActiveBotMemberIDs(ctx context.Context, viewerUserID, channelID int64, limit int) ([]int64, error) { + if s == nil || s.channels == nil || viewerUserID == 0 || channelID == 0 { + return nil, domain.ErrChannelInvalid + } + if limit <= 0 || limit > domain.MaxSynchronousChannelDialogFanout { + limit = domain.MaxSynchronousChannelDialogFanout + } + key := activeBotMemberIDsCacheKey{viewerUserID: viewerUserID, channelID: channelID, limit: limit} + if s.versions != nil { + hash, err := s.channelBotMemberIDsHash(ctx, viewerUserID, channelID, key) + if err != nil { + return nil, err + } + if hash != 0 { + return s.botMemberIDsCache.getOrLoadVersioned(ctx, key, hash, func() ([]int64, error) { + return s.loadActiveBotMemberIDs(ctx, viewerUserID, channelID, limit) + }) + } + return s.loadActiveBotMemberIDs(ctx, viewerUserID, channelID, limit) + } + return s.botMemberIDsCache.getOrLoad(ctx, key, func() ([]int64, error) { + return s.loadActiveBotMemberIDs(ctx, viewerUserID, channelID, limit) + }) +} + +func (s *Service) loadActiveBotMemberIDs(ctx context.Context, viewerUserID, channelID int64, limit int) ([]int64, error) { + if s == nil || s.channels == nil || viewerUserID == 0 || channelID == 0 { + return nil, domain.ErrChannelInvalid + } + if limit <= 0 || limit > domain.MaxSynchronousChannelDialogFanout { + limit = domain.MaxSynchronousChannelDialogFanout + } + if lister, ok := s.channels.(activeChannelBotMemberIDLister); ok { + return lister.ListActiveChannelBotMemberIDs(ctx, viewerUserID, channelID, limit) + } + if s.bots == nil { + return nil, nil + } + memberIDs, err := s.channels.ListActiveChannelMemberIDs(ctx, viewerUserID, channelID, limit) + if err != nil { + return nil, err + } + profiles, err := s.botProfiles(ctx, memberIDs) + if err != nil { + return nil, err + } + out := make([]int64, 0, len(profiles)) + for _, id := range uniqueNonZero(memberIDs) { + if _, found := profiles[id]; found { + out = append(out, id) + } + } + sort.Slice(out, func(i, j int) bool { return out[i] < out[j] }) + return out, nil +} + func (s *Service) skippedBotDeliveryUserIDsForIDs(ctx context.Context, req domain.SendChannelMessageRequest, memberIDs []int64) ([]int64, error) { profiles, err := s.botProfiles(ctx, memberIDs) if err != nil { diff --git a/internal/app/channels/service.go b/internal/app/channels/service.go index baf16920..7f075b7b 100644 --- a/internal/app/channels/service.go +++ b/internal/app/channels/service.go @@ -17,11 +17,12 @@ type Service struct { versions store.ReadModelVersionStore sendGate SendPermissionChecker - viewCache *channelViewReadModelCache - resolveCache *channelResolveReadModelCache - mediaCountCache *mediaCountReadModelCache - participantCache *participantsReadModelCache - activeIDsCache *activeChannelIDsReadModelCache + viewCache *channelViewReadModelCache + resolveCache *channelResolveReadModelCache + mediaCountCache *mediaCountReadModelCache + participantCache *participantsReadModelCache + activeIDsCache *activeChannelIDsReadModelCache + botMemberIDsCache *activeBotMemberIDsCache } type Option func(*Service) @@ -33,12 +34,13 @@ type SendPermissionChecker interface { // NewService creates a channel service. func NewService(channels store.ChannelStore, opts ...Option) *Service { s := &Service{ - channels: channels, - viewCache: newChannelViewReadModelCache(defaultChannelViewReadModelTTL), - resolveCache: newChannelResolveReadModelCache(defaultChannelResolveReadModelTTL), - mediaCountCache: newMediaCountReadModelCache(defaultMediaCountReadModelTTL), - participantCache: newParticipantsReadModelCache(defaultParticipantsReadModelTTL), - activeIDsCache: newActiveChannelIDsReadModelCache(defaultActiveChannelIDsReadModelTTL), + channels: channels, + viewCache: newChannelViewReadModelCache(defaultChannelViewReadModelTTL), + resolveCache: newChannelResolveReadModelCache(defaultChannelResolveReadModelTTL), + mediaCountCache: newMediaCountReadModelCache(defaultMediaCountReadModelTTL), + participantCache: newParticipantsReadModelCache(defaultParticipantsReadModelTTL), + activeIDsCache: newActiveChannelIDsReadModelCache(defaultActiveChannelIDsReadModelTTL), + botMemberIDsCache: newActiveBotMemberIDsCache(), } for _, opt := range opts { opt(s) @@ -238,6 +240,7 @@ func (s *Service) InviteToChannel(ctx context.Context, userID, channelID int64, if err == nil { s.invalidateActiveChannelIDs(activeMembershipUserIDsFromMembers(0, res.Members)...) s.participantCache.invalidateChannel(channelID) + s.invalidateActiveBotMemberIDs(channelID) } return res, err } @@ -251,6 +254,7 @@ func (s *Service) JoinChannel(ctx context.Context, userID, channelID int64, date if err == nil { s.invalidateActiveChannelIDs(userID) s.participantCache.invalidateChannel(channelID) + s.invalidateActiveBotMemberIDs(channelID) } return res, err } @@ -264,6 +268,7 @@ func (s *Service) LeaveChannel(ctx context.Context, userID, channelID int64, dat if err == nil { s.invalidateActiveChannelIDs(userID) s.participantCache.invalidateChannel(channelID) + s.invalidateActiveBotMemberIDs(channelID) } return res, err } @@ -325,6 +330,7 @@ func (s *Service) EditAdmin(ctx context.Context, userID int64, req domain.EditCh if err == nil { s.invalidateActiveChannelIDs(req.MemberID) s.participantCache.invalidateChannel(req.ChannelID) + s.invalidateActiveBotMemberIDs(req.ChannelID) } return res, err } @@ -344,6 +350,7 @@ func (s *Service) TransferOwnership(ctx context.Context, userID int64, req domai if err == nil { s.invalidateActiveChannelIDs(req.UserID, req.NewOwnerID) s.participantCache.invalidateChannel(req.ChannelID) + s.invalidateActiveBotMemberIDs(req.ChannelID) } return res, err } @@ -363,6 +370,7 @@ func (s *Service) EditMemberRank(ctx context.Context, userID int64, req domain.E res, err := s.channels.EditChannelMemberRank(ctx, req) if err == nil { s.participantCache.invalidateChannel(req.ChannelID) + s.invalidateActiveBotMemberIDs(req.ChannelID) } return res, err } @@ -382,6 +390,7 @@ func (s *Service) EditBanned(ctx context.Context, userID int64, req domain.EditC if err == nil { s.invalidateActiveChannelIDs(req.Participant.ID) s.participantCache.invalidateChannel(req.ChannelID) + s.invalidateActiveBotMemberIDs(req.ChannelID) } return res, err } @@ -414,6 +423,7 @@ func (s *Service) DeleteChannel(ctx context.Context, userID int64, req domain.De res, err := s.channels.DeleteChannel(ctx, req) if err == nil { s.invalidateActiveChannelIDs(uniqueUserIDs(append([]int64{userID}, res.Recipients...)...)...) + s.invalidateActiveBotMemberIDs(req.ChannelID) } return res, err } @@ -2124,6 +2134,24 @@ func (s *Service) invalidateActiveChannelIDs(userIDs ...int64) { s.activeIDsCache.invalidateUsers(userIDs...) } +func (s *Service) invalidateActiveBotMemberIDs(channelID int64) { + if s == nil || s.botMemberIDsCache == nil { + return + } + s.botMemberIDsCache.invalidateChannel(channelID) +} + +func (s *Service) InvalidateActiveBotMemberIDsReadModel(channelID int64) { + s.invalidateActiveBotMemberIDs(channelID) +} + +func (s *Service) FlushActiveBotMemberIDsReadModel() { + if s == nil || s.botMemberIDsCache == nil { + return + } + s.botMemberIDsCache.flush() +} + func normalizeChannelUsername(username string) string { username = strings.TrimSpace(username) username = strings.TrimPrefix(username, "@") diff --git a/internal/app/channels/service_test.go b/internal/app/channels/service_test.go index 265caf9e..e463eca2 100644 --- a/internal/app/channels/service_test.go +++ b/internal/app/channels/service_test.go @@ -57,15 +57,16 @@ func (p testBotProfiles) BotInfo(_ context.Context, botUserID int64) (domain.Bot type countingChannelStore struct { *memory.ChannelStore - mu sync.Mutex - getChannelCalls int - resolveChannelCalls int - countMediaCalls int - getParticipantCalls int - listActiveIDsCalls int - resolveStarted chan struct{} - resolveRelease <-chan struct{} - resolveStartOnce sync.Once + mu sync.Mutex + getChannelCalls int + resolveChannelCalls int + countMediaCalls int + getParticipantCalls int + listActiveIDsCalls int + listActiveMemberIDsCalls int + resolveStarted chan struct{} + resolveRelease <-chan struct{} + resolveStartOnce sync.Once } func (s *countingChannelStore) GetChannel(ctx context.Context, viewerUserID, channelID int64) (domain.ChannelView, error) { @@ -102,6 +103,11 @@ func (s *countingChannelStore) ListActiveChannelIDsForUser(ctx context.Context, return s.ChannelStore.ListActiveChannelIDsForUser(ctx, userID, afterChannelID, limit) } +func (s *countingChannelStore) ListActiveChannelMemberIDs(ctx context.Context, viewerUserID, channelID int64, limit int) ([]int64, error) { + s.listActiveMemberIDsCalls++ + return s.ChannelStore.ListActiveChannelMemberIDs(ctx, viewerUserID, channelID, limit) +} + type fakeReadModelVersions struct { hashes map[store.ReadModelKey]int64 } @@ -343,6 +349,102 @@ func TestActiveChannelIDsForUserCachesEmptyMissingReadModelHash(t *testing.T) { } } +func TestActiveBotMemberIDsCachesAndInvalidatesOnMembershipWrite(t *testing.T) { + ctx := context.Background() + base := &countingChannelStore{ChannelStore: memory.NewChannelStore()} + bots := testBotProfiles{ + 1003: {BotUserID: 1003}, + 1004: {BotUserID: 1004}, + } + service := NewService(base, WithBotProfileResolver(bots)) + created, err := service.CreateMegagroupFromCreateChat(ctx, 1001, domain.CreateChannelRequest{ + Title: "Bot Cache", + MemberUserIDs: []int64{1002, 1003}, + Date: 1700004115, + }) + if err != nil { + t.Fatalf("CreateMegagroupFromCreateChat: %v", err) + } + channelID := created.Channel.ID + + first, err := service.ActiveBotMemberIDs(ctx, 1001, channelID, domain.MaxSynchronousChannelDialogFanout) + if err != nil { + t.Fatalf("ActiveBotMemberIDs first: %v", err) + } + if want := []int64{1003}; !slices.Equal(first, want) { + t.Fatalf("ActiveBotMemberIDs first = %v, want %v", first, want) + } + first[0] = 9999 + second, err := service.ActiveBotMemberIDs(ctx, 1001, channelID, domain.MaxSynchronousChannelDialogFanout) + if err != nil { + t.Fatalf("ActiveBotMemberIDs second: %v", err) + } + if want := []int64{1003}; !slices.Equal(second, want) { + t.Fatalf("ActiveBotMemberIDs cached = %v, want %v", second, want) + } + if base.listActiveMemberIDsCalls != 1 { + t.Fatalf("ListActiveChannelMemberIDs calls = %d, want 1 after cache hit", base.listActiveMemberIDsCalls) + } + + if _, err := service.InviteToChannel(ctx, 1001, channelID, []int64{1004}, 1700004116); err != nil { + t.Fatalf("InviteToChannel bot: %v", err) + } + third, err := service.ActiveBotMemberIDs(ctx, 1001, channelID, domain.MaxSynchronousChannelDialogFanout) + if err != nil { + t.Fatalf("ActiveBotMemberIDs after invite: %v", err) + } + if want := []int64{1003, 1004}; !slices.Equal(third, want) { + t.Fatalf("ActiveBotMemberIDs after invite = %v, want %v", third, want) + } + if base.listActiveMemberIDsCalls != 2 { + t.Fatalf("ListActiveChannelMemberIDs calls = %d, want reload after invalidation", base.listActiveMemberIDsCalls) + } +} + +func TestActiveBotMemberIDsReloadsOnReadModelHashChange(t *testing.T) { + ctx := context.Background() + base := &countingChannelStore{ChannelStore: memory.NewChannelStore()} + bots := testBotProfiles{1003: {BotUserID: 1003}} + creator := NewService(base, WithBotProfileResolver(bots)) + created, err := creator.CreateMegagroupFromCreateChat(ctx, 1001, domain.CreateChannelRequest{ + Title: "Bot Hash", + MemberUserIDs: []int64{1002, 1003}, + Date: 1700004117, + }) + if err != nil { + t.Fatalf("CreateMegagroupFromCreateChat: %v", err) + } + channelID := created.Channel.ID + peer := domain.Peer{Type: domain.PeerTypeChannel, ID: channelID} + baseKey := store.ReadModelKey{Model: readmodel.ModelChannelBase, OwnerUserID: 0, PeerType: peer.Type, PeerID: peer.ID} + participantsKey := store.ReadModelKey{Model: readmodel.ModelChannelParticipants, OwnerUserID: 0, PeerType: peer.Type, PeerID: peer.ID} + memberKey := store.ReadModelKey{Model: readmodel.ModelChannelMember, OwnerUserID: 1001, PeerType: peer.Type, PeerID: peer.ID} + versions := &fakeReadModelVersions{hashes: map[store.ReadModelKey]int64{ + baseKey: 401, + participantsKey: 402, + memberKey: 403, + }} + service := NewService(base, WithBotProfileResolver(bots), WithReadModelVersions(versions)) + + if _, err := service.ActiveBotMemberIDs(ctx, 1001, channelID, domain.MaxSynchronousChannelDialogFanout); err != nil { + t.Fatalf("ActiveBotMemberIDs first: %v", err) + } + if _, err := service.ActiveBotMemberIDs(ctx, 1001, channelID, domain.MaxSynchronousChannelDialogFanout); err != nil { + t.Fatalf("ActiveBotMemberIDs cached: %v", err) + } + if base.listActiveMemberIDsCalls != 1 { + t.Fatalf("ListActiveChannelMemberIDs calls = %d, want 1 before hash change", base.listActiveMemberIDsCalls) + } + + versions.hashes[participantsKey] = 404 + if _, err := service.ActiveBotMemberIDs(ctx, 1001, channelID, domain.MaxSynchronousChannelDialogFanout); err != nil { + t.Fatalf("ActiveBotMemberIDs after hash change: %v", err) + } + if base.listActiveMemberIDsCalls != 2 { + t.Fatalf("ListActiveChannelMemberIDs calls = %d, want reload after hash change", base.listActiveMemberIDsCalls) + } +} + func TestActiveChannelIDsCacheInvalidatesOnMembershipWrite(t *testing.T) { ctx := context.Background() const ownerID int64 = 1001 diff --git a/internal/app/maintenance/retention.go b/internal/app/maintenance/retention.go index b668c7c6..fdbb54a8 100644 --- a/internal/app/maintenance/retention.go +++ b/internal/app/maintenance/retention.go @@ -17,6 +17,16 @@ type TempAuthKeyRetentionStore interface { DeleteExpired(ctx context.Context, expiredBefore int64, limit int) (int, error) } +// BotAPIUpdateRetentionStore 回收 Bot API getUpdates 投递队列的死行（性能审计 H1）： +// 已确认且超过宽限期的行 + 按消息 date 超过保留期的行（官方 Bot API updates 最多保留 24h）。 +type BotAPIUpdateRetentionStore interface { + DeleteDeliveredOrExpired(ctx context.Context, confirmedGrace, maxAge time.Duration, limit int) (int, error) +} + +// botAPIConfirmedGrace 是已确认 Bot API update 行的删除宽限：确认水位之下的行不会再被 +// getUpdates 读取（fromID 恒 > confirmed），宽限仅防御 offset 回拨调试；回收目标是清堆积。 +const botAPIConfirmedGrace = 15 * time.Minute + // tempAuthKeyExpiryGrace 是 temp key 过期后的回收宽限：ResolveAuthKey 对 // 「已过期但 perm 已授权」的绑定是容忍的，立即删除会突然断掉这批宽限中的 // 连接；回收目标是清堆积，晚一天无妨。 @@ -31,12 +41,14 @@ const tempAuthKeyExpiryGrace = 24 * time.Hour // 丢消息。详见 docs/performance-audit.md 与 docs/compatibility-matrix.md。user_update_events // 长期膨胀作为已知 todo。 type RetentionWorker struct { - outbox DispatchOutboxRetentionStore - tempKeys TempAuthKeyRetentionStore // 可为 nil（不回收 temp key 绑定） - logger *zap.Logger - retention time.Duration - interval time.Duration - batch int + outbox DispatchOutboxRetentionStore + tempKeys TempAuthKeyRetentionStore // 可为 nil（不回收 temp key 绑定） + botAPIUpdates BotAPIUpdateRetentionStore // 可为 nil（不回收 Bot API 队列） + logger *zap.Logger + retention time.Duration + botAPIRetention time.Duration + interval time.Duration + batch int } func NewRetentionWorker(outbox DispatchOutboxRetentionStore, tempKeys TempAuthKeyRetentionStore, logger *zap.Logger, retention, interval time.Duration, batch int) *RetentionWorker { @@ -62,6 +74,16 @@ func NewRetentionWorker(outbox DispatchOutboxRetentionStore, tempKeys TempAuthKe } } +// WithBotAPIUpdateRetention 启用 bot_api_updates 队列回收；retention <=0 时用官方语义默认 24h。 +func (w *RetentionWorker) WithBotAPIUpdateRetention(store BotAPIUpdateRetentionStore, retention time.Duration) *RetentionWorker { + if retention <= 0 { + retention = 24 * time.Hour + } + w.botAPIUpdates = store + w.botAPIRetention = retention + return w +} + func (w *RetentionWorker) Run(ctx context.Context) { w.runOnce(ctx) ticker := time.NewTicker(w.interval) @@ -92,4 +114,12 @@ func (w *RetentionWorker) runOnce(ctx context.Context) { w.logger.Info("回收过期 temp auth key 绑定完成", zap.Int("deleted", tempDeleted)) } } + if w.botAPIUpdates != nil { + botAPIDeleted, err := w.botAPIUpdates.DeleteDeliveredOrExpired(ctx, botAPIConfirmedGrace, w.botAPIRetention, w.batch) + if err != nil { + w.logger.Warn("回收 bot_api_updates 队列失败", zap.Error(err)) + } else if botAPIDeleted > 0 { + w.logger.Info("回收 bot_api_updates 队列完成", zap.Int("deleted", botAPIDeleted)) + } + } } diff --git a/internal/app/maintenance/retention_test.go b/internal/app/maintenance/retention_test.go index ca34aaba..14da5fdd 100644 --- a/internal/app/maintenance/retention_test.go +++ b/internal/app/maintenance/retention_test.go @@ -57,3 +57,45 @@ func TestRetentionWorkerSkipsNilTempKeyStore(t *testing.T) { t.Fatalf("outbox calls = %d, want 1", outbox.calls) } } + +type fakeBotAPIRetention struct { + calls int + confirmedGrace time.Duration + maxAge time.Duration + limit int +} + +func (f *fakeBotAPIRetention) DeleteDeliveredOrExpired(_ context.Context, confirmedGrace, maxAge time.Duration, limit int) (int, error) { + f.calls++ + f.confirmedGrace = confirmedGrace + f.maxAge = maxAge + f.limit = limit + return 5, nil +} + +func TestRetentionWorkerReclaimsBotAPIUpdates(t *testing.T) { + outbox := &fakeOutboxRetention{} + botAPI := &fakeBotAPIRetention{} + w := NewRetentionWorker(outbox, nil, zap.NewNop(), time.Hour, time.Hour, 100). + WithBotAPIUpdateRetention(botAPI, 24*time.Hour) + + w.runOnce(context.Background()) + + if botAPI.calls != 1 { + t.Fatalf("bot api retention calls = %d, want 1", botAPI.calls) + } + if botAPI.confirmedGrace != botAPIConfirmedGrace || botAPI.maxAge != 24*time.Hour || botAPI.limit != 100 { + t.Fatalf("bot api retention args = (%v, %v, %d), want (%v, 24h, 100)", + botAPI.confirmedGrace, botAPI.maxAge, botAPI.limit, botAPIConfirmedGrace) + } +} + +func TestRetentionWorkerBotAPIRetentionDefaultsTo24h(t *testing.T) { + botAPI := &fakeBotAPIRetention{} + w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 100). + WithBotAPIUpdateRetention(botAPI, 0) + w.runOnce(context.Background()) + if botAPI.maxAge != 24*time.Hour { + t.Fatalf("default bot api retention = %v, want 24h", botAPI.maxAge) + } +} diff --git a/internal/app/updates/service.go b/internal/app/updates/service.go index 0b0a7c21..7ad972eb 100644 --- a/internal/app/updates/service.go +++ b/internal/app/updates/service.go @@ -99,6 +99,20 @@ func (s *Service) CurrentState(ctx context.Context, userID int64) (domain.Update return s.currentState(ctx, userID) } +// ConfirmedState returns the device-local confirmed update state, if any, +// without bootstrapping it to the account-current pts. +func (s *Service) ConfirmedState(ctx context.Context, authKeyID [8]byte, userID int64) (domain.UpdateState, bool, error) { + if s == nil || s.states == nil { + return domain.UpdateState{}, false, nil + } + st, found, err := s.states.Get(ctx, authKeyID, userID) + if err != nil { + return domain.UpdateState{}, false, err + } + st.Seq = 0 + return st, found, nil +} + // AcknowledgeCurrentState 返回账号当前最大连续状态，并把该设备的确认水位推进到此。 // // 供 updates.getState 使用：协议语义是客户端宣告「从现在开始同步」，启动期的 diff --git a/internal/botapi/projection.go b/internal/botapi/projection.go new file mode 100644 index 00000000..5595eddd --- /dev/null +++ b/internal/botapi/projection.go @@ -0,0 +1,431 @@ +package botapi + +import ( + "encoding/base64" + "encoding/json" + "errors" + "strconv" + "strings" + + "telesrv/internal/domain" +) + +func apiInt(raw string, fallback int) int { + raw = strings.TrimSpace(raw) + if raw == "" { + return fallback + } + v, err := strconv.Atoi(raw) + if err != nil { + return fallback + } + return v +} + +func botAPIMessageEntities(raw string) ([]domain.MessageEntity, error) { + if strings.TrimSpace(raw) == "" { + return nil, nil + } + var payload []apiMessageEntity + if err := json.Unmarshal([]byte(raw), &payload); err != nil { + return nil, errors.New("ENTITY_INVALID") + } + return messageEntitiesFromAPI(payload) +} + +func allowedUpdates(raw string) map[string]struct{} { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil + } + var items []string + if err := json.Unmarshal([]byte(raw), &items); err != nil { + return nil + } + out := make(map[string]struct{}, len(items)) + for _, item := range items { + item = strings.TrimSpace(item) + if item != "" { + out[item] = struct{}{} + } + } + return out +} + +func apiUpdates(events []domain.UpdateEvent, allowed map[string]struct{}, limit int) []map[string]any { + if limit <= 0 || limit > 100 { + limit = 100 + } + out := make([]map[string]any, 0, min(len(events), limit)) + for _, event := range events { + item, kind, ok := apiUpdate(event) + if !ok || !updateAllowed(kind, allowed) { + continue + } + out = append(out, item) + if len(out) >= limit { + break + } + } + if out == nil { + return []map[string]any{} + } + return out +} + +func updateAllowed(kind string, allowed map[string]struct{}) bool { + if len(allowed) == 0 { + return true + } + _, ok := allowed[kind] + return ok +} + +func apiUpdate(event domain.UpdateEvent) (map[string]any, string, bool) { + if event.Pts <= 0 { + return nil, "", false + } + switch event.Type { + case domain.UpdateEventNewMessage: + if !apiMessageProjectable(event.Message) { + return nil, "", false + } + return map[string]any{ + "update_id": event.Pts, + "message": apiMessage(event.Message, event.Users), + }, "message", true + case domain.UpdateEventEditMessage: + if !apiMessageProjectable(event.Message) { + return nil, "", false + } + return map[string]any{ + "update_id": event.Pts, + "edited_message": apiMessage(event.Message, event.Users), + }, "edited_message", true + default: + return nil, "", false + } +} + +func apiMessageProjectable(msg domain.Message) bool { + if msg.Out || msg.ID <= 0 { + return false + } + return msg.Body != "" || len(apiMessageMedia(msg.Media)) > 0 +} + +func apiUser(u domain.User) map[string]any { + first := u.FirstName + if strings.TrimSpace(first) == "" { + first = "User " + strconv.FormatInt(u.ID, 10) + } + out := map[string]any{ + "id": u.ID, + "is_bot": u.Bot, + "first_name": first, + } + if u.LastName != "" { + out["last_name"] = u.LastName + } + if u.Username != "" { + out["username"] = u.Username + } + return out +} + +func apiMessage(msg domain.Message, users []domain.User) map[string]any { + userByID := map[int64]domain.User{} + for _, u := range users { + userByID[u.ID] = u + } + out := map[string]any{ + "message_id": msg.ID, + "date": msg.Date, + "chat": apiChat(msg.Peer, userByID), + } + if msg.From.Type == domain.PeerTypeUser && msg.From.ID != 0 { + from := userByID[msg.From.ID] + if from.ID == 0 { + from = domain.User{ID: msg.From.ID} + } + if msg.Out && msg.From.ID == msg.OwnerUserID { + from.Bot = true + } + out["from"] = apiUser(from) + } + media := apiMessageMedia(msg.Media) + if msg.Body != "" { + if len(media) > 0 { + out["caption"] = msg.Body + } else { + out["text"] = msg.Body + } + } + if entities := apiMessageEntities(msg.Entities, userByID); len(entities) > 0 { + if len(media) > 0 { + out["caption_entities"] = entities + } else { + out["entities"] = entities + } + } + if msg.EditDate > 0 { + out["edit_date"] = msg.EditDate + } + if msg.ReplyTo != nil && msg.ReplyTo.MessageID > 0 { + out["reply_to_message"] = map[string]any{ + "message_id": msg.ReplyTo.MessageID, + "date": 0, + "chat": apiChat(msg.ReplyTo.Peer, userByID), + } + } + if markup := apiReplyMarkup(msg.ReplyMarkup); markup != nil { + out["reply_markup"] = markup + } + if len(media) > 0 { + for k, v := range media { + out[k] = v + } + } + return out +} + +func apiChat(peer domain.Peer, users map[int64]domain.User) map[string]any { + switch peer.Type { + case domain.PeerTypeUser: + out := map[string]any{ + "id": peer.ID, + "type": "private", + } + if u := users[peer.ID]; u.ID != 0 { + out["first_name"] = apiUserFirstName(u) + if u.LastName != "" { + out["last_name"] = u.LastName + } + if u.Username != "" { + out["username"] = u.Username + } + } + return out + case domain.PeerTypeChannel: + return map[string]any{ + "id": -1000000000000 - peer.ID, + "type": "supergroup", + } + default: + return map[string]any{ + "id": peer.ID, + "type": "private", + } + } +} + +func apiUserFirstName(u domain.User) string { + if strings.TrimSpace(u.FirstName) != "" { + return u.FirstName + } + return "User " + strconv.FormatInt(u.ID, 10) +} + +func apiMessageEntities(in []domain.MessageEntity, users map[int64]domain.User) []map[string]any { + if len(in) == 0 { + return nil + } + out := make([]map[string]any, 0, len(in)) + for _, entity := range in { + typ, ok := botAPIEntityType(entity.Type) + if !ok || entity.Offset < 0 || entity.Length <= 0 { + continue + } + item := map[string]any{ + "type": typ, + "offset": entity.Offset, + "length": entity.Length, + } + if entity.URL != "" { + item["url"] = entity.URL + } + if entity.Language != "" { + item["language"] = entity.Language + } + if entity.UserID != 0 { + u := users[entity.UserID] + if u.ID == 0 { + u = domain.User{ID: entity.UserID} + } + item["user"] = apiUser(u) + } + if entity.DocumentID != 0 { + item["custom_emoji_id"] = strconv.FormatInt(entity.DocumentID, 10) + } + out = append(out, item) + } + return out +} + +func botAPIEntityType(in domain.MessageEntityType) (string, bool) { + switch in { + case domain.MessageEntityBold: + return "bold", true + case domain.MessageEntityItalic: + return "italic", true + case domain.MessageEntityUnderline: + return "underline", true + case domain.MessageEntityStrike: + return "strikethrough", true + case domain.MessageEntityCode: + return "code", true + case domain.MessageEntityPre: + return "pre", true + case domain.MessageEntityTextURL: + return "text_link", true + case domain.MessageEntityMentionName: + return "text_mention", true + case domain.MessageEntitySpoiler: + return "spoiler", true + case domain.MessageEntityBlockquote: + return "blockquote", true + case domain.MessageEntityCustomEmoji: + return "custom_emoji", true + case domain.MessageEntityMention: + return "mention", true + case domain.MessageEntityHashtag: + return "hashtag", true + case domain.MessageEntityCashtag: + return "cashtag", true + case domain.MessageEntityBotCommand: + return "bot_command", true + case domain.MessageEntityURL: + return "url", true + case domain.MessageEntityEmail: + return "email", true + case domain.MessageEntityPhone: + return "phone_number", true + default: + return "", false + } +} + +func apiReplyMarkup(markup *domain.MessageReplyMarkup) map[string]any { + if markup.IsZero() { + return nil + } + rows := make([][]map[string]any, 0, len(markup.Inline)) + for _, row := range markup.Inline { + if len(row) == 0 { + continue + } + apiRow := make([]map[string]any, 0, len(row)) + for _, button := range row { + item := map[string]any{"text": button.Text} + switch button.Type { + case domain.MarkupButtonURL: + item["url"] = button.URL + case domain.MarkupButtonCallback: + item["callback_data"] = string(button.Data) + default: + continue + } + apiRow = append(apiRow, item) + } + if len(apiRow) > 0 { + rows = append(rows, apiRow) + } + } + if len(rows) == 0 { + return nil + } + return map[string]any{"inline_keyboard": rows} +} + +func apiMessageMedia(media *domain.MessageMedia) map[string]any { + if media.IsZero() { + return nil + } + switch media.Kind { + case domain.MessageMediaKindPhoto: + if media.Photo == nil { + return nil + } + photos := apiPhotoSizes(*media.Photo) + if len(photos) == 0 { + return nil + } + return map[string]any{"photo": photos} + case domain.MessageMediaKindDocument: + if media.Document == nil { + return nil + } + return map[string]any{"document": apiDocument(*media.Document)} + default: + return nil + } +} + +func apiPhotoSizes(photo domain.Photo) []map[string]any { + return apiPhotoSizesWithPrefix(photo.Sizes, "photo:"+strconv.FormatInt(photo.ID, 10)+":") +} + +func apiPhotoSizesWithPrefix(sizes []domain.PhotoSize, locationPrefix string) []map[string]any { + out := make([]map[string]any, 0, len(sizes)) + for _, size := range sizes { + if !size.Downloadable() || size.Type == "" { + continue + } + fileID := encodeBotAPIFileID(locationPrefix + size.Type) + item := map[string]any{ + "file_id": fileID, + "file_unique_id": fileID, + "width": size.W, + "height": size.H, + } + if size.Size > 0 { + item["file_size"] = size.Size + } + out = append(out, item) + } + return out +} + +func apiDocument(doc domain.Document) map[string]any { + fileID := encodeBotAPIFileID("doc:" + strconv.FormatInt(doc.ID, 10)) + out := map[string]any{ + "file_id": fileID, + "file_unique_id": fileID, + } + if doc.MimeType != "" { + out["mime_type"] = doc.MimeType + } + if doc.Size > 0 { + out["file_size"] = doc.Size + } + for _, attr := range doc.Attributes { + if attr.Kind == domain.DocAttrFilename && strings.TrimSpace(attr.FileName) != "" { + out["file_name"] = attr.FileName + break + } + } + if thumbs := apiPhotoSizesWithPrefix(doc.Thumbs, "doc:"+strconv.FormatInt(doc.ID, 10)+":"); len(thumbs) > 0 { + out["thumbnail"] = thumbs[len(thumbs)-1] + } + return out +} + +func encodeBotAPIFileID(locationKey string) string { + return base64.RawURLEncoding.EncodeToString([]byte(locationKey)) +} + +func decodeBotAPIFileID(fileID string) (string, bool) { + fileID = strings.TrimSpace(fileID) + if fileID == "" { + return "", false + } + data, err := base64.RawURLEncoding.DecodeString(fileID) + if err != nil { + return "", false + } + locationKey := string(data) + if strings.HasPrefix(locationKey, "doc:") || strings.HasPrefix(locationKey, "photo:") { + return locationKey, true + } + return "", false +} diff --git a/internal/botapi/server.go b/internal/botapi/server.go index 6c9d7ad4..85607a1f 100644 --- a/internal/botapi/server.go +++ b/internal/botapi/server.go @@ -35,18 +35,36 @@ type WebAppService interface { SavePreparedInlineMessageFromBotAPI(ctx context.Context, botID, userID int64, result domain.BotInlineResult, peerTypes []string) (id string, expireDate int, err error) } -func Start(ctx context.Context, addr string, bots BotsService, users UsersService, webapps WebAppService, logger *zap.Logger) (*http.Server, error) { +type GatewayService interface { + BotAPISelf(ctx context.Context, botID int64) (domain.User, error) + BotAPIUpdates(ctx context.Context, botID int64, offset int64) ([]domain.UpdateEvent, error) + BotAPISendMessage(ctx context.Context, botID, chatID int64, text string, entities []domain.MessageEntity, replyMarkup *domain.MessageReplyMarkup, disableWebPagePreview, silent bool, replyToMessageID int) (domain.Message, error) + BotAPISendMedia(ctx context.Context, botID, chatID int64, kind, locationKey, remoteURL, fileName, mimeType string, fileBytes []byte, caption string, entities []domain.MessageEntity, replyMarkup *domain.MessageReplyMarkup, silent bool, replyToMessageID int) (domain.Message, error) + BotAPIEditMessageText(ctx context.Context, botID, chatID int64, messageID int, text string, entities []domain.MessageEntity, setReplyMarkup bool, replyMarkup *domain.MessageReplyMarkup, disableWebPagePreview bool) (domain.Message, error) + BotAPIDeleteMessage(ctx context.Context, botID, chatID int64, messageID int) (bool, error) + BotAPIAnswerCallbackQuery(ctx context.Context, botID int64, callbackQueryID, text, url string, showAlert bool, cacheTime int) (bool, error) + BotAPIGetFile(ctx context.Context, botID int64, locationKey string, offset int64, limit int) (domain.FileChunk, bool, error) +} + +type GatewayUpdateWaiter interface { + BotAPIUpdateWaitVersion(botID int64) uint64 + WaitBotAPIUpdate(ctx context.Context, botID int64, version uint64, timeout time.Duration) bool +} + +func Start(ctx context.Context, addr string, bots BotsService, users UsersService, webapps WebAppService, gateway GatewayService, logger *zap.Logger) (*http.Server, error) { if strings.TrimSpace(addr) == "" { return nil, nil } if logger == nil { logger = zap.NewNop() } - handler := &handler{bots: bots, users: users, webapps: webapps, logger: logger} + handler := &handler{bots: bots, users: users, webapps: webapps, gateway: gateway, logger: logger} srv := &http.Server{ Addr: addr, Handler: handler.routes(), ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, } ln, err := net.Listen("tcp", addr) if err != nil { @@ -71,9 +89,23 @@ type handler struct { bots BotsService users UsersService webapps WebAppService + gateway GatewayService logger *zap.Logger } +const ( + maxBotAPIUploadBytes = 25 << 20 + maxBotAPIRequestOverheadBytes = 1 << 20 + maxBotAPIRequestBytes = maxBotAPIUploadBytes + maxBotAPIRequestOverheadBytes + botAPILongPollFallback = 5 * time.Second +) + +type uploadedFile struct { + Name string + MimeType string + Bytes []byte +} + func (h *handler) routes() http.Handler { mux := http.NewServeMux() mux.HandleFunc("/", h.handle) @@ -81,6 +113,11 @@ func (h *handler) routes() http.Handler { } func (h *handler) handle(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxBotAPIRequestBytes) + if strings.HasPrefix(r.URL.Path, "/file/bot") { + h.downloadFile(w, r) + return + } token, method, ok := splitBotPath(r.URL.Path) if !ok { writeAPIError(w, http.StatusNotFound, "METHOD_NOT_FOUND") @@ -92,6 +129,30 @@ func (h *handler) handle(w http.ResponseWriter, r *http.Request) { return } switch strings.ToLower(method) { + case "getme": + h.getMe(w, r, botID) + case "getupdates": + h.getUpdates(w, r, botID) + case "sendmessage": + h.sendMessage(w, r, botID) + case "sendphoto": + h.sendMedia(w, r, botID, "photo") + case "senddocument": + h.sendMedia(w, r, botID, "document") + case "editmessagetext": + h.editMessageText(w, r, botID) + case "deletemessage": + h.deleteMessage(w, r, botID) + case "answercallbackquery": + h.answerCallbackQuery(w, r, botID) + case "getfile": + h.getFile(w, r, botID) + case "deletewebhook": + writeAPIOK(w, true) + case "getwebhookinfo": + writeAPIOK(w, map[string]any{"url": "", "has_custom_certificate": false, "pending_update_count": 0}) + case "setwebhook": + h.setWebhook(w, r) case "setchatmenubutton": h.setChatMenuButton(w, r, botID) case "getchatmenubutton": @@ -122,6 +183,393 @@ func splitBotPath(path string) (token, method string, ok bool) { return token, method, true } +func splitFilePath(path string) (token, fileID string, ok bool) { + rest := strings.TrimPrefix(path, "/file/bot") + rest = strings.TrimPrefix(rest, "/") + token, fileID, found := strings.Cut(rest, "/") + if !found || token == "" || fileID == "" || strings.Contains(fileID, "/") { + return "", "", false + } + return token, fileID, true +} + +func (h *handler) getMe(w http.ResponseWriter, r *http.Request, botID int64) { + if h.gateway == nil { + writeAPIError(w, http.StatusNotImplemented, "METHOD_NOT_FOUND") + return + } + u, err := h.gateway.BotAPISelf(r.Context(), botID) + if err != nil { + writeAPIError(w, http.StatusBadRequest, apiErrorDescription(err)) + return + } + writeAPIOK(w, apiUser(u)) +} + +func (h *handler) getUpdates(w http.ResponseWriter, r *http.Request, botID int64) { + if h.gateway == nil { + writeAPIError(w, http.StatusNotImplemented, "METHOD_NOT_FOUND") + return + } + values, err := requestValues(r) + if err != nil { + writeAPIError(w, http.StatusBadRequest, "BAD_REQUEST") + return + } + offset, _ := strconv.ParseInt(strings.TrimSpace(values["offset"]), 10, 64) + limit := apiInt(values["limit"], 100) + if limit <= 0 { + limit = 100 + } + if limit > 100 { + limit = 100 + } + timeoutSeconds := apiInt(values["timeout"], 0) + if timeoutSeconds < 0 { + timeoutSeconds = 0 + } + if timeoutSeconds > 50 { + timeoutSeconds = 50 + } + allowed := allowedUpdates(values["allowed_updates"]) + deadline := time.Now().Add(time.Duration(timeoutSeconds) * time.Second) + for { + version := botAPIUpdateWaitVersion(h.gateway, botID) + events, err := h.gateway.BotAPIUpdates(r.Context(), botID, offset) + if err != nil { + writeAPIError(w, http.StatusInternalServerError, "INTERNAL_SERVER_ERROR") + return + } + updates := apiUpdates(events, allowed, limit) + if len(updates) > 0 || timeoutSeconds == 0 || time.Now().After(deadline) { + writeAPIOK(w, updates) + return + } + waitForBotAPIUpdate(r.Context(), h.gateway, botID, version, time.Until(deadline)) + } +} + +func botAPIUpdateWaitVersion(gateway GatewayService, botID int64) uint64 { + waiter, ok := gateway.(GatewayUpdateWaiter) + if !ok { + return 0 + } + return waiter.BotAPIUpdateWaitVersion(botID) +} + +func waitForBotAPIUpdate(ctx context.Context, gateway GatewayService, botID int64, version uint64, timeout time.Duration) { + if timeout <= 0 { + return + } + if timeout > botAPILongPollFallback { + timeout = botAPILongPollFallback + } + if waiter, ok := gateway.(GatewayUpdateWaiter); ok { + waiter.WaitBotAPIUpdate(ctx, botID, version, timeout) + return + } + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} + +func (h *handler) sendMessage(w http.ResponseWriter, r *http.Request, botID int64) { + if h.gateway == nil { + writeAPIError(w, http.StatusNotImplemented, "METHOD_NOT_FOUND") + return + } + values, err := requestValues(r) + if err != nil { + writeAPIError(w, http.StatusBadRequest, "BAD_REQUEST") + return + } + chatID, err := strconv.ParseInt(strings.TrimSpace(values["chat_id"]), 10, 64) + if err != nil || chatID == 0 { + writeAPIError(w, http.StatusBadRequest, "CHAT_ID_INVALID") + return + } + text := values["text"] + if strings.TrimSpace(values["parse_mode"]) != "" { + writeAPIError(w, http.StatusBadRequest, "ENTITY_PARSE_UNSUPPORTED") + return + } + entities, err := botAPIMessageEntities(values["entities"]) + if err != nil { + writeAPIError(w, http.StatusBadRequest, err.Error()) + return + } + var markup *domain.MessageReplyMarkup + if raw := strings.TrimSpace(values["reply_markup"]); raw != "" { + markup, err = replyMarkupFromAPI(json.RawMessage(raw)) + if err != nil { + writeAPIError(w, http.StatusBadRequest, err.Error()) + return + } + } + replyTo := apiInt(values["reply_to_message_id"], 0) + msg, err := h.gateway.BotAPISendMessage(r.Context(), botID, chatID, text, entities, markup, apiBool(values["disable_web_page_preview"]), apiBool(values["disable_notification"]), replyTo) + if err != nil { + writeAPIError(w, http.StatusBadRequest, apiErrorDescription(err)) + return + } + users := []domain.User(nil) + if self, err := h.gateway.BotAPISelf(r.Context(), botID); err == nil && self.ID != 0 { + users = append(users, self) + } + writeAPIOK(w, apiMessage(msg, users)) +} + +func (h *handler) sendMedia(w http.ResponseWriter, r *http.Request, botID int64, kind string) { + if h.gateway == nil { + writeAPIError(w, http.StatusNotImplemented, "METHOD_NOT_FOUND") + return + } + values, files, err := requestValuesWithFiles(r) + if err != nil { + writeAPIError(w, http.StatusBadRequest, apiErrorDescription(err)) + return + } + chatID, err := strconv.ParseInt(strings.TrimSpace(values["chat_id"]), 10, 64) + if err != nil || chatID == 0 { + writeAPIError(w, http.StatusBadRequest, "CHAT_ID_INVALID") + return + } + if strings.TrimSpace(values["parse_mode"]) != "" { + writeAPIError(w, http.StatusBadRequest, "ENTITY_PARSE_UNSUPPORTED") + return + } + entities, err := botAPIMessageEntities(values["caption_entities"]) + if err != nil { + writeAPIError(w, http.StatusBadRequest, err.Error()) + return + } + var markup *domain.MessageReplyMarkup + if raw := strings.TrimSpace(values["reply_markup"]); raw != "" { + markup, err = replyMarkupFromAPI(json.RawMessage(raw)) + if err != nil { + writeAPIError(w, http.StatusBadRequest, err.Error()) + return + } + } + locationKey, remoteURL, fileName, mimeType, fileBytes, ok := mediaInput(values[kind], files, kind) + if !ok { + writeAPIError(w, http.StatusBadRequest, "FILE_ID_INVALID") + return + } + msg, err := h.gateway.BotAPISendMedia(r.Context(), botID, chatID, kind, locationKey, remoteURL, fileName, mimeType, fileBytes, values["caption"], entities, markup, apiBool(values["disable_notification"]), apiInt(values["reply_to_message_id"], 0)) + if err != nil { + writeAPIError(w, http.StatusBadRequest, apiErrorDescription(err)) + return + } + users := []domain.User(nil) + if self, err := h.gateway.BotAPISelf(r.Context(), botID); err == nil && self.ID != 0 { + users = append(users, self) + } + writeAPIOK(w, apiMessage(msg, users)) +} + +func (h *handler) editMessageText(w http.ResponseWriter, r *http.Request, botID int64) { + if h.gateway == nil { + writeAPIError(w, http.StatusNotImplemented, "METHOD_NOT_FOUND") + return + } + values, err := requestValues(r) + if err != nil { + writeAPIError(w, http.StatusBadRequest, "BAD_REQUEST") + return + } + chatID, err := strconv.ParseInt(strings.TrimSpace(values["chat_id"]), 10, 64) + if err != nil || chatID == 0 { + writeAPIError(w, http.StatusBadRequest, "CHAT_ID_INVALID") + return + } + messageID := apiInt(values["message_id"], 0) + if messageID <= 0 { + writeAPIError(w, http.StatusBadRequest, "MESSAGE_ID_INVALID") + return + } + if strings.TrimSpace(values["parse_mode"]) != "" { + writeAPIError(w, http.StatusBadRequest, "ENTITY_PARSE_UNSUPPORTED") + return + } + entities, err := botAPIMessageEntities(values["entities"]) + if err != nil { + writeAPIError(w, http.StatusBadRequest, err.Error()) + return + } + var markup *domain.MessageReplyMarkup + _, setReplyMarkup := values["reply_markup"] + if raw := strings.TrimSpace(values["reply_markup"]); raw != "" { + markup, err = replyMarkupFromAPI(json.RawMessage(raw)) + if err != nil { + writeAPIError(w, http.StatusBadRequest, err.Error()) + return + } + } + msg, err := h.gateway.BotAPIEditMessageText(r.Context(), botID, chatID, messageID, values["text"], entities, setReplyMarkup, markup, apiBool(values["disable_web_page_preview"])) + if err != nil { + writeAPIError(w, http.StatusBadRequest, apiErrorDescription(err)) + return + } + users := []domain.User(nil) + if self, err := h.gateway.BotAPISelf(r.Context(), botID); err == nil && self.ID != 0 { + users = append(users, self) + } + writeAPIOK(w, apiMessage(msg, users)) +} + +func (h *handler) deleteMessage(w http.ResponseWriter, r *http.Request, botID int64) { + if h.gateway == nil { + writeAPIError(w, http.StatusNotImplemented, "METHOD_NOT_FOUND") + return + } + values, err := requestValues(r) + if err != nil { + writeAPIError(w, http.StatusBadRequest, "BAD_REQUEST") + return + } + chatID, err := strconv.ParseInt(strings.TrimSpace(values["chat_id"]), 10, 64) + if err != nil || chatID == 0 { + writeAPIError(w, http.StatusBadRequest, "CHAT_ID_INVALID") + return + } + messageID := apiInt(values["message_id"], 0) + if messageID <= 0 { + writeAPIError(w, http.StatusBadRequest, "MESSAGE_ID_INVALID") + return + } + ok, err := h.gateway.BotAPIDeleteMessage(r.Context(), botID, chatID, messageID) + if err != nil { + writeAPIError(w, http.StatusBadRequest, apiErrorDescription(err)) + return + } + writeAPIOK(w, ok) +} + +func (h *handler) answerCallbackQuery(w http.ResponseWriter, r *http.Request, botID int64) { + if h.gateway == nil { + writeAPIError(w, http.StatusNotImplemented, "METHOD_NOT_FOUND") + return + } + values, err := requestValues(r) + if err != nil { + writeAPIError(w, http.StatusBadRequest, "BAD_REQUEST") + return + } + queryID := strings.TrimSpace(values["callback_query_id"]) + if queryID == "" { + writeAPIError(w, http.StatusBadRequest, "QUERY_ID_INVALID") + return + } + ok, err := h.gateway.BotAPIAnswerCallbackQuery(r.Context(), botID, queryID, values["text"], values["url"], apiBool(values["show_alert"]), apiInt(values["cache_time"], 0)) + if err != nil { + writeAPIError(w, http.StatusBadRequest, apiErrorDescription(err)) + return + } + writeAPIOK(w, ok) +} + +func (h *handler) getFile(w http.ResponseWriter, r *http.Request, botID int64) { + if h.gateway == nil { + writeAPIError(w, http.StatusNotImplemented, "METHOD_NOT_FOUND") + return + } + values, err := requestValues(r) + if err != nil { + writeAPIError(w, http.StatusBadRequest, "BAD_REQUEST") + return + } + fileID := strings.TrimSpace(values["file_id"]) + locationKey, ok := decodeBotAPIFileID(fileID) + if !ok { + writeAPIError(w, http.StatusBadRequest, "FILE_ID_INVALID") + return + } + chunk, found, err := h.gateway.BotAPIGetFile(r.Context(), botID, locationKey, 0, 1) + if err != nil { + writeAPIError(w, http.StatusInternalServerError, "INTERNAL_SERVER_ERROR") + return + } + if !found { + writeAPIError(w, http.StatusBadRequest, "FILE_ID_INVALID") + return + } + writeAPIOK(w, map[string]any{ + "file_id": fileID, + "file_unique_id": fileID, + "file_size": chunk.Total, + "file_path": fileID, + }) +} + +func (h *handler) downloadFile(w http.ResponseWriter, r *http.Request) { + if h.gateway == nil { + writeAPIError(w, http.StatusNotImplemented, "METHOD_NOT_FOUND") + return + } + token, fileID, ok := splitFilePath(r.URL.Path) + if !ok { + writeAPIError(w, http.StatusNotFound, "FILE_NOT_FOUND") + return + } + botID, ok := h.authenticate(r.Context(), token) + if !ok { + writeAPIError(w, http.StatusUnauthorized, "ACCESS_TOKEN_INVALID") + return + } + locationKey, ok := decodeBotAPIFileID(fileID) + if !ok { + writeAPIError(w, http.StatusBadRequest, "FILE_ID_INVALID") + return + } + var offset int64 + for { + chunk, found, err := h.gateway.BotAPIGetFile(r.Context(), botID, locationKey, offset, 1<<20) + if err != nil { + writeAPIError(w, http.StatusInternalServerError, "INTERNAL_SERVER_ERROR") + return + } + if !found { + writeAPIError(w, http.StatusBadRequest, "FILE_ID_INVALID") + return + } + if offset == 0 { + if chunk.MimeType != "" { + w.Header().Set("Content-Type", chunk.MimeType) + } + if chunk.Total > 0 { + w.Header().Set("Content-Length", strconv.FormatInt(chunk.Total, 10)) + } + } + if len(chunk.Bytes) == 0 { + return + } + if _, err := w.Write(chunk.Bytes); err != nil { + return + } + offset += int64(len(chunk.Bytes)) + if chunk.Total == 0 || offset >= chunk.Total { + return + } + } +} + +func (h *handler) setWebhook(w http.ResponseWriter, r *http.Request) { + values, err := requestValues(r) + if err != nil { + writeAPIError(w, http.StatusBadRequest, "BAD_REQUEST") + return + } + if strings.TrimSpace(values["url"]) == "" { + writeAPIOK(w, true) + return + } + writeAPIError(w, http.StatusNotImplemented, "WEBHOOK_NOT_IMPLEMENTED") +} + func (h *handler) authenticate(ctx context.Context, token string) (int64, bool) { if h.bots == nil { return 0, false @@ -286,11 +734,16 @@ func (h *handler) savePreparedInlineMessage(w http.ResponseWriter, r *http.Reque } func requestValues(r *http.Request) (map[string]string, error) { + values, _, err := requestValuesWithFiles(r) + return values, err +} + +func requestValuesWithFiles(r *http.Request) (map[string]string, map[string]uploadedFile, error) { out := map[string]string{} if strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { var body map[string]any if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&body); err != nil { - return nil, err + return nil, nil, err } for k, v := range body { switch x := v.(type) { @@ -307,17 +760,79 @@ func requestValues(r *http.Request) (map[string]string, error) { out["menu_button"] = string(b) } } - return out, nil + return out, nil, nil + } + if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") { + if err := r.ParseMultipartForm(maxBotAPIUploadBytes); err != nil { + if strings.Contains(strings.ToLower(err.Error()), "request body too large") { + return nil, nil, errors.New("FILE_TOO_BIG") + } + return nil, nil, errors.New("FILE_ID_INVALID") + } + for k, v := range r.MultipartForm.Value { + if len(v) > 0 { + out[k] = v[0] + } + } + files := map[string]uploadedFile{} + for field, headers := range r.MultipartForm.File { + if len(headers) == 0 { + continue + } + header := headers[0] + file, err := header.Open() + if err != nil { + return nil, nil, errors.New("FILE_ID_INVALID") + } + data, readErr := io.ReadAll(io.LimitReader(file, maxBotAPIUploadBytes+1)) + closeErr := file.Close() + if readErr != nil { + return nil, nil, errors.New("FILE_ID_INVALID") + } + if closeErr != nil { + return nil, nil, errors.New("FILE_ID_INVALID") + } + if len(data) > maxBotAPIUploadBytes { + return nil, nil, errors.New("FILE_TOO_BIG") + } + files[field] = uploadedFile{ + Name: header.Filename, + MimeType: header.Header.Get("Content-Type"), + Bytes: data, + } + } + return out, files, nil } if err := r.ParseForm(); err != nil { - return nil, err + return nil, nil, err } for k, v := range r.Form { if len(v) > 0 { out[k] = v[0] } } - return out, nil + return out, nil, nil +} + +func mediaInput(raw string, files map[string]uploadedFile, defaultField string) (locationKey, remoteURL, fileName, mimeType string, fileBytes []byte, ok bool) { + raw = strings.TrimSpace(raw) + if strings.HasPrefix(raw, "attach://") { + field := strings.TrimPrefix(raw, "attach://") + if file, found := files[field]; found && len(file.Bytes) > 0 { + return "", "", file.Name, file.MimeType, file.Bytes, true + } + return "", "", "", "", nil, false + } + if file, found := files[defaultField]; found && len(file.Bytes) > 0 { + return "", "", file.Name, file.MimeType, file.Bytes, true + } + if strings.HasPrefix(strings.ToLower(raw), "http://") || strings.HasPrefix(strings.ToLower(raw), "https://") { + return "", raw, "", "", nil, true + } + if key, decoded := decodeBotAPIFileID(raw); decoded { + return key, "", "", "", nil, true + } + return "", "", "", "", nil, false } func menuButtonFromAPI(raw string) (domain.BotMenuButton, error) { @@ -394,7 +909,23 @@ func apiErrorDescription(err error) string { "BUTTON_DATA_INVALID", "BUTTON_URL_INVALID", "BOT_INVALID", + "CHAT_ID_INVALID", + "ENTITY_INVALID", + "ENTITY_PARSE_UNSUPPORTED", + "ENTITIES_TOO_LONG", + "ENTITY_BOUNDS_INVALID", + "ENTITY_TYPE_UNSUPPORTED", + "FILE_ID_INVALID", + "FILE_TOO_BIG", + "MEDIA_INVALID", "USER_BOT_REQUIRED", + "QUERY_ID_INVALID", + "MESSAGE_ID_INVALID", + "MESSAGE_NOT_MODIFIED", + "CHAT_WRITE_FORBIDDEN", + "CHAT_ADMIN_REQUIRED", + "REPLY_MESSAGE_ID_INVALID", + "WEBHOOK_NOT_IMPLEMENTED", } { if strings.Contains(text, marker) { return marker diff --git a/internal/botapi/server_test.go b/internal/botapi/server_test.go index b2f57a4f..c6039595 100644 --- a/internal/botapi/server_test.go +++ b/internal/botapi/server_test.go @@ -1,8 +1,10 @@ package botapi import ( + "bytes" "context" "encoding/json" + "mime/multipart" "net/http" "net/http/httptest" "reflect" @@ -129,6 +131,429 @@ func TestAnswerWebAppQueryRejectsUnsupportedResult(t *testing.T) { } } +func TestGetMeUsesGateway(t *testing.T) { + bots := &fakeBotAPIBots{profile: domain.BotProfile{BotUserID: 1001, TokenSecret: "secret"}} + gateway := &fakeBotAPIGateway{ + self: domain.User{ID: 1001, FirstName: "Echo", Username: "echo_bot", Bot: true}, + } + h := (&handler{bots: bots, gateway: gateway}).routes() + + rec := performBotAPIRequest(t, h, bots.profile, "getMe", `{}`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body = %s", rec.Code, rec.Body.String()) + } + var resp struct { + OK bool `json:"ok"` + Result struct { + ID int64 `json:"id"` + IsBot bool `json:"is_bot"` + FirstName string `json:"first_name"` + Username string `json:"username"` + } `json:"result"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if !resp.OK || resp.Result.ID != 1001 || !resp.Result.IsBot || resp.Result.Username != "echo_bot" { + t.Fatalf("response = %s", rec.Body.String()) + } +} + +func TestGetUpdatesProjectsIncomingPrivateText(t *testing.T) { + bots := &fakeBotAPIBots{profile: domain.BotProfile{BotUserID: 1001, TokenSecret: "secret"}} + gateway := &fakeBotAPIGateway{ + updates: []domain.UpdateEvent{{ + UserID: 1001, + Type: domain.UpdateEventNewMessage, + Pts: 7, + Message: domain.Message{ + ID: 3, + OwnerUserID: 1001, + Peer: domain.Peer{Type: domain.PeerTypeUser, ID: 2001}, + From: domain.Peer{Type: domain.PeerTypeUser, ID: 2001}, + Date: 1700000000, + Body: "/start", + Entities: []domain.MessageEntity{{Type: domain.MessageEntityBotCommand, Offset: 0, Length: 6}}, + }, + Users: []domain.User{{ID: 2001, FirstName: "Alice", Username: "alice"}}, + }}, + } + h := (&handler{bots: bots, gateway: gateway}).routes() + + rec := performBotAPIRequest(t, h, bots.profile, "getUpdates", `{"offset":1,"allowed_updates":["message"]}`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body = %s", rec.Code, rec.Body.String()) + } + var resp struct { + OK bool `json:"ok"` + Result []struct { + UpdateID int `json:"update_id"` + Message struct { + MessageID int `json:"message_id"` + Text string `json:"text"` + From struct { + ID int64 `json:"id"` + FirstName string `json:"first_name"` + } `json:"from"` + Chat struct { + ID int64 `json:"id"` + Type string `json:"type"` + } `json:"chat"` + Entities []struct { + Type string `json:"type"` + Offset int `json:"offset"` + Length int `json:"length"` + } `json:"entities"` + } `json:"message"` + } `json:"result"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if !resp.OK || len(resp.Result) != 1 { + t.Fatalf("response = %s", rec.Body.String()) + } + got := resp.Result[0] + if got.UpdateID != 7 || got.Message.MessageID != 3 || got.Message.Text != "/start" || got.Message.From.ID != 2001 || got.Message.Chat.ID != 2001 || got.Message.Chat.Type != "private" { + t.Fatalf("update = %#v", got) + } + if len(got.Message.Entities) != 1 || got.Message.Entities[0].Type != "bot_command" { + t.Fatalf("entities = %#v", got.Message.Entities) + } + if gateway.updateOffset != 1 { + t.Fatalf("offset = %d, want 1", gateway.updateOffset) + } +} + +func TestGetUpdatesSkipsOutgoingBotMessage(t *testing.T) { + bots := &fakeBotAPIBots{profile: domain.BotProfile{BotUserID: 1001, TokenSecret: "secret"}} + gateway := &fakeBotAPIGateway{ + updates: []domain.UpdateEvent{{ + UserID: 1001, + Type: domain.UpdateEventNewMessage, + Pts: 8, + Message: domain.Message{ + ID: 4, + OwnerUserID: 1001, + Peer: domain.Peer{Type: domain.PeerTypeUser, ID: 2001}, + From: domain.Peer{Type: domain.PeerTypeUser, ID: 1001}, + Date: 1700000001, + Body: "sent by bot", + Out: true, + }, + }}, + } + h := (&handler{bots: bots, gateway: gateway}).routes() + + rec := performBotAPIRequest(t, h, bots.profile, "getUpdates", `{}`) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body = %s", rec.Code, rec.Body.String()) + } + var resp struct { + OK bool `json:"ok"` + Result []json.RawMessage `json:"result"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if !resp.OK || len(resp.Result) != 0 { + t.Fatalf("response = %s", rec.Body.String()) + } +} + +func TestSendMessageParsesEntitiesMarkupAndCallsGateway(t *testing.T) { + bots := &fakeBotAPIBots{profile: domain.BotProfile{BotUserID: 1001, TokenSecret: "secret"}} + gateway := &fakeBotAPIGateway{ + self: domain.User{ID: 1001, FirstName: "Echo", Username: "echo_bot", Bot: true}, + sendMessage: domain.Message{ + ID: 9, + OwnerUserID: 1001, + Peer: domain.Peer{Type: domain.PeerTypeUser, ID: 2001}, + From: domain.Peer{Type: domain.PeerTypeUser, ID: 1001}, + Date: 1700000002, + Body: "hello", + Out: true, + Entities: []domain.MessageEntity{{Type: domain.MessageEntityBold, Offset: 0, Length: 5}}, + ReplyMarkup: &domain.MessageReplyMarkup{Inline: [][]domain.MarkupButton{{{Type: domain.MarkupButtonCallback, Text: "Tap", Data: []byte("cb")}}}}, + }, + } + h := (&handler{bots: bots, gateway: gateway}).routes() + body := `{ + "chat_id": 2001, + "text": "hello", + "entities": [{"type":"bold","offset":0,"length":5}], + "reply_markup": {"inline_keyboard": [[{"text":"Tap","callback_data":"cb"}]]}, + "disable_notification": true, + "reply_to_message_id": 5 + }` + + rec := performBotAPIRequest(t, h, bots.profile, "sendMessage", body) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body = %s", rec.Code, rec.Body.String()) + } + if !gateway.sendCalled || gateway.sendBotID != 1001 || gateway.sendChatID != 2001 || gateway.sendText != "hello" || !gateway.sendSilent || gateway.sendReplyTo != 5 { + t.Fatalf("send call = %#v", gateway) + } + if len(gateway.sendEntities) != 1 || gateway.sendEntities[0].Type != domain.MessageEntityBold { + t.Fatalf("entities = %#v", gateway.sendEntities) + } + if gateway.sendMarkup == nil || len(gateway.sendMarkup.Inline) != 1 || len(gateway.sendMarkup.Inline[0]) != 1 || string(gateway.sendMarkup.Inline[0][0].Data) != "cb" { + t.Fatalf("markup = %#v", gateway.sendMarkup) + } + var resp struct { + OK bool `json:"ok"` + Result struct { + MessageID int `json:"message_id"` + Text string `json:"text"` + From struct { + ID int64 `json:"id"` + IsBot bool `json:"is_bot"` + } `json:"from"` + ReplyMarkup struct { + InlineKeyboard [][]struct { + Text string `json:"text"` + CallbackData string `json:"callback_data"` + } `json:"inline_keyboard"` + } `json:"reply_markup"` + } `json:"result"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if !resp.OK || resp.Result.MessageID != 9 || resp.Result.Text != "hello" || resp.Result.From.ID != 1001 || !resp.Result.From.IsBot { + t.Fatalf("response = %s", rec.Body.String()) + } + if len(resp.Result.ReplyMarkup.InlineKeyboard) != 1 || resp.Result.ReplyMarkup.InlineKeyboard[0][0].CallbackData != "cb" { + t.Fatalf("reply_markup response = %#v", resp.Result.ReplyMarkup) + } +} + +func TestSendDocumentMultipartParsesFileAndCaption(t *testing.T) { + bots := &fakeBotAPIBots{profile: domain.BotProfile{BotUserID: 1001, TokenSecret: "secret"}} + gateway := &fakeBotAPIGateway{ + self: domain.User{ID: 1001, FirstName: "Echo", Username: "echo_bot", Bot: true}, + sendMediaMessage: domain.Message{ + ID: 11, + OwnerUserID: 1001, + Peer: domain.Peer{Type: domain.PeerTypeUser, ID: 2001}, + From: domain.Peer{Type: domain.PeerTypeUser, ID: 1001}, + Date: 1700000006, + Body: "doc caption", + Out: true, + Media: &domain.MessageMedia{ + Kind: domain.MessageMediaKindDocument, + Document: &domain.Document{ + ID: 42, + MimeType: "text/plain", + Size: 10, + Attributes: []domain.DocumentAttribute{{ + Kind: domain.DocAttrFilename, + FileName: "note.txt", + }}, + }, + }, + }, + } + h := (&handler{bots: bots, gateway: gateway}).routes() + + var body bytes.Buffer + writer := multipart.NewWriter(&body) + _ = writer.WriteField("chat_id", "2001") + _ = writer.WriteField("caption", "doc caption") + part, err := writer.CreateFormFile("document", "note.txt") + if err != nil { + t.Fatalf("create form file: %v", err) + } + if _, err := part.Write([]byte("hello file")); err != nil { + t.Fatalf("write file: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("close multipart: %v", err) + } + token := domain.FormatBotToken(bots.profile.BotUserID, bots.profile.TokenSecret) + req := httptest.NewRequest(http.MethodPost, "/bot"+token+"/sendDocument", &body) + req.Header.Set("Content-Type", writer.FormDataContentType()) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body = %s", rec.Code, rec.Body.String()) + } + if !gateway.sendMediaCalled || gateway.sendMediaKind != "document" || gateway.sendMediaChatID != 2001 || gateway.sendMediaCaption != "doc caption" { + t.Fatalf("send media call = %#v", gateway) + } + if gateway.sendMediaFileName != "note.txt" || string(gateway.sendMediaBytes) != "hello file" { + t.Fatalf("file = name %q bytes %q", gateway.sendMediaFileName, string(gateway.sendMediaBytes)) + } + var resp struct { + OK bool `json:"ok"` + Result struct { + Caption string `json:"caption"` + Document struct { + FileName string `json:"file_name"` + FileID string `json:"file_id"` + } `json:"document"` + } `json:"result"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if !resp.OK || resp.Result.Caption != "doc caption" || resp.Result.Document.FileName != "note.txt" { + t.Fatalf("response = %s", rec.Body.String()) + } +} + +func TestEditDeleteCallbackAndFileEndpoints(t *testing.T) { + bots := &fakeBotAPIBots{profile: domain.BotProfile{BotUserID: 1001, TokenSecret: "secret"}} + locationKey := "doc:42" + fileID := encodeBotAPIFileID(locationKey) + gateway := &fakeBotAPIGateway{ + self: domain.User{ID: 1001, FirstName: "Echo", Username: "echo_bot", Bot: true}, + editMessage: domain.Message{ + ID: 9, + OwnerUserID: 1001, + Peer: domain.Peer{Type: domain.PeerTypeUser, ID: 2001}, + From: domain.Peer{Type: domain.PeerTypeUser, ID: 1001}, + Date: 1700000003, + EditDate: 1700000004, + Body: "edited", + Out: true, + }, + fileChunks: map[string]domain.FileChunk{ + locationKey: {Bytes: []byte("hello file"), MimeType: "text/plain", Total: int64(len("hello file"))}, + }, + } + h := (&handler{bots: bots, gateway: gateway}).routes() + + edit := performBotAPIRequest(t, h, bots.profile, "editMessageText", `{"chat_id":2001,"message_id":9,"text":"edited","reply_markup":{"inline_keyboard":[]}}`) + if edit.Code != http.StatusOK { + t.Fatalf("edit status = %d body = %s", edit.Code, edit.Body.String()) + } + if !gateway.editCalled || !gateway.editSetMarkup { + t.Fatalf("edit gateway = %#v", gateway) + } + + del := performBotAPIRequest(t, h, bots.profile, "deleteMessage", `{"chat_id":2001,"message_id":9}`) + if del.Code != http.StatusOK || !gateway.deleteCalled { + t.Fatalf("delete status = %d body = %s gateway=%#v", del.Code, del.Body.String(), gateway) + } + + cb := performBotAPIRequest(t, h, bots.profile, "answerCallbackQuery", `{"callback_query_id":"123","text":"ok"}`) + if cb.Code != http.StatusOK || !gateway.callbackCalled || gateway.callbackID != "123" { + t.Fatalf("callback status = %d body = %s gateway=%#v", cb.Code, cb.Body.String(), gateway) + } + + file := performBotAPIRequest(t, h, bots.profile, "getFile", `{"file_id":"`+fileID+`"}`) + if file.Code != http.StatusOK { + t.Fatalf("getFile status = %d body = %s", file.Code, file.Body.String()) + } + var fileResp struct { + OK bool `json:"ok"` + Result struct { + FileID string `json:"file_id"` + FilePath string `json:"file_path"` + FileSize int64 `json:"file_size"` + } `json:"result"` + } + if err := json.Unmarshal(file.Body.Bytes(), &fileResp); err != nil { + t.Fatalf("decode getFile: %v", err) + } + if !fileResp.OK || fileResp.Result.FileID != fileID || fileResp.Result.FilePath != fileID || fileResp.Result.FileSize != int64(len("hello file")) || gateway.fileLocationKey != locationKey { + t.Fatalf("getFile response = %s gateway=%#v", file.Body.String(), gateway) + } + + token := domain.FormatBotToken(bots.profile.BotUserID, bots.profile.TokenSecret) + req := httptest.NewRequest(http.MethodGet, "/file/bot"+token+"/"+fileID, nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK || rec.Body.String() != "hello file" || rec.Header().Get("Content-Type") != "text/plain" { + t.Fatalf("download status=%d content-type=%q body=%q", rec.Code, rec.Header().Get("Content-Type"), rec.Body.String()) + } +} + +func TestAPIMessageProjectsMediaCaptionAndFileID(t *testing.T) { + msg := domain.Message{ + ID: 3, + OwnerUserID: 1001, + Peer: domain.Peer{Type: domain.PeerTypeUser, ID: 2001}, + From: domain.Peer{Type: domain.PeerTypeUser, ID: 2001}, + Date: 1700000005, + Body: "caption", + Entities: []domain.MessageEntity{{Type: domain.MessageEntityBold, Offset: 0, Length: 7}}, + Media: &domain.MessageMedia{ + Kind: domain.MessageMediaKindDocument, + Document: &domain.Document{ + ID: 42, + MimeType: "text/plain", + Size: 10, + Attributes: []domain.DocumentAttribute{{ + Kind: domain.DocAttrFilename, + FileName: "note.txt", + }}, + }, + }, + } + projected := apiMessage(msg, []domain.User{{ID: 2001, FirstName: "Alice"}}) + if _, hasText := projected["text"]; hasText { + t.Fatalf("media message has text field: %#v", projected) + } + if projected["caption"] != "caption" { + t.Fatalf("caption = %#v", projected["caption"]) + } + if _, ok := projected["caption_entities"].([]map[string]any); !ok { + t.Fatalf("caption_entities = %#v", projected["caption_entities"]) + } + document, ok := projected["document"].(map[string]any) + if !ok { + t.Fatalf("document = %#v", projected["document"]) + } + fileID, _ := document["file_id"].(string) + if locationKey, ok := decodeBotAPIFileID(fileID); !ok || locationKey != "doc:42" { + t.Fatalf("file_id %q decodes to %q ok=%v", fileID, locationKey, ok) + } + if document["file_name"] != "note.txt" || document["mime_type"] != "text/plain" { + t.Fatalf("document = %#v", document) + } +} + +func TestAPIUpdateProjectsCaptionlessMediaMessage(t *testing.T) { + item, kind, ok := apiUpdate(domain.UpdateEvent{ + Type: domain.UpdateEventNewMessage, + Pts: 12, + Message: domain.Message{ + ID: 4, + OwnerUserID: 1001, + Peer: domain.Peer{Type: domain.PeerTypeUser, ID: 2001}, + From: domain.Peer{Type: domain.PeerTypeUser, ID: 2001}, + Date: 1700000006, + Media: &domain.MessageMedia{ + Kind: domain.MessageMediaKindDocument, + Document: &domain.Document{ + ID: 43, + MimeType: "application/octet-stream", + Size: 4, + }, + }, + }, + }) + if !ok || kind != "message" { + t.Fatalf("apiUpdate ok=%v kind=%q item=%#v", ok, kind, item) + } + msg, ok := item["message"].(map[string]any) + if !ok { + t.Fatalf("message = %#v", item["message"]) + } + if _, hasText := msg["text"]; hasText { + t.Fatalf("captionless media has text: %#v", msg) + } + if _, hasCaption := msg["caption"]; hasCaption { + t.Fatalf("captionless media has caption: %#v", msg) + } + if _, ok := msg["document"].(map[string]any); !ok { + t.Fatalf("document = %#v", msg["document"]) + } +} + func performBotAPIRequest(t *testing.T, h http.Handler, profile domain.BotProfile, method, body string) *httptest.ResponseRecorder { t.Helper() token := domain.FormatBotToken(profile.BotUserID, profile.TokenSecret) @@ -196,3 +621,105 @@ func (f *fakeWebAppService) SavePreparedInlineMessageFromBotAPI(_ context.Contex f.preparedPeerTypes = append([]string(nil), peerTypes...) return f.preparedID, f.preparedExpire, nil } + +type fakeBotAPIGateway struct { + self domain.User + + updates []domain.UpdateEvent + updateBotID int64 + updateOffset int64 + + sendCalled bool + sendBotID int64 + sendChatID int64 + sendText string + sendEntities []domain.MessageEntity + sendMarkup *domain.MessageReplyMarkup + sendNoWebpage bool + sendSilent bool + sendReplyTo int + sendMessage domain.Message + sendMediaCalled bool + sendMediaKind string + sendMediaChatID int64 + sendMediaFileName string + sendMediaBytes []byte + sendMediaCaption string + sendMediaMessage domain.Message + editCalled bool + editSetMarkup bool + editMessage domain.Message + deleteCalled bool + callbackCalled bool + callbackID string + fileLocationKey string + fileChunks map[string]domain.FileChunk +} + +func (f *fakeBotAPIGateway) BotAPISelf(context.Context, int64) (domain.User, error) { + return f.self, nil +} + +func (f *fakeBotAPIGateway) BotAPIUpdates(_ context.Context, botID int64, offset int64) ([]domain.UpdateEvent, error) { + f.updateBotID = botID + f.updateOffset = offset + return append([]domain.UpdateEvent(nil), f.updates...), nil +} + +func (f *fakeBotAPIGateway) BotAPISendMessage(_ context.Context, botID, chatID int64, text string, entities []domain.MessageEntity, replyMarkup *domain.MessageReplyMarkup, disableWebPagePreview, silent bool, replyToMessageID int) (domain.Message, error) { + f.sendCalled = true + f.sendBotID = botID + f.sendChatID = chatID + f.sendText = text + f.sendEntities = append([]domain.MessageEntity(nil), entities...) + f.sendMarkup = replyMarkup + f.sendNoWebpage = disableWebPagePreview + f.sendSilent = silent + f.sendReplyTo = replyToMessageID + return f.sendMessage, nil +} + +func (f *fakeBotAPIGateway) BotAPISendMedia(_ context.Context, botID, chatID int64, kind, locationKey, remoteURL, fileName, mimeType string, fileBytes []byte, caption string, entities []domain.MessageEntity, replyMarkup *domain.MessageReplyMarkup, silent bool, replyToMessageID int) (domain.Message, error) { + f.sendMediaCalled = true + f.sendMediaKind = kind + f.sendMediaChatID = chatID + f.sendMediaFileName = fileName + f.sendMediaBytes = append([]byte(nil), fileBytes...) + f.sendMediaCaption = caption + return f.sendMediaMessage, nil +} + +func (f *fakeBotAPIGateway) BotAPIEditMessageText(_ context.Context, botID, chatID int64, messageID int, text string, entities []domain.MessageEntity, setReplyMarkup bool, replyMarkup *domain.MessageReplyMarkup, disableWebPagePreview bool) (domain.Message, error) { + f.editCalled = true + f.editSetMarkup = setReplyMarkup + return f.editMessage, nil +} + +func (f *fakeBotAPIGateway) BotAPIDeleteMessage(context.Context, int64, int64, int) (bool, error) { + f.deleteCalled = true + return true, nil +} + +func (f *fakeBotAPIGateway) BotAPIAnswerCallbackQuery(_ context.Context, _ int64, callbackQueryID, text, url string, showAlert bool, cacheTime int) (bool, error) { + f.callbackCalled = true + f.callbackID = callbackQueryID + return true, nil +} + +func (f *fakeBotAPIGateway) BotAPIGetFile(_ context.Context, _ int64, locationKey string, offset int64, limit int) (domain.FileChunk, bool, error) { + f.fileLocationKey = locationKey + chunk, ok := f.fileChunks[locationKey] + if !ok { + return domain.FileChunk{}, false, nil + } + if offset >= int64(len(chunk.Bytes)) { + return domain.FileChunk{MimeType: chunk.MimeType, Total: chunk.Total}, true, nil + } + end := offset + int64(limit) + if end > int64(len(chunk.Bytes)) { + end = int64(len(chunk.Bytes)) + } + out := chunk + out.Bytes = append([]byte(nil), chunk.Bytes[offset:end]...) + return out, true, nil +} diff --git a/internal/config/config.go b/internal/config/config.go index 9e89dc94..d8129718 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -46,7 +46,7 @@ type Config struct { // 生产默认 https://telesrv.net；本地可设为 http://127.0.0.1:2401。 PublicBaseURL string // PublicLinkWebAddr 是公开链接落地页监听地址；为空关闭。 - // 生产应只监听 loopback，并由 nginx 将 /addstickers/、/addemoji/ 与 /addlist/ 反代到该地址。 + // 生产应只监听 loopback，并由 nginx 将 /、/addstickers/、/addemoji/ 与 /addlist/ 反代到该地址。 PublicLinkWebAddr string // Admin UI 独立进程配置项保留在统一配置中，cmd/telesrv-admin 也按同名 env 读取。 AdminUIAddr string @@ -179,6 +179,9 @@ type Config struct { ChannelNudgeMaxTargets int // UpdateEventRetention 是 durable update log 保留期；只清理已被水位/state 覆盖的事件。 UpdateEventRetention time.Duration + // BotAPIUpdateRetention 是 bot_api_updates 投递队列的最大保留期（官方 Bot API 语义 24h）； + // 已确认的行另按固定短宽限提前回收（性能审计 H1）。 + BotAPIUpdateRetention time.Duration // RetentionInterval 是 retention worker 的运行间隔。 RetentionInterval time.Duration // RetentionBatch 是单次 retention 最多删除的行数。 @@ -392,6 +395,7 @@ func Load() (Config, error) { CatchupRateWindow: envDurationOr("TELESRV_CATCHUP_RATE_WINDOW", time.Minute), ChannelNudgeMaxTargets: envIntOr("TELESRV_CHANNEL_NUDGE_MAX_TARGETS", 0), UpdateEventRetention: envDurationOr("TELESRV_UPDATE_EVENT_RETENTION", 168*time.Hour), + BotAPIUpdateRetention: envDurationOr("TELESRV_BOT_API_UPDATE_RETENTION", 24*time.Hour), RetentionInterval: envDurationOr("TELESRV_RETENTION_INTERVAL", time.Hour), RetentionBatch: envIntOr("TELESRV_RETENTION_BATCH", 10000), UploadPartTTL: envDurationOr("TELESRV_UPLOAD_PART_TTL", 24*time.Hour), diff --git a/internal/domain/botapi_update.go b/internal/domain/botapi_update.go new file mode 100644 index 00000000..08b8cc07 --- /dev/null +++ b/internal/domain/botapi_update.go @@ -0,0 +1,33 @@ +package domain + +// BotAPIUpdateKind is the Bot API delivery shape for a queued update. +type BotAPIUpdateKind string + +const ( + BotAPIUpdateMessage BotAPIUpdateKind = "message" + BotAPIUpdateEditedMessage BotAPIUpdateKind = "edited_message" +) + +// BotAPIUpdate is a durable Bot API update cursor. ID is the Bot API update_id +// and is global across all bots, matching Telegram Bot API's monotonic offset +// contract without reusing MTProto pts from user/channel logs. +type BotAPIUpdate struct { + ID int64 + BotUserID int64 + Kind BotAPIUpdateKind + Peer Peer + MessageID int + SourcePts int + Date int +} + +// EnqueueBotAPIUpdateRequest describes a message-like update that should be +// delivered to one bot via getUpdates. +type EnqueueBotAPIUpdateRequest struct { + BotUserID int64 + Kind BotAPIUpdateKind + Peer Peer + MessageID int + SourcePts int + Date int +} diff --git a/internal/rpc/botapi_enqueue_dispatcher.go b/internal/rpc/botapi_enqueue_dispatcher.go new file mode 100644 index 00000000..75bf3248 --- /dev/null +++ b/internal/rpc/botapi_enqueue_dispatcher.go @@ -0,0 +1,82 @@ +package rpc + +import ( + "context" + "sync/atomic" + "time" + + "go.uber.org/zap" +) + +// Bot API 私聊 enqueue 异步化（性能审计 H2）：user→bot 私聊发送/编辑时，把 +// bot_api_updates 的 INSERT（以及可能打 PG 的 bot 判定）移出发送者 RPC 同步路径， +// 发送者不再为 Bot API 队列写入多等一次 PG 往返。 +// +// 与 channel fanout dispatcher 的关键差异：bot_api_updates 行本身就是投递真值 +// （getUpdates 只读该表，没有 getDifference 类兜底），因此队列满时**同步回退执行** +// 而不是丢弃——发送者多等一次 INSERT，换 update 不丢。 +// +// 单 worker FIFO：保证同一 bot 的 update_id 顺序与发送顺序一致（并发 goroutine 池 +// 会让 bot 侧 getUpdates 看到乱序消息）。 + +const ( + defaultBotAPIEnqueueBuffer = 4096 + botAPIEnqueueJobTimeout = 10 * time.Second + botAPIEnqueueFallbackReason = "bot api enqueue queue full, falling back to synchronous insert" +) + +type botAPIEnqueueDispatcher struct { + log *zap.Logger + jobs chan func(context.Context) + started atomic.Bool +} + +func newBotAPIEnqueueDispatcher(log *zap.Logger, buffer int) *botAPIEnqueueDispatcher { + if buffer <= 0 { + buffer = defaultBotAPIEnqueueBuffer + } + return &botAPIEnqueueDispatcher{ + log: log.Named("botapi-enqueue"), + jobs: make(chan func(context.Context), buffer), + } +} + +// RunBotAPIEnqueue 启动 Bot API enqueue 后台 worker，由 main 与其它 dispatcher 一同 go 起。 +// 阻塞到 ctx 取消；未调用前 enqueue 同步执行（行为同旧版，测试/未装配场景不变）。 +func (r *Router) RunBotAPIEnqueue(ctx context.Context) { + r.botAPIEnqueueQueue.Run(ctx) +} + +func (d *botAPIEnqueueDispatcher) Run(ctx context.Context) { + if d == nil || !d.started.CompareAndSwap(false, true) { + return + } + for { + select { + case <-ctx.Done(): + return + case job := <-d.jobs: + jobCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), botAPIEnqueueJobTimeout) + job(jobCtx) + cancel() + } + } +} + +// Enqueue 投递一个 Bot API 队列写入任务。dispatcher 未启动时同步执行（用请求 ctx）； +// 已启动时投入 FIFO，满则同步回退执行——绝不丢弃（队列行是投递真值）。 +func (d *botAPIEnqueueDispatcher) Enqueue(reqCtx context.Context, job func(context.Context)) { + if d == nil || job == nil { + return + } + if !d.started.Load() { + job(reqCtx) + return + } + select { + case d.jobs <- job: + default: + d.log.Warn(botAPIEnqueueFallbackReason) + job(reqCtx) + } +} diff --git a/internal/rpc/botapi_enqueue_dispatcher_test.go b/internal/rpc/botapi_enqueue_dispatcher_test.go new file mode 100644 index 00000000..5f46e5df --- /dev/null +++ b/internal/rpc/botapi_enqueue_dispatcher_test.go @@ -0,0 +1,76 @@ +package rpc + +import ( + "context" + "sync" + "testing" + "time" + + "go.uber.org/zap/zaptest" +) + +// TestBotAPIEnqueueDispatcherSynchronousBeforeRun 锁定未启动时的同步回退： +// 测试/未装配场景下 enqueue 行为与旧版完全一致（job 在调用方 goroutine 内即时执行）。 +func TestBotAPIEnqueueDispatcherSynchronousBeforeRun(t *testing.T) { + d := newBotAPIEnqueueDispatcher(zaptest.NewLogger(t), 4) + ran := false + d.Enqueue(context.Background(), func(context.Context) { ran = true }) + if !ran { + t.Fatal("job must run synchronously before Run is called") + } +} + +// TestBotAPIEnqueueDispatcherFIFOOrder 锁定启动后单 worker FIFO：同一 bot 的 +// update_id 顺序必须与 enqueue 顺序一致（乱序会让 bot 侧 getUpdates 看到错序消息）。 +func TestBotAPIEnqueueDispatcherFIFOOrder(t *testing.T) { + d := newBotAPIEnqueueDispatcher(zaptest.NewLogger(t), 16) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go d.Run(ctx) + for !d.started.Load() { + time.Sleep(time.Millisecond) + } + + var mu sync.Mutex + var order []int + done := make(chan struct{}) + for i := 0; i < 5; i++ { + i := i + d.Enqueue(context.Background(), func(context.Context) { + mu.Lock() + order = append(order, i) + if len(order) == 5 { + close(done) + } + mu.Unlock() + }) + } + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("jobs did not complete") + } + mu.Lock() + defer mu.Unlock() + for i, got := range order { + if got != i { + t.Fatalf("order = %v, want FIFO", order) + } + } +} + +// TestBotAPIEnqueueDispatcherFallsBackWhenFull 锁定队列满时的同步回退：Bot API 队列行 +// 是投递真值（无 getDifference 类兜底），满时发送者多等一次 INSERT，绝不丢。 +func TestBotAPIEnqueueDispatcherFallsBackWhenFull(t *testing.T) { + d := newBotAPIEnqueueDispatcher(zaptest.NewLogger(t), 1) + d.started.Store(true) // 模拟已启动但 worker 不消费（阻塞场景） + + // 塞满容量 1 的队列。 + d.Enqueue(context.Background(), func(context.Context) {}) + + ran := false + d.Enqueue(context.Background(), func(context.Context) { ran = true }) + if !ran { + t.Fatal("job must fall back to synchronous execution when queue is full") + } +} diff --git a/internal/rpc/botapi_gateway.go b/internal/rpc/botapi_gateway.go new file mode 100644 index 00000000..a0d307a1 --- /dev/null +++ b/internal/rpc/botapi_gateway.go @@ -0,0 +1,469 @@ +package rpc + +import ( + "context" + "errors" + "strconv" + "strings" + "time" + "unicode/utf8" + + "telesrv/internal/domain" +) + +var botAPIAuthKeyID = [8]byte{'B', 'O', 'T', 'A', 'P', 'I', 0, 1} + +const botAPIChannelChatIDBase int64 = 1000000000000 + +// BotAPISelf returns the authenticated bot as a domain user. +func (r *Router) BotAPISelf(ctx context.Context, botID int64) (domain.User, error) { + if r == nil || r.deps.Users == nil || botID == 0 { + return domain.User{}, errors.New("BOT_INVALID") + } + u, found, err := r.deps.Users.ByID(ctx, botID, botID) + if err != nil { + return domain.User{}, err + } + if !found || !u.Bot { + return domain.User{}, errors.New("BOT_INVALID") + } + return u, nil +} + +// BotAPIUpdates returns durable update_id based events projected for the HTTP +// Bot API. New deployments use the dedicated Bot API queue; the legacy +// user_update_events fallback is kept for tests that have not wired the queue. +func (r *Router) BotAPIUpdates(ctx context.Context, botID int64, offset int64) ([]domain.UpdateEvent, error) { + if r == nil || botID == 0 { + return nil, nil + } + if r.deps.BotAPIUpdates != nil { + return r.botAPIQueuedUpdates(ctx, botID, offset) + } + if r.deps.Updates == nil { + return nil, nil + } + fromPts := 0 + if offset > 0 { + fromPts = int(offset - 1) + } else if st, found, err := r.deps.Updates.ConfirmedState(ctx, botAPIAuthKeyID, botID); err != nil { + return nil, err + } else if found { + fromPts = st.Pts + } + diff, err := r.deps.Updates.GetDifference(ctx, botAPIAuthKeyID, botID, domain.UpdateState{Pts: fromPts}) + if err != nil { + return nil, err + } + if len(diff.Events) == 0 { + return nil, nil + } + return r.enrichUpdateEvents(ctx, botID, diff.Events), nil +} + +// BotAPISendMessage sends a text message as a bot through the normal private +// or channel message state machine. Positive chat_id is a user private chat; +// -1000000000000-channel_id is a supergroup/channel chat. +func (r *Router) BotAPISendMessage(ctx context.Context, botID, chatID int64, text string, entities []domain.MessageEntity, replyMarkup *domain.MessageReplyMarkup, disableWebPagePreview, silent bool, replyToMessageID int) (domain.Message, error) { + if r == nil || botID == 0 { + return domain.Message{}, errors.New("BOT_INVALID") + } + peer, ok := botAPIPeerFromChatID(chatID) + if !ok { + return domain.Message{}, errors.New("CHAT_ID_INVALID") + } + if text == "" { + return domain.Message{}, errors.New("MESSAGE_EMPTY") + } + if utf8.RuneCountInString(text) > domain.MaxMessageTextLength { + return domain.Message{}, errors.New("MESSAGE_TOO_LONG") + } + var reply *domain.MessageReply + if replyToMessageID > 0 { + reply = &domain.MessageReply{Peer: peer, MessageID: replyToMessageID} + } + if peer.Type == domain.PeerTypeChannel { + return r.botAPISendChannelMessage(ctx, botID, peer.ID, text, entities, nil, replyMarkup, silent, reply) + } + if r.deps.Messages == nil { + return domain.Message{}, errors.New("BOT_INVALID") + } + if r.deps.Users != nil && peer.ID != botID { + if _, found, err := r.deps.Users.ByID(ctx, botID, peer.ID); err != nil { + return domain.Message{}, err + } else if !found { + return domain.Message{}, errors.New("CHAT_ID_INVALID") + } + } + res, err := r.deps.Messages.SendPrivateText(ctx, botID, domain.SendPrivateTextRequest{ + SenderUserID: botID, + RecipientUserID: peer.ID, + RandomID: randomNonZeroInt64(), + Message: text, + Entities: append([]domain.MessageEntity(nil), entities...), + Silent: silent, + ReplyTo: reply, + Date: int(time.Now().Unix()), + ReplyMarkup: replyMarkup, + }) + if err != nil { + return domain.Message{}, err + } + return res.SenderMessage, nil +} + +// BotAPISendMedia sends a photo/document message through the same files service +// and private/channel message state machines used by MTProto sendMedia. +func (r *Router) BotAPISendMedia(ctx context.Context, botID, chatID int64, kind, locationKey, remoteURL, fileName, mimeType string, fileBytes []byte, caption string, entities []domain.MessageEntity, replyMarkup *domain.MessageReplyMarkup, silent bool, replyToMessageID int) (domain.Message, error) { + if r == nil || r.deps.Files == nil || botID == 0 { + return domain.Message{}, errors.New("BOT_INVALID") + } + peer, ok := botAPIPeerFromChatID(chatID) + if !ok { + return domain.Message{}, errors.New("CHAT_ID_INVALID") + } + if utf8.RuneCountInString(caption) > domain.MaxMessageTextLength { + return domain.Message{}, errors.New("MESSAGE_TOO_LONG") + } + media, err := r.botAPIMedia(ctx, botID, kind, locationKey, remoteURL, fileName, mimeType, fileBytes) + if err != nil { + return domain.Message{}, err + } + var reply *domain.MessageReply + if replyToMessageID > 0 { + reply = &domain.MessageReply{Peer: peer, MessageID: replyToMessageID} + } + if peer.Type == domain.PeerTypeChannel { + return r.botAPISendChannelMessage(ctx, botID, peer.ID, caption, entities, media, replyMarkup, silent, reply) + } + if r.deps.Messages == nil { + return domain.Message{}, errors.New("BOT_INVALID") + } + if r.deps.Users != nil && peer.ID != botID { + if _, found, err := r.deps.Users.ByID(ctx, botID, peer.ID); err != nil { + return domain.Message{}, err + } else if !found { + return domain.Message{}, errors.New("CHAT_ID_INVALID") + } + } + res, err := r.deps.Messages.SendPrivateText(ctx, botID, domain.SendPrivateTextRequest{ + SenderUserID: botID, + RecipientUserID: peer.ID, + RandomID: randomNonZeroInt64(), + Message: caption, + Entities: append([]domain.MessageEntity(nil), entities...), + Media: media, + Silent: silent, + ReplyTo: reply, + Date: int(time.Now().Unix()), + ReplyMarkup: replyMarkup, + }) + if err != nil { + return domain.Message{}, err + } + return res.SenderMessage, nil +} + +func botAPIPeerFromChatID(chatID int64) (domain.Peer, bool) { + switch { + case chatID > 0: + return domain.Peer{Type: domain.PeerTypeUser, ID: chatID}, true + case chatID < -botAPIChannelChatIDBase: + channelID := -botAPIChannelChatIDBase - chatID + if channelID > 0 { + return domain.Peer{Type: domain.PeerTypeChannel, ID: channelID}, true + } + } + return domain.Peer{}, false +} + +func (r *Router) botAPISendChannelMessage(ctx context.Context, botID, channelID int64, text string, entities []domain.MessageEntity, media *domain.MessageMedia, replyMarkup *domain.MessageReplyMarkup, silent bool, reply *domain.MessageReply) (domain.Message, error) { + if r.deps.Channels == nil { + return domain.Message{}, errors.New("CHAT_ID_INVALID") + } + mentionUserIDs := r.mentionUserIDsFromDomain(ctx, botID, text, entities) + res, err := r.deps.Channels.SendMessage(ctx, botID, domain.SendChannelMessageRequest{ + UserID: botID, + ChannelID: channelID, + RandomID: randomNonZeroInt64(), + Message: text, + Entities: append([]domain.MessageEntity(nil), entities...), + Media: media, + MentionUserIDs: mentionUserIDs, + SkipRecipientLookup: true, + PostAuthor: r.channelPostAuthorName(ctx, botID), + Silent: silent, + ReplyTo: reply, + ReplyMarkup: replyMarkup, + Date: int(time.Now().Unix()), + }) + if err != nil { + return domain.Message{}, botAPIChannelSendErr(err) + } + if !res.Duplicate { + r.enqueueChannelMessageFanout(ctx, botID, res, nil) + r.pushChannelDiscussionUpdate(ctx, botID, res.Discussion) + r.maybeEnqueueWebPageResolve(botID, domain.Peer{Type: domain.PeerTypeChannel, ID: channelID}, res.Message.ID, res.Message.Media) + } + return botAPIMessageFromChannel(botID, res.Message), nil +} + +func botAPIMessageFromChannel(botID int64, msg domain.ChannelMessage) domain.Message { + from := msg.From + if from.Type == "" && msg.SenderUserID != 0 { + from = domain.Peer{Type: domain.PeerTypeUser, ID: msg.SenderUserID} + } + return domain.Message{ + ID: msg.ID, + RandomID: msg.RandomID, + OwnerUserID: botID, + Peer: domain.Peer{Type: domain.PeerTypeChannel, ID: msg.ChannelID}, + From: from, + Date: msg.Date, + EditDate: msg.EditDate, + Out: msg.SenderUserID == botID, + Silent: msg.Silent, + NoForwards: msg.NoForwards, + Body: msg.Body, + Entities: append([]domain.MessageEntity(nil), msg.Entities...), + ReplyTo: msg.ReplyTo, + Forward: msg.Forward, + Reactions: msg.Reactions, + Pts: msg.Pts, + TTLPeriod: msg.TTLPeriod, + ExpiresAt: msg.ExpiresAt, + Media: msg.Media, + MediaUnread: msg.MediaUnread, + ViaBotID: msg.ViaBotID, + GroupedID: msg.GroupedID, + ReplyMarkup: msg.ReplyMarkup, + RichMessage: msg.RichMessage, + Pinned: msg.Pinned, + } +} + +func botAPIChannelSendErr(err error) error { + switch { + case errors.Is(err, domain.ErrChannelInvalid), + errors.Is(err, domain.ErrChannelPrivate), + errors.Is(err, domain.ErrChannelUserBanned): + return errors.New("CHAT_ID_INVALID") + case errors.Is(err, domain.ErrChannelWriteForbidden): + return errors.New("CHAT_WRITE_FORBIDDEN") + case errors.Is(err, domain.ErrChannelAdminRequired): + return errors.New("CHAT_ADMIN_REQUIRED") + case errors.Is(err, domain.ErrReplyMessageIDInvalid): + return errors.New("REPLY_MESSAGE_ID_INVALID") + default: + return channelInvalidErr(err) + } +} + +func (r *Router) botAPIMedia(ctx context.Context, botID int64, kind, locationKey, remoteURL, fileName, mimeType string, fileBytes []byte) (*domain.MessageMedia, error) { + switch kind { + case "photo": + var photo domain.Photo + var err error + switch { + case len(fileBytes) > 0: + photo, err = r.deps.Files.CreatePhotoFromBytes(ctx, fileBytes) + case remoteURL != "": + photo, err = r.deps.Files.CreatePhotoFromURL(ctx, remoteURL) + case locationKey != "": + id, ok := botAPIPhotoID(locationKey) + if !ok { + return nil, errors.New("FILE_ID_INVALID") + } + var found bool + photo, found, err = r.deps.Files.GetPhoto(ctx, id) + if err == nil && !found { + err = errors.New("FILE_ID_INVALID") + } + default: + err = errors.New("FILE_ID_INVALID") + } + if err != nil { + return nil, botAPIMediaErr(err) + } + return &domain.MessageMedia{Kind: domain.MessageMediaKindPhoto, Photo: &photo}, nil + case "document": + var doc domain.Document + var err error + switch { + case len(fileBytes) > 0: + doc, err = r.deps.Files.CreateDocumentFromBytes(ctx, fileBytes, domain.DocumentSpec{ + MimeType: mimeType, + Attributes: botAPIDocumentAttributes(fileName), + ForceFile: true, + }) + case remoteURL != "": + doc, err = r.deps.Files.CreateDocumentFromURL(ctx, remoteURL) + case locationKey != "": + id, ok := botAPIDocumentID(locationKey) + if !ok { + return nil, errors.New("FILE_ID_INVALID") + } + var found bool + doc, found, err = r.deps.Files.GetDocument(ctx, id) + if err == nil && !found { + err = errors.New("FILE_ID_INVALID") + } + default: + err = errors.New("FILE_ID_INVALID") + } + if err != nil { + return nil, botAPIMediaErr(err) + } + return &domain.MessageMedia{Kind: domain.MessageMediaKindDocument, Document: &doc}, nil + default: + return nil, errors.New("MEDIA_INVALID") + } +} + +func botAPIPhotoID(locationKey string) (int64, bool) { + if !strings.HasPrefix(locationKey, "photo:") { + return 0, false + } + rest := strings.TrimPrefix(locationKey, "photo:") + idText, _, ok := strings.Cut(rest, ":") + if !ok { + return 0, false + } + id, err := strconv.ParseInt(idText, 10, 64) + return id, err == nil && id > 0 +} + +func botAPIDocumentID(locationKey string) (int64, bool) { + if !strings.HasPrefix(locationKey, "doc:") { + return 0, false + } + rest := strings.TrimPrefix(locationKey, "doc:") + idText, _, _ := strings.Cut(rest, ":") + id, err := strconv.ParseInt(idText, 10, 64) + return id, err == nil && id > 0 +} + +func botAPIDocumentAttributes(fileName string) []domain.DocumentAttribute { + fileName = strings.TrimSpace(fileName) + if fileName == "" { + return nil + } + return []domain.DocumentAttribute{{Kind: domain.DocAttrFilename, FileName: fileName}} +} + +func botAPIMediaErr(err error) error { + if err == nil { + return nil + } + if strings.Contains(strings.ToUpper(err.Error()), "FILE_ID_INVALID") { + return err + } + return errors.New("MEDIA_INVALID") +} + +// BotAPIEditMessageText edits a bot-owned private text message through the +// normal durable edit state machine. Positive chat_id is a private user chat. +func (r *Router) BotAPIEditMessageText(ctx context.Context, botID, chatID int64, messageID int, text string, entities []domain.MessageEntity, setReplyMarkup bool, replyMarkup *domain.MessageReplyMarkup, disableWebPagePreview bool) (domain.Message, error) { + if r == nil || r.deps.Messages == nil || botID == 0 { + return domain.Message{}, errors.New("BOT_INVALID") + } + if chatID <= 0 { + return domain.Message{}, errors.New("CHAT_ID_INVALID") + } + if messageID <= 0 || messageID > domain.MaxMessageBoxID { + return domain.Message{}, errors.New("MESSAGE_ID_INVALID") + } + if text == "" { + return domain.Message{}, errors.New("MESSAGE_EMPTY") + } + if utf8.RuneCountInString(text) > domain.MaxMessageTextLength { + return domain.Message{}, errors.New("MESSAGE_TOO_LONG") + } + peer := domain.Peer{Type: domain.PeerTypeUser, ID: chatID} + res, err := r.deps.Messages.EditMessage(ctx, botID, domain.EditMessageRequest{ + OwnerUserID: botID, + Peer: peer, + ID: messageID, + Message: text, + Entities: append([]domain.MessageEntity(nil), entities...), + EditDate: int(time.Now().Unix()), + SetReplyMarkup: setReplyMarkup, + ReplyMarkup: replyMarkup, + }) + if err != nil { + return domain.Message{}, err + } + self := res.Self() + if self.Message.ID == 0 { + return domain.Message{}, errors.New("MESSAGE_ID_INVALID") + } + return self.Message, nil +} + +// BotAPIDeleteMessage deletes a bot-owned private message with revoke=true so +// the target user's MTProto clients observe the normal delete update. +func (r *Router) BotAPIDeleteMessage(ctx context.Context, botID, chatID int64, messageID int) (bool, error) { + if r == nil || r.deps.Messages == nil || botID == 0 { + return false, errors.New("BOT_INVALID") + } + if chatID <= 0 { + return false, errors.New("CHAT_ID_INVALID") + } + if messageID <= 0 || messageID > domain.MaxMessageBoxID { + return false, errors.New("MESSAGE_ID_INVALID") + } + _, err := r.deps.Messages.DeleteMessages(ctx, botID, domain.DeleteMessagesRequest{ + OwnerUserID: botID, + IDs: []int{messageID}, + Revoke: true, + Date: int(time.Now().Unix()), + }) + if err != nil { + return false, err + } + return true, nil +} + +// BotAPIAnswerCallbackQuery bridges Bot API answerCallbackQuery to the same +// process-local callback registry used by messages.setBotCallbackAnswer. +func (r *Router) BotAPIAnswerCallbackQuery(ctx context.Context, botID int64, callbackQueryID, text, url string, showAlert bool, cacheTime int) (bool, error) { + if r == nil || r.callbacks == nil || botID == 0 { + return false, errors.New("BOT_INVALID") + } + queryID, err := strconv.ParseInt(callbackQueryID, 10, 64) + if err != nil || queryID == 0 { + return false, errors.New("QUERY_ID_INVALID") + } + if utf8.RuneCountInString(text) > domain.MaxBotCallbackAnswerLen { + return false, errors.New("MESSAGE_TOO_LONG") + } + if cacheTime < 0 { + cacheTime = 0 + } + r.callbacks.resolve(botID, queryID, domain.BotCallbackAnswer{ + Alert: showAlert, + Message: text, + URL: url, + CacheTime: cacheTime, + }) + return true, nil +} + +// BotAPIGetFile exposes the existing upload.getFile blob location space to the +// HTTP file endpoint after the bot token has authenticated the request. +func (r *Router) BotAPIGetFile(ctx context.Context, botID int64, locationKey string, offset int64, limit int) (domain.FileChunk, bool, error) { + if r == nil || r.deps.Files == nil || botID == 0 { + return domain.FileChunk{}, false, nil + } + if limit <= 0 || limit > maxUploadGetFileChunkLimit { + limit = maxUploadGetFileChunkLimit + } + if offset < 0 { + offset = 0 + } + return r.deps.Files.GetFile(ctx, domain.FileDownloadRequest{ + LocationKey: locationKey, + Offset: offset, + Limit: limit, + }) +} diff --git a/internal/rpc/botapi_gateway_test.go b/internal/rpc/botapi_gateway_test.go new file mode 100644 index 00000000..7cb23d4c --- /dev/null +++ b/internal/rpc/botapi_gateway_test.go @@ -0,0 +1,387 @@ +package rpc + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/gotd/td/clock" + "go.uber.org/zap/zaptest" + + appbots "telesrv/internal/app/bots" + appchannels "telesrv/internal/app/channels" + appmessages "telesrv/internal/app/messages" + appusers "telesrv/internal/app/users" + "telesrv/internal/domain" + "telesrv/internal/store/memory" +) + +func TestBotAPISendMessageToSupergroupChatID(t *testing.T) { + ctx := context.Background() + userStore := memory.NewUserStore() + owner, err := userStore.Create(ctx, domain.User{AccessHash: 1001, Phone: "15550008001", FirstName: "Owner"}) + if err != nil { + t.Fatalf("create owner: %v", err) + } + bot, err := userStore.Create(ctx, domain.User{AccessHash: 2001, Phone: "15550008002", FirstName: "TetrisBot", Username: "TetrisBot", Bot: true}) + if err != nil { + t.Fatalf("create bot: %v", err) + } + + channelStore := memory.NewChannelStore() + channelService := appchannels.NewService(channelStore) + created, err := channelService.CreateMegagroupFromCreateChat(ctx, owner.ID, domain.CreateChannelRequest{ + Title: "Group1", + MemberUserIDs: []int64{bot.ID}, + Date: 10, + }) + if err != nil { + t.Fatalf("create megagroup: %v", err) + } + sessions := &captureSessions{ + channelMembers: map[int64][]int64{created.Channel.ID: {owner.ID}}, + } + r := New(Config{}, Deps{ + Users: appusers.NewService(userStore), + Channels: channelService, + Sessions: sessions, + }, zaptest.NewLogger(t), clock.System) + + chatID := -botAPIChannelChatIDBase - created.Channel.ID + msg, err := r.BotAPISendMessage(ctx, bot.ID, chatID, "hello Group1 from bot api", nil, nil, false, false, 0) + if err != nil { + t.Fatalf("BotAPISendMessage: %v", err) + } + if msg.Peer.Type != domain.PeerTypeChannel || msg.Peer.ID != created.Channel.ID { + t.Fatalf("msg peer = %+v, want channel %d", msg.Peer, created.Channel.ID) + } + if msg.From.Type != domain.PeerTypeUser || msg.From.ID != bot.ID || !msg.Out { + t.Fatalf("msg from/out = %+v out=%v, want bot outbound", msg.From, msg.Out) + } + if msg.Body != "hello Group1 from bot api" || msg.ID == 0 || msg.Pts == 0 { + t.Fatalf("msg = %+v, want durable channel message with id and pts", msg) + } + + history, err := channelService.GetHistory(ctx, owner.ID, domain.ChannelHistoryFilter{ChannelID: created.Channel.ID, Limit: 10}) + if err != nil { + t.Fatalf("GetHistory: %v", err) + } + if len(history.Messages) < 1 || history.Messages[0].SenderUserID != bot.ID || history.Messages[0].Body != msg.Body { + t.Fatalf("history messages = %+v, want bot channel message", history.Messages) + } + if pushed := sessions.pushedUserIDs(); !fanoutHasID(pushed, owner.ID) { + t.Fatalf("fanout pushed = %v, want owner %d to receive online channel update", pushed, owner.ID) + } +} + +func TestBotAPISendMessageRejectsUnsupportedNegativeChatID(t *testing.T) { + r := New(Config{}, Deps{}, zaptest.NewLogger(t), clock.System) + + _, err := r.BotAPISendMessage(context.Background(), 1234, -42, "hello", nil, nil, false, false, 0) + if err == nil || !strings.Contains(err.Error(), "CHAT_ID_INVALID") { + t.Fatalf("BotAPISendMessage err = %v, want CHAT_ID_INVALID", err) + } +} + +func TestBotAPISendMessageMissingSupergroupReturnsChatIDInvalid(t *testing.T) { + channelService := appchannels.NewService(memory.NewChannelStore()) + r := New(Config{}, Deps{Channels: channelService}, zaptest.NewLogger(t), clock.System) + + chatID := -botAPIChannelChatIDBase - 9999 + _, err := r.BotAPISendMessage(context.Background(), 1234, chatID, "hello", nil, nil, false, false, 0) + if err == nil || !strings.Contains(err.Error(), "CHAT_ID_INVALID") { + t.Fatalf("BotAPISendMessage err = %v, want CHAT_ID_INVALID", err) + } +} + +func TestBotAPIGetUpdatesReceivesVisibleSupergroupMessage(t *testing.T) { + fixture := newBotAPIReceiveFixture(t, false) + res, err := fixture.channels.SendMessage(fixture.ctx, fixture.owner.ID, domain.SendChannelMessageRequest{ + UserID: fixture.owner.ID, + ChannelID: fixture.channel.ID, + RandomID: 1001, + Message: "/ping from group", + SkipRecipientLookup: true, + Date: 100, + }) + if err != nil { + t.Fatalf("SendMessage: %v", err) + } + fixture.router.enqueueChannelMessageFanout(fixture.ctx, fixture.owner.ID, res, nil) + + events, err := fixture.router.BotAPIUpdates(fixture.ctx, fixture.bot.ID, 0) + if err != nil { + t.Fatalf("BotAPIUpdates: %v", err) + } + if len(events) != 1 { + t.Fatalf("events = %+v, want one bot api update", events) + } + event := events[0] + if event.Type != domain.UpdateEventNewMessage || event.Pts <= 0 { + t.Fatalf("event = %+v, want new_message with update_id", event) + } + if event.Message.Peer.Type != domain.PeerTypeChannel || event.Message.Peer.ID != fixture.channel.ID { + t.Fatalf("message peer = %+v, want channel %d", event.Message.Peer, fixture.channel.ID) + } + if event.Message.From.Type != domain.PeerTypeUser || event.Message.From.ID != fixture.owner.ID || event.Message.Body != "/ping from group" || event.Message.Out { + t.Fatalf("message = %+v, want incoming owner command", event.Message) + } + + next, err := fixture.router.BotAPIUpdates(fixture.ctx, fixture.bot.ID, int64(event.Pts)+1) + if err != nil { + t.Fatalf("BotAPIUpdates confirm: %v", err) + } + if len(next) != 0 { + t.Fatalf("next events = %+v, want empty after offset confirm", next) + } +} + +func TestBotAPIGetUpdatesSkipsHiddenPrivacySupergroupMessage(t *testing.T) { + fixture := newBotAPIReceiveFixture(t, false) + res, err := fixture.channels.SendMessage(fixture.ctx, fixture.owner.ID, domain.SendChannelMessageRequest{ + UserID: fixture.owner.ID, + ChannelID: fixture.channel.ID, + RandomID: 1002, + Message: "plain group chatter", + SkipRecipientLookup: true, + Date: 101, + }) + if err != nil { + t.Fatalf("SendMessage: %v", err) + } + if len(res.SkipDeliveryUserIDs) == 0 { + t.Fatalf("SkipDeliveryUserIDs empty, want privacy bot excluded") + } + fixture.router.enqueueChannelMessageFanout(fixture.ctx, fixture.owner.ID, res, nil) + + events, err := fixture.router.BotAPIUpdates(fixture.ctx, fixture.bot.ID, 0) + if err != nil { + t.Fatalf("BotAPIUpdates: %v", err) + } + if len(events) != 0 { + t.Fatalf("events = %+v, want hidden privacy message excluded", events) + } +} + +func TestBotAPIGetUpdatesReceivesPrivateBotMessage(t *testing.T) { + fixture := newBotAPIReceiveFixture(t, false) + res, err := fixture.messages.SendPrivateText(fixture.ctx, fixture.owner.ID, domain.SendPrivateTextRequest{ + SenderUserID: fixture.owner.ID, + RecipientUserID: fixture.bot.ID, + RandomID: 2001, + Message: "private hello", + Date: 102, + }) + if err != nil { + t.Fatalf("SendPrivateText: %v", err) + } + fixture.router.enqueueBotAPIPrivateMessageUpdate(fixture.ctx, res) + + events, err := fixture.router.BotAPIUpdates(fixture.ctx, fixture.bot.ID, 0) + if err != nil { + t.Fatalf("BotAPIUpdates: %v", err) + } + if len(events) != 1 || events[0].Message.Peer.Type != domain.PeerTypeUser || events[0].Message.Peer.ID != fixture.owner.ID || events[0].Message.Body != "private hello" { + t.Fatalf("events = %+v, want private incoming message", events) + } +} + +func TestBotAPIGetUpdatesBatchesPrivateMessageProjection(t *testing.T) { + fixture := newBotAPIReceiveFixture(t, false) + counting := &countingBotAPIMessagesService{Service: fixture.messages} + fixture.router.deps.Messages = counting + for i, text := range []string{"private one", "private two"} { + res, err := fixture.messages.SendPrivateText(fixture.ctx, fixture.owner.ID, domain.SendPrivateTextRequest{ + SenderUserID: fixture.owner.ID, + RecipientUserID: fixture.bot.ID, + RandomID: int64(2100 + i), + Message: text, + Date: 120 + i, + }) + if err != nil { + t.Fatalf("SendPrivateText %d: %v", i, err) + } + fixture.router.enqueueBotAPIPrivateMessageUpdate(fixture.ctx, res) + } + + events, err := fixture.router.BotAPIUpdates(fixture.ctx, fixture.bot.ID, 0) + if err != nil { + t.Fatalf("BotAPIUpdates: %v", err) + } + if len(events) != 2 { + t.Fatalf("events = %+v, want two private updates", events) + } + if counting.getMessagesCalls != 1 { + t.Fatalf("private GetMessages calls = %d, want 1 batched projection", counting.getMessagesCalls) + } +} + +func TestBotAPIUpdateWaiterWakesOnPrivateEnqueue(t *testing.T) { + fixture := newBotAPIReceiveFixture(t, false) + version := fixture.router.BotAPIUpdateWaitVersion(fixture.bot.ID) + woke := make(chan bool, 1) + go func() { + woke <- fixture.router.WaitBotAPIUpdate(fixture.ctx, fixture.bot.ID, version, time.Second) + }() + res, err := fixture.messages.SendPrivateText(fixture.ctx, fixture.owner.ID, domain.SendPrivateTextRequest{ + SenderUserID: fixture.owner.ID, + RecipientUserID: fixture.bot.ID, + RandomID: 2201, + Message: "wake bot api polling", + Date: 121, + }) + if err != nil { + t.Fatalf("SendPrivateText: %v", err) + } + fixture.router.enqueueBotAPIPrivateMessageUpdate(fixture.ctx, res) + select { + case ok := <-woke: + if !ok { + t.Fatalf("WaitBotAPIUpdate returned false, want notify wake") + } + case <-time.After(time.Second): + t.Fatal("WaitBotAPIUpdate did not wake after enqueue") + } +} + +func TestBotAPIChannelBatchEnqueueLoadsBotCandidatesOnce(t *testing.T) { + fixture := newBotAPIReceiveFixture(t, false) + counting := &countingBotCandidateChannelsService{Service: fixture.channels} + fixture.router.deps.Channels = counting + first, err := fixture.channels.SendMessage(fixture.ctx, fixture.owner.ID, domain.SendChannelMessageRequest{ + UserID: fixture.owner.ID, + ChannelID: fixture.channel.ID, + RandomID: 3001, + Message: "/first batch command", + SkipRecipientLookup: true, + Date: 103, + }) + if err != nil { + t.Fatalf("SendMessage first: %v", err) + } + second, err := fixture.channels.SendMessage(fixture.ctx, fixture.owner.ID, domain.SendChannelMessageRequest{ + UserID: fixture.owner.ID, + ChannelID: fixture.channel.ID, + RandomID: 3002, + Message: "/second batch command", + SkipRecipientLookup: true, + Date: 104, + }) + if err != nil { + t.Fatalf("SendMessage second: %v", err) + } + + fixture.router.enqueueBotAPIChannelMessagesUpdate(fixture.ctx, fixture.owner.ID, []domain.SendChannelMessageResult{first, second}) + if counting.activeBotMemberIDsCalls != 1 { + t.Fatalf("ActiveBotMemberIDs calls = %d, want 1 for same-channel batch", counting.activeBotMemberIDsCalls) + } + if counting.activeMemberIDsCalls != 0 { + t.Fatalf("ActiveMemberIDs calls = %d, want 0 on Bot API enqueue path", counting.activeMemberIDsCalls) + } + if counting.getMessagesCalls != 0 { + t.Fatalf("channel GetMessages calls during enqueue = %d, want 0 on ordinary send path", counting.getMessagesCalls) + } + events, err := fixture.router.BotAPIUpdates(fixture.ctx, fixture.bot.ID, 0) + if err != nil { + t.Fatalf("BotAPIUpdates: %v", err) + } + if len(events) != 2 { + t.Fatalf("events = %+v, want two bot api updates", events) + } + if counting.getMessagesCalls != 1 { + t.Fatalf("channel GetMessages calls after getUpdates = %d, want 1 batched projection", counting.getMessagesCalls) + } +} + +type countingBotCandidateChannelsService struct { + *appchannels.Service + activeBotMemberIDsCalls int + activeMemberIDsCalls int + getMessagesCalls int +} + +func (s *countingBotCandidateChannelsService) ActiveBotMemberIDs(ctx context.Context, viewerUserID, channelID int64, limit int) ([]int64, error) { + s.activeBotMemberIDsCalls++ + return s.Service.ActiveBotMemberIDs(ctx, viewerUserID, channelID, limit) +} + +func (s *countingBotCandidateChannelsService) ActiveMemberIDs(ctx context.Context, userID, channelID int64, limit int) ([]int64, error) { + s.activeMemberIDsCalls++ + return s.Service.ActiveMemberIDs(ctx, userID, channelID, limit) +} + +func (s *countingBotCandidateChannelsService) GetMessages(ctx context.Context, userID, channelID int64, ids []int) (domain.ChannelHistory, error) { + s.getMessagesCalls++ + return s.Service.GetMessages(ctx, userID, channelID, ids) +} + +type countingBotAPIMessagesService struct { + *appmessages.Service + getMessagesCalls int +} + +func (s *countingBotAPIMessagesService) GetMessages(ctx context.Context, userID int64, ids []int) (domain.MessageList, error) { + s.getMessagesCalls++ + return s.Service.GetMessages(ctx, userID, ids) +} + +type botAPIReceiveFixture struct { + ctx context.Context + owner domain.User + bot domain.User + channel domain.Channel + router *Router + channels *appchannels.Service + messages *appmessages.Service +} + +func newBotAPIReceiveFixture(t *testing.T, botChatHistory bool) botAPIReceiveFixture { + t.Helper() + ctx := context.Background() + userStore := memory.NewUserStore() + owner, err := userStore.Create(ctx, domain.User{AccessHash: 1001, Phone: "15550008101", FirstName: "Owner"}) + if err != nil { + t.Fatalf("create owner: %v", err) + } + dialogStore := memory.NewDialogStore() + messageStore := memory.NewMessageStore(dialogStore) + botStore := memory.NewBotStore(userStore) + bot, _, err := botStore.CreateBotAccount(ctx, domain.User{AccessHash: 2001, FirstName: "TetrisBot", Username: "TetrisBot"}, domain.BotProfile{ + OwnerUserID: owner.ID, + TokenSecret: "secret", + ChatHistory: botChatHistory, + }) + if err != nil { + t.Fatalf("create bot: %v", err) + } + botsService := appbots.NewService(userStore, botStore, messageStore) + channelStore := memory.NewChannelStore() + channelsService := appchannels.NewService(channelStore, appchannels.WithBotProfileResolver(botsService)) + created, err := channelsService.CreateMegagroupFromCreateChat(ctx, owner.ID, domain.CreateChannelRequest{ + Title: "Group1", + MemberUserIDs: []int64{bot.ID}, + Date: 10, + }) + if err != nil { + t.Fatalf("create megagroup: %v", err) + } + messagesService := appmessages.NewService(messageStore, dialogStore) + router := New(Config{}, Deps{ + Users: appusers.NewService(userStore), + Messages: messagesService, + Channels: channelsService, + Bots: botsService, + BotAPIUpdates: memory.NewBotAPIUpdateStore(), + Sessions: &captureSessions{channelMembers: map[int64][]int64{created.Channel.ID: {owner.ID, bot.ID}}}, + }, zaptest.NewLogger(t), clock.System) + return botAPIReceiveFixture{ + ctx: ctx, + owner: owner, + bot: bot, + channel: created.Channel, + router: router, + channels: channelsService, + messages: messagesService, + } +} diff --git a/internal/rpc/botapi_update_notifier.go b/internal/rpc/botapi_update_notifier.go new file mode 100644 index 00000000..ea4f67de --- /dev/null +++ b/internal/rpc/botapi_update_notifier.go @@ -0,0 +1,111 @@ +package rpc + +import ( + "context" + "sync" + "time" +) + +type botAPIUpdateNotifier struct { + mu sync.Mutex + version map[int64]uint64 + waiters map[int64]map[chan struct{}]struct{} +} + +func newBotAPIUpdateNotifier() *botAPIUpdateNotifier { + return &botAPIUpdateNotifier{ + version: make(map[int64]uint64), + waiters: make(map[int64]map[chan struct{}]struct{}), + } +} + +func (n *botAPIUpdateNotifier) current(botID int64) uint64 { + if n == nil || botID == 0 { + return 0 + } + n.mu.Lock() + defer n.mu.Unlock() + return n.version[botID] +} + +func (n *botAPIUpdateNotifier) wait(ctx context.Context, botID int64, version uint64, timeout time.Duration) bool { + if n == nil || botID == 0 || timeout <= 0 { + return false + } + ch := make(chan struct{}) + n.mu.Lock() + if n.version[botID] != version { + n.mu.Unlock() + return true + } + waiters := n.waiters[botID] + if waiters == nil { + waiters = make(map[chan struct{}]struct{}) + n.waiters[botID] = waiters + } + waiters[ch] = struct{}{} + n.mu.Unlock() + + timer := time.NewTimer(timeout) + defer timer.Stop() + defer n.remove(botID, ch) + select { + case <-ch: + return true + case <-ctx.Done(): + return false + case <-timer.C: + return false + } +} + +func (n *botAPIUpdateNotifier) notify(botID int64) { + if n == nil || botID == 0 { + return + } + n.mu.Lock() + n.version[botID]++ + waiters := n.waiters[botID] + delete(n.waiters, botID) + n.mu.Unlock() + for ch := range waiters { + close(ch) + } +} + +func (n *botAPIUpdateNotifier) remove(botID int64, ch chan struct{}) { + if n == nil || botID == 0 || ch == nil { + return + } + n.mu.Lock() + defer n.mu.Unlock() + waiters := n.waiters[botID] + if waiters == nil { + return + } + delete(waiters, ch) + if len(waiters) == 0 { + delete(n.waiters, botID) + } +} + +func (r *Router) BotAPIUpdateWaitVersion(botID int64) uint64 { + if r == nil { + return 0 + } + return r.botAPIUpdates.current(botID) +} + +func (r *Router) WaitBotAPIUpdate(ctx context.Context, botID int64, version uint64, timeout time.Duration) bool { + if r == nil { + return false + } + return r.botAPIUpdates.wait(ctx, botID, version, timeout) +} + +func (r *Router) notifyBotAPIUpdate(botID int64) { + if r == nil { + return + } + r.botAPIUpdates.notify(botID) +} diff --git a/internal/rpc/botapi_update_queue.go b/internal/rpc/botapi_update_queue.go new file mode 100644 index 00000000..5aec6a53 --- /dev/null +++ b/internal/rpc/botapi_update_queue.go @@ -0,0 +1,416 @@ +package rpc + +import ( + "context" + + "telesrv/internal/domain" +) + +const botAPIGetUpdatesLimit = 100 + +type botAPIChannelBotMemberProvider interface { + ActiveBotMemberIDs(ctx context.Context, viewerUserID, channelID int64, limit int) ([]int64, error) +} + +func (r *Router) botAPIQueuedUpdates(ctx context.Context, botID int64, offset int64) ([]domain.UpdateEvent, error) { + if r == nil || r.deps.BotAPIUpdates == nil || botID == 0 { + return nil, nil + } + fromID := int64(1) + if offset > 0 { + if err := r.deps.BotAPIUpdates.ConfirmBotAPIUpdates(ctx, botID, offset-1); err != nil { + return nil, err + } + fromID = offset + } else if confirmed, found, err := r.deps.BotAPIUpdates.ConfirmedBotAPIUpdateID(ctx, botID); err != nil { + return nil, err + } else if found { + fromID = confirmed + 1 + } + items, err := r.deps.BotAPIUpdates.ListBotAPIUpdates(ctx, botID, fromID, botAPIGetUpdatesLimit) + if err != nil { + return nil, err + } + if len(items) == 0 { + return nil, nil + } + events, leadingSkipped := r.botAPIQueuedUpdateEvents(ctx, botID, items) + if leadingSkipped > 0 { + if err := r.deps.BotAPIUpdates.ConfirmBotAPIUpdates(ctx, botID, leadingSkipped); err != nil { + return nil, err + } + } + if len(events) == 0 { + return nil, nil + } + return r.enrichUpdateEvents(ctx, botID, events), nil +} + +func (r *Router) botAPIQueuedUpdateEvents(ctx context.Context, botID int64, items []domain.BotAPIUpdate) ([]domain.UpdateEvent, int64) { + privateIDs := make([]int, 0) + privateSeen := make(map[int]struct{}) + channelIDs := make(map[int64][]int) + channelSeen := make(map[int64]map[int]struct{}) + for _, item := range items { + if _, ok := botAPIQueuedUpdateKind(botID, item); !ok { + continue + } + switch item.Peer.Type { + case domain.PeerTypeUser: + if _, exists := privateSeen[item.MessageID]; !exists { + privateSeen[item.MessageID] = struct{}{} + privateIDs = append(privateIDs, item.MessageID) + } + case domain.PeerTypeChannel: + seen := channelSeen[item.Peer.ID] + if seen == nil { + seen = make(map[int]struct{}) + channelSeen[item.Peer.ID] = seen + } + if _, exists := seen[item.MessageID]; !exists { + seen[item.MessageID] = struct{}{} + channelIDs[item.Peer.ID] = append(channelIDs[item.Peer.ID], item.MessageID) + } + } + } + + privateMessages := r.botAPIQueuedPrivateMessages(ctx, botID, privateIDs) + channelMessages := r.botAPIQueuedChannelMessages(ctx, botID, channelIDs) + events := make([]domain.UpdateEvent, 0, len(items)) + leadingSkipped := int64(0) + for _, item := range items { + event, ok := botAPIQueuedUpdateEventFromMessages(botID, item, privateMessages, channelMessages) + if !ok { + if len(events) == 0 { + leadingSkipped = item.ID + } + continue + } + events = append(events, event) + } + return events, leadingSkipped +} + +func (r *Router) botAPIQueuedPrivateMessages(ctx context.Context, botID int64, ids []int) map[int]domain.Message { + if r == nil || r.deps.Messages == nil || len(ids) == 0 { + return nil + } + list, err := r.deps.Messages.GetMessages(ctx, botID, ids) + if err != nil { + return nil + } + out := make(map[int]domain.Message, len(list.Messages)) + for _, msg := range list.Messages { + if msg.ID <= 0 || msg.Out || !botAPIMessageProjectable(msg) { + continue + } + out[msg.ID] = msg + } + return out +} + +func (r *Router) botAPIQueuedChannelMessages(ctx context.Context, botID int64, idsByChannel map[int64][]int) map[int64]map[int]domain.ChannelMessage { + if r == nil || r.deps.Channels == nil || len(idsByChannel) == 0 { + return nil + } + out := make(map[int64]map[int]domain.ChannelMessage, len(idsByChannel)) + for channelID, ids := range idsByChannel { + if channelID == 0 || len(ids) == 0 { + continue + } + history, err := r.deps.Channels.GetMessages(ctx, botID, channelID, ids) + if err != nil { + continue + } + byID := make(map[int]domain.ChannelMessage, len(history.Messages)) + for _, msg := range history.Messages { + if msg.ID <= 0 || msg.Deleted || msg.Action != nil { + continue + } + projected := botAPIMessageFromChannel(botID, msg) + if projected.Out || !botAPIMessageProjectable(projected) { + continue + } + byID[msg.ID] = msg + } + if len(byID) > 0 { + out[channelID] = byID + } + } + return out +} + +func botAPIQueuedUpdateKind(botID int64, item domain.BotAPIUpdate) (domain.UpdateEventType, bool) { + if item.ID <= 0 || item.BotUserID != botID || item.MessageID <= 0 { + return "", false + } + eventType, ok := botAPIUpdateEventType(item.Kind) + if !ok { + return "", false + } + switch item.Peer.Type { + case domain.PeerTypeUser, domain.PeerTypeChannel: + if item.Peer.ID <= 0 { + return "", false + } + default: + return "", false + } + return eventType, true +} + +func botAPIQueuedUpdateEventFromMessages(botID int64, item domain.BotAPIUpdate, privateMessages map[int]domain.Message, channelMessages map[int64]map[int]domain.ChannelMessage) (domain.UpdateEvent, bool) { + eventType, ok := botAPIQueuedUpdateKind(botID, item) + if !ok { + return domain.UpdateEvent{}, false + } + switch item.Peer.Type { + case domain.PeerTypeUser: + msg, found := privateMessages[item.MessageID] + if !found { + return domain.UpdateEvent{}, false + } + msg.Pts = int(item.ID) + return domain.UpdateEvent{ + UserID: botID, + Type: eventType, + Pts: int(item.ID), + PtsCount: 1, + Date: item.Date, + Peer: msg.Peer, + Message: msg, + }, true + case domain.PeerTypeChannel: + msg, found := channelMessages[item.Peer.ID][item.MessageID] + if !found { + return domain.UpdateEvent{}, false + } + projected := botAPIMessageFromChannel(botID, msg) + projected.Pts = int(item.ID) + return domain.UpdateEvent{ + UserID: botID, + Type: eventType, + Pts: int(item.ID), + PtsCount: 1, + Date: item.Date, + Peer: projected.Peer, + Message: projected, + }, true + default: + return domain.UpdateEvent{}, false + } +} + +func botAPIUpdateEventType(kind domain.BotAPIUpdateKind) (domain.UpdateEventType, bool) { + switch kind { + case domain.BotAPIUpdateMessage: + return domain.UpdateEventNewMessage, true + case domain.BotAPIUpdateEditedMessage: + return domain.UpdateEventEditMessage, true + default: + return "", false + } +} + +// enqueueBotAPIPrivateMessageUpdateAsync 把私聊消息的 Bot API 队列写入投给后台 +// dispatcher（性能审计 H2）：发送者 RPC 不再为 bot 判定 miss / INSERT 多等 PG 往返。 +// dispatcher 未启动（测试/未装配）时同步执行，行为不变。 +func (r *Router) enqueueBotAPIPrivateMessageUpdateAsync(ctx context.Context, res domain.SendPrivateTextResult) { + if r == nil || r.deps.BotAPIUpdates == nil || res.Duplicate || res.RecipientMessage.ID <= 0 { + return + } + r.botAPIEnqueueQueue.Enqueue(ctx, func(jobCtx context.Context) { + r.enqueueBotAPIPrivateMessageUpdate(jobCtx, res) + }) +} + +// enqueueBotAPIPrivateEditUpdatesAsync 同上，覆盖私聊编辑的 edited_message 队列写入。 +func (r *Router) enqueueBotAPIPrivateEditUpdatesAsync(ctx context.Context, res domain.EditMessageResult) { + if r == nil || r.deps.BotAPIUpdates == nil || len(res.Edited) == 0 { + return + } + r.botAPIEnqueueQueue.Enqueue(ctx, func(jobCtx context.Context) { + r.enqueueBotAPIPrivateEditUpdates(jobCtx, res) + }) +} + +func (r *Router) enqueueBotAPIPrivateMessageUpdate(ctx context.Context, res domain.SendPrivateTextResult) { + if r == nil || r.deps.BotAPIUpdates == nil || res.Duplicate || res.RecipientMessage.ID <= 0 || res.RecipientMessage.Out { + return + } + botID := res.RecipientMessage.OwnerUserID + if botID == 0 || !botAPIMessageProjectable(res.RecipientMessage) { + return + } + isBot, err := r.botAPIKnownBot(ctx, botID) + if err != nil || !isBot { + return + } + if _, created, err := r.deps.BotAPIUpdates.EnqueueBotAPIUpdate(ctx, domain.EnqueueBotAPIUpdateRequest{ + BotUserID: botID, + Kind: domain.BotAPIUpdateMessage, + Peer: res.RecipientMessage.Peer, + MessageID: res.RecipientMessage.ID, + SourcePts: res.RecipientEvent.Pts, + Date: res.RecipientMessage.Date, + }); err == nil && created { + r.notifyBotAPIUpdate(botID) + } +} + +func (r *Router) enqueueBotAPIPrivateEditUpdates(ctx context.Context, res domain.EditMessageResult) { + if r == nil || r.deps.BotAPIUpdates == nil { + return + } + for _, item := range res.Edited { + if item.UserID == 0 || item.Message.ID <= 0 || item.Message.Out || !botAPIMessageProjectable(item.Message) { + continue + } + isBot, err := r.botAPIKnownBot(ctx, item.UserID) + if err != nil || !isBot { + continue + } + if _, created, err := r.deps.BotAPIUpdates.EnqueueBotAPIUpdate(ctx, domain.EnqueueBotAPIUpdateRequest{ + BotUserID: item.UserID, + Kind: domain.BotAPIUpdateEditedMessage, + Peer: item.Message.Peer, + MessageID: item.Message.ID, + SourcePts: item.Event.Pts, + Date: item.Message.EditDate, + }); err == nil && created { + r.notifyBotAPIUpdate(item.UserID) + } + } +} + +func (r *Router) enqueueBotAPIChannelMessageUpdate(ctx context.Context, originUserID int64, res domain.SendChannelMessageResult) { + if r == nil || r.deps.BotAPIUpdates == nil || r.deps.Channels == nil || res.Duplicate || res.Message.ID <= 0 || res.Message.ChannelID == 0 { + return + } + botIDs, err := r.botAPIChannelBotCandidates(ctx, originUserID, res.Message.ChannelID) + if err != nil { + return + } + r.enqueueBotAPIChannelMessageUpdateForBots(ctx, res, botIDs) +} + +func (r *Router) enqueueBotAPIChannelMessageUpdateForBots(ctx context.Context, res domain.SendChannelMessageResult, botIDs []int64) { + if r == nil || r.deps.BotAPIUpdates == nil || res.Duplicate || res.Message.ID <= 0 || res.Message.ChannelID == 0 || len(botIDs) == 0 { + return + } + skip := skipDeliverySet(res.SkipDeliveryUserIDs) + for _, botID := range botIDs { + if botID == 0 || botID == res.Message.SenderUserID { + continue + } + if _, skipped := skip[botID]; skipped { + continue + } + if _, created, err := r.deps.BotAPIUpdates.EnqueueBotAPIUpdate(ctx, domain.EnqueueBotAPIUpdateRequest{ + BotUserID: botID, + Kind: domain.BotAPIUpdateMessage, + Peer: domain.Peer{Type: domain.PeerTypeChannel, ID: res.Message.ChannelID}, + MessageID: res.Message.ID, + SourcePts: res.Event.Pts, + Date: res.Message.Date, + }); err == nil && created { + r.notifyBotAPIUpdate(botID) + } + } +} + +func (r *Router) enqueueBotAPIChannelMessagesUpdate(ctx context.Context, originUserID int64, results []domain.SendChannelMessageResult) { + candidates := make(map[int64][]int64) + for _, res := range results { + if r == nil || r.deps.BotAPIUpdates == nil || r.deps.Channels == nil || res.Duplicate || res.Message.ID <= 0 || res.Message.ChannelID == 0 { + continue + } + botIDs, ok := candidates[res.Message.ChannelID] + if !ok { + loaded, err := r.botAPIChannelBotCandidates(ctx, originUserID, res.Message.ChannelID) + if err != nil { + candidates[res.Message.ChannelID] = nil + continue + } + botIDs = loaded + candidates[res.Message.ChannelID] = botIDs + } + r.enqueueBotAPIChannelMessageUpdateForBots(ctx, res, botIDs) + } +} + +func (r *Router) enqueueBotAPIChannelEditMessageUpdate(ctx context.Context, originUserID int64, res domain.EditChannelMessageResult) { + if r == nil || r.deps.BotAPIUpdates == nil || r.deps.Channels == nil || res.Message.ID <= 0 || res.Message.ChannelID == 0 || res.Event.Pts == 0 { + return + } + botIDs, err := r.botAPIChannelBotCandidates(ctx, originUserID, res.Message.ChannelID) + if err != nil { + return + } + date := res.Message.EditDate + if date == 0 { + date = res.Message.Date + } + for _, botID := range botIDs { + if botID == 0 || botID == res.Message.SenderUserID { + continue + } + if _, created, err := r.deps.BotAPIUpdates.EnqueueBotAPIUpdate(ctx, domain.EnqueueBotAPIUpdateRequest{ + BotUserID: botID, + Kind: domain.BotAPIUpdateEditedMessage, + Peer: domain.Peer{Type: domain.PeerTypeChannel, ID: res.Message.ChannelID}, + MessageID: res.Message.ID, + SourcePts: res.Event.Pts, + Date: date, + }); err == nil && created { + r.notifyBotAPIUpdate(botID) + } + } +} + +func (r *Router) botAPIChannelBotCandidates(ctx context.Context, viewerUserID, channelID int64) ([]int64, error) { + if r == nil || r.deps.Channels == nil || channelID == 0 { + return nil, nil + } + provider, ok := r.deps.Channels.(botAPIChannelBotMemberProvider) + if !ok { + return nil, nil + } + return provider.ActiveBotMemberIDs(ctx, viewerUserID, channelID, domain.MaxSynchronousChannelDialogFanout) +} + +func (r *Router) botAPIKnownBot(ctx context.Context, botID int64) (bool, error) { + if botID == 0 { + return false, nil + } + if r.deps.Bots != nil { + if _, found, err := r.deps.Bots.BotInfo(ctx, botID); err != nil || found { + return found, err + } + } + return r.userIsBot(ctx, botID), nil +} + +func botAPIMessageProjectable(msg domain.Message) bool { + if msg.ID <= 0 || msg.Out { + return false + } + if msg.Body != "" { + return true + } + return botAPIMessageMediaProjectable(msg.Media) +} + +func botAPIMessageMediaProjectable(media *domain.MessageMedia) bool { + if media.IsZero() { + return false + } + switch media.Kind { + case domain.MessageMediaKindPhoto: + return media.Photo != nil + case domain.MessageMediaKindDocument: + return media.Document != nil + default: + return false + } +} diff --git a/internal/rpc/channel_fanout_dispatcher.go b/internal/rpc/channel_fanout_dispatcher.go index 2faf4353..29ad32fc 100644 --- a/internal/rpc/channel_fanout_dispatcher.go +++ b/internal/rpc/channel_fanout_dispatcher.go @@ -273,6 +273,7 @@ func channelMessagesFanoutOwnerIDs(results []domain.SendChannelMessageResult, ex // 类事件的常见形态：发送/转发单条/讨论组联动/forum topic 消息）。语义与 enqueueChannelFanout 一致， // 仅多了把每 viewer 投影一次性算好预热进共享 cache（O(owner)），不改变投递/排除/nudge 行为。 func (r *Router) enqueueChannelMessageFanout(ctx context.Context, originUserID int64, res domain.SendChannelMessageResult, extraUserIDs []int64) { + r.enqueueBotAPIChannelMessageUpdate(ctx, originUserID, res) fanoutCache := newViewerPeerCache(r) ownerIDs := channelMessageFanoutOwnerIDs(res, extraUserIDs) skip := skipDeliverySet(res.SkipDeliveryUserIDs) @@ -336,6 +337,7 @@ func channelEditMessageFanoutOwnerIDs(res domain.EditChannelMessageResult) []int // nudge 须带 channel 当前最高 pts 才能让 >cap 在线成员的 getChannelDifference 拉齐到末尾——用 Event.Pts // 会在 Event.Pts==0 时漏发 nudge、或低于真实 pts。max() 在三种形态（仅 Event/仅 ServiceEvent/两者）都正确。 func (r *Router) enqueueChannelEditMessageFanout(ctx context.Context, originUserID int64, res domain.EditChannelMessageResult) { + r.enqueueBotAPIChannelEditMessageUpdate(ctx, originUserID, res) fanoutCache := newViewerPeerCache(r) ownerIDs := channelEditMessageFanoutOwnerIDs(res) nudgePts := max(res.Event.Pts, res.ServiceEvent.Pts) @@ -352,6 +354,7 @@ func (r *Router) enqueueChannelEditMessageFanout(ctx context.Context, originUser // 一个 Updates 内含多条 UpdateNewChannelMessage），peer refs 取全部结果并集预热。channelID/pts/ // recipients 由调用方按批量语义给定（pts 取最后一条；recipients 受大群截断口径影响）。 func (r *Router) enqueueChannelMessagesFanout(ctx context.Context, originUserID, channelID int64, pts int, recipients []int64, results []domain.SendChannelMessageResult, extraUserIDs []int64) { + r.enqueueBotAPIChannelMessagesUpdate(ctx, originUserID, results) fanoutCache := newViewerPeerCache(r) ownerIDs := channelMessagesFanoutOwnerIDs(results, extraUserIDs) r.enqueueChannelFanoutWithPrefetch(ctx, channelFanoutMembers, originUserID, channelID, pts, recipients, diff --git a/internal/rpc/deps.go b/internal/rpc/deps.go index 0dcf407d..912155e6 100644 --- a/internal/rpc/deps.go +++ b/internal/rpc/deps.go @@ -328,6 +328,7 @@ type HelpService interface { type UpdatesService interface { GetState(ctx context.Context, authKeyID [8]byte, userID int64) (domain.UpdateState, error) CurrentState(ctx context.Context, userID int64) (domain.UpdateState, error) + ConfirmedState(ctx context.Context, authKeyID [8]byte, userID int64) (domain.UpdateState, bool, error) AcknowledgeCurrentState(ctx context.Context, authKeyID [8]byte, userID int64) (domain.UpdateState, error) GetDifference(ctx context.Context, authKeyID [8]byte, userID int64, from domain.UpdateState) (domain.UpdateDifference, error) ClearAuthKey(ctx context.Context, authKeyID [8]byte) error @@ -715,6 +716,7 @@ type Deps struct { Users UsersService Updates UpdatesService BootstrapUpdates store.BootstrapUpdateJobStore + BotAPIUpdates store.BotAPIUpdateStore Contacts ContactsService Dialogs DialogsService Chatlists ChatlistsService diff --git a/internal/rpc/messages_edit.go b/internal/rpc/messages_edit.go index 88e28265..ad49ff59 100644 --- a/internal/rpc/messages_edit.go +++ b/internal/rpc/messages_edit.go @@ -161,6 +161,7 @@ func (r *Router) onMessagesEditMessage(ctx context.Context, req *tg.MessagesEdit if err != nil { return nil, messageEditErr(err) } + r.enqueueBotAPIPrivateEditUpdatesAsync(ctx, res) self := res.Self() if self.Event.Pts == 0 || self.Message.ID == 0 { return nil, messageIDInvalidErr() diff --git a/internal/rpc/messages_forward.go b/internal/rpc/messages_forward.go index 6fe26f0b..fccfccde 100644 --- a/internal/rpc/messages_forward.go +++ b/internal/rpc/messages_forward.go @@ -193,6 +193,9 @@ func (r *Router) onMessagesForwardMessages(ctx context.Context, req *tg.Messages if err != nil { return nil, messageForwardErr(err) } + if !sent.Duplicate { + r.enqueueBotAPIPrivateMessageUpdateAsync(ctx, sent) + } res.SenderMessages = append(res.SenderMessages, sent.SenderMessage) res.RecipientMessages = append(res.RecipientMessages, sent.RecipientMessage) res.SenderEvents = append(res.SenderEvents, sent.SenderEvent) diff --git a/internal/rpc/messages_quick_replies.go b/internal/rpc/messages_quick_replies.go index c2e5516a..34571474 100644 --- a/internal/rpc/messages_quick_replies.go +++ b/internal/rpc/messages_quick_replies.go @@ -216,6 +216,9 @@ func (r *Router) onMessagesSendQuickReplyMessages(ctx context.Context, req *tg.M if err != nil { return nil, messageSendErr(err) } + if !sent.Duplicate { + r.enqueueBotAPIPrivateMessageUpdateAsync(ctx, sent) + } res.SenderMessages = append(res.SenderMessages, sent.SenderMessage) res.RecipientMessages = append(res.RecipientMessages, sent.RecipientMessage) res.SenderEvents = append(res.SenderEvents, sent.SenderEvent) diff --git a/internal/rpc/router.go b/internal/rpc/router.go index 422228b8..24862f73 100644 --- a/internal/rpc/router.go +++ b/internal/rpc/router.go @@ -107,8 +107,12 @@ type Router struct { inlines *inlineRegistry webviews *webViewRegistry loginTokens *loginTokenRegistry + botAPIUpdates *botAPIUpdateNotifier instanceID string channelFanout *channelFanoutDispatcher + // botAPIEnqueueQueue 把 user→bot 私聊消息的 bot_api_updates 写入移出发送者 RPC 同步 + // 路径（性能审计 H2）；队列满同步回退，绝不丢（队列行是 Bot API 投递真值）。 + botAPIEnqueueQueue *botAPIEnqueueDispatcher // presenceCandidateCache 缓存 presence fan-out 的候选 peer 集合（联系人 ∪ 私聊对端， // online 过滤前），按 userID 短 TTL；零值 sync.Map 即可用，无需构造器初始化。候选集变动 @@ -170,8 +174,9 @@ func New(cfg Config, deps Deps, log *zap.Logger, clk clock.Clock) *Router { if instanceID == "" { instanceID = fmt.Sprintf("%016x", randomNonZeroInt64()) } - r := &Router{cfg: cfg, log: log, clock: clk, deps: deps, presence: newPresenceTracker(), callbacks: newCallbackRegistry(), inlines: newInlineRegistry(botInlineQueryTTL, deps.Inline), webviews: newWebViewRegistry(webViewSessionTTL, deps.Inline), loginTokens: newLoginTokenRegistry(), tempKeyResolveCache: newTempKeyResolveCache(cfg.TempKeyResolveCacheMaxEntries), storyProjectionCache: newStoryProjectionCache(clk.Now), storyPinnedCache: newStoryPinnedAvailableCache(clk.Now), storyPinnedListCache: newStoryPinnedStoriesCache(clk.Now), channelFullBotCache: newChannelFullBotInfoCache(clk.Now), userFullProjectionCache: newUserFullProjectionCache(clk.Now), peerSettingsProjectionCache: newPeerSettingsProjectionCache(clk.Now), channelFullProjectionCache: newChannelFullProjectionCache(clk.Now), emojiStickers: newEmojiStickerIndex(clk.Now), notifySettings: newNotifySettingsCache(clk.Now), stickerCatalog: newStickerCatalogCache(clk.Now), accountSettings: newAccountSettingsCache(clk.Now), instanceID: instanceID} + r := &Router{cfg: cfg, log: log, clock: clk, deps: deps, presence: newPresenceTracker(), callbacks: newCallbackRegistry(), inlines: newInlineRegistry(botInlineQueryTTL, deps.Inline), webviews: newWebViewRegistry(webViewSessionTTL, deps.Inline), loginTokens: newLoginTokenRegistry(), botAPIUpdates: newBotAPIUpdateNotifier(), tempKeyResolveCache: newTempKeyResolveCache(cfg.TempKeyResolveCacheMaxEntries), storyProjectionCache: newStoryProjectionCache(clk.Now), storyPinnedCache: newStoryPinnedAvailableCache(clk.Now), storyPinnedListCache: newStoryPinnedStoriesCache(clk.Now), channelFullBotCache: newChannelFullBotInfoCache(clk.Now), userFullProjectionCache: newUserFullProjectionCache(clk.Now), peerSettingsProjectionCache: newPeerSettingsProjectionCache(clk.Now), channelFullProjectionCache: newChannelFullProjectionCache(clk.Now), emojiStickers: newEmojiStickerIndex(clk.Now), notifySettings: newNotifySettingsCache(clk.Now), stickerCatalog: newStickerCatalogCache(clk.Now), accountSettings: newAccountSettingsCache(clk.Now), instanceID: instanceID} r.channelFanout = newChannelFanoutDispatcher(r, defaultChannelFanoutShards, defaultChannelFanoutBuffer) + r.botAPIEnqueueQueue = newBotAPIEnqueueDispatcher(log, defaultBotAPIEnqueueBuffer) r.webPageResolveSem = make(chan struct{}, webPageResolveConcurrency) r.selfPhotoEchoPushDelay = defaultSelfPhotoEchoPushDelay if cfg.DC > 0 { diff --git a/internal/rpc/rpc_testkit_updates_test.go b/internal/rpc/rpc_testkit_updates_test.go index 8c7e7eb5..8efd5c4c 100644 --- a/internal/rpc/rpc_testkit_updates_test.go +++ b/internal/rpc/rpc_testkit_updates_test.go @@ -37,6 +37,12 @@ func (s *captureUpdates) CurrentState(_ context.Context, userID int64) (domain.U return s.state, nil } +func (s *captureUpdates) ConfirmedState(_ context.Context, authKeyID [8]byte, userID int64) (domain.UpdateState, bool, error) { + s.authKeyID = authKeyID + s.userID = userID + return s.state, s.state.Pts != 0 || s.state.Date != 0, nil +} + func (s *captureUpdates) AcknowledgeCurrentState(_ context.Context, authKeyID [8]byte, userID int64) (domain.UpdateState, error) { s.authKeyID = authKeyID s.userID = userID diff --git a/internal/rpc/send_media.go b/internal/rpc/send_media.go index 8670cf80..72eeb2f8 100644 --- a/internal/rpc/send_media.go +++ b/internal/rpc/send_media.go @@ -187,6 +187,7 @@ func (r *Router) sendOutgoing(ctx context.Context, userID int64, peer domain.Pee if !res.Duplicate { // 链接预览 pending 占位：带外解析并就地替换（异步，不阻塞发送 echo）。 r.maybeEnqueueWebPageResolve(userID, peer, res.SenderMessage.ID, res.SenderMessage.Media) + r.enqueueBotAPIPrivateMessageUpdateAsync(ctx, res) } return tgPrivateMessageUpdates(res.SenderEvent, res.SenderMessage, p.randomID, true, users, chats), res.Duplicate, nil } diff --git a/internal/store/botapi_update.go b/internal/store/botapi_update.go new file mode 100644 index 00000000..c76a9a33 --- /dev/null +++ b/internal/store/botapi_update.go @@ -0,0 +1,15 @@ +package store + +import ( + "context" + + "telesrv/internal/domain" +) + +// BotAPIUpdateStore persists update_id based Bot API delivery queues. +type BotAPIUpdateStore interface { + EnqueueBotAPIUpdate(ctx context.Context, req domain.EnqueueBotAPIUpdateRequest) (domain.BotAPIUpdate, bool, error) + ListBotAPIUpdates(ctx context.Context, botUserID, fromUpdateID int64, limit int) ([]domain.BotAPIUpdate, error) + ConfirmBotAPIUpdates(ctx context.Context, botUserID, confirmedUpdateID int64) error + ConfirmedBotAPIUpdateID(ctx context.Context, botUserID int64) (int64, bool, error) +} diff --git a/internal/store/memory/botapi_update.go b/internal/store/memory/botapi_update.go new file mode 100644 index 00000000..1703fda4 --- /dev/null +++ b/internal/store/memory/botapi_update.go @@ -0,0 +1,129 @@ +package memory + +import ( + "context" + "fmt" + "sync" + + "telesrv/internal/domain" +) + +// BotAPIUpdateStore is an in-memory implementation of store.BotAPIUpdateStore. +type BotAPIUpdateStore struct { + mu sync.RWMutex + nextID int64 + rows []domain.BotAPIUpdate + state map[int64]int64 + byKey map[string]int64 +} + +// NewBotAPIUpdateStore creates an in-memory Bot API update queue. +func NewBotAPIUpdateStore() *BotAPIUpdateStore { + return &BotAPIUpdateStore{ + nextID: 1, + state: make(map[int64]int64), + byKey: make(map[string]int64), + } +} + +func (s *BotAPIUpdateStore) EnqueueBotAPIUpdate(_ context.Context, req domain.EnqueueBotAPIUpdateRequest) (domain.BotAPIUpdate, bool, error) { + if err := validateBotAPIUpdateRequest(req); err != nil { + return domain.BotAPIUpdate{}, false, err + } + key := botAPIUpdateKey(req) + s.mu.Lock() + defer s.mu.Unlock() + if existingID, ok := s.byKey[key]; ok { + for _, row := range s.rows { + if row.ID == existingID { + return cloneBotAPIUpdate(row), false, nil + } + } + } + row := domain.BotAPIUpdate{ + ID: s.nextID, + BotUserID: req.BotUserID, + Kind: req.Kind, + Peer: req.Peer, + MessageID: req.MessageID, + SourcePts: req.SourcePts, + Date: req.Date, + } + s.nextID++ + s.rows = append(s.rows, row) + s.byKey[key] = row.ID + return cloneBotAPIUpdate(row), true, nil +} + +func (s *BotAPIUpdateStore) ListBotAPIUpdates(_ context.Context, botUserID, fromUpdateID int64, limit int) ([]domain.BotAPIUpdate, error) { + if botUserID == 0 { + return nil, nil + } + if fromUpdateID <= 0 { + fromUpdateID = 1 + } + if limit <= 0 || limit > 100 { + limit = 100 + } + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]domain.BotAPIUpdate, 0, limit) + for _, row := range s.rows { + if row.BotUserID != botUserID || row.ID < fromUpdateID { + continue + } + out = append(out, cloneBotAPIUpdate(row)) + if len(out) >= limit { + break + } + } + return out, nil +} + +func (s *BotAPIUpdateStore) ConfirmBotAPIUpdates(_ context.Context, botUserID, confirmedUpdateID int64) error { + if botUserID == 0 || confirmedUpdateID <= 0 { + return nil + } + s.mu.Lock() + if confirmedUpdateID > s.state[botUserID] { + s.state[botUserID] = confirmedUpdateID + } + s.mu.Unlock() + return nil +} + +func (s *BotAPIUpdateStore) ConfirmedBotAPIUpdateID(_ context.Context, botUserID int64) (int64, bool, error) { + if botUserID == 0 { + return 0, false, nil + } + s.mu.RLock() + id, ok := s.state[botUserID] + s.mu.RUnlock() + return id, ok, nil +} + +func validateBotAPIUpdateRequest(req domain.EnqueueBotAPIUpdateRequest) error { + if req.BotUserID == 0 || req.MessageID <= 0 { + return fmt.Errorf("invalid bot api update") + } + if req.Kind != domain.BotAPIUpdateMessage && req.Kind != domain.BotAPIUpdateEditedMessage { + return fmt.Errorf("invalid bot api update kind %q", req.Kind) + } + switch req.Peer.Type { + case domain.PeerTypeUser, domain.PeerTypeChannel: + if req.Peer.ID <= 0 { + return fmt.Errorf("invalid bot api update peer") + } + default: + return fmt.Errorf("invalid bot api update peer type %q", req.Peer.Type) + } + return nil +} + +func botAPIUpdateKey(req domain.EnqueueBotAPIUpdateRequest) string { + return fmt.Sprintf("%d:%s:%s:%d:%d:%d", req.BotUserID, req.Kind, req.Peer.Type, req.Peer.ID, req.MessageID, req.SourcePts) +} + +func cloneBotAPIUpdate(row domain.BotAPIUpdate) domain.BotAPIUpdate { + return row +} diff --git a/internal/store/postgres/botapi_update.go b/internal/store/postgres/botapi_update.go new file mode 100644 index 00000000..c69b96de --- /dev/null +++ b/internal/store/postgres/botapi_update.go @@ -0,0 +1,214 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "telesrv/internal/domain" + "telesrv/internal/store/postgres/sqlcgen" +) + +// BotAPIUpdateStore persists Bot API getUpdates queues in PostgreSQL. +type BotAPIUpdateStore struct { + db sqlcgen.DBTX +} + +func NewBotAPIUpdateStore(db sqlcgen.DBTX) *BotAPIUpdateStore { + return &BotAPIUpdateStore{db: db} +} + +func (s *BotAPIUpdateStore) EnqueueBotAPIUpdate(ctx context.Context, req domain.EnqueueBotAPIUpdateRequest) (domain.BotAPIUpdate, bool, error) { + if err := validateBotAPIUpdateRequest(req); err != nil { + return domain.BotAPIUpdate{}, false, err + } + row, err := s.scanBotAPIUpdate(s.db.QueryRow(ctx, ` +INSERT INTO bot_api_updates ( + bot_user_id, update_kind, peer_type, peer_id, message_id, source_pts, date +) VALUES ($1, $2, $3, $4, $5, $6, $7) +ON CONFLICT (bot_user_id, update_kind, peer_type, peer_id, message_id, source_pts) DO NOTHING +RETURNING id, bot_user_id, update_kind, peer_type, peer_id, message_id, source_pts, date +`, req.BotUserID, string(req.Kind), string(req.Peer.Type), req.Peer.ID, req.MessageID, req.SourcePts, req.Date)) + if err == nil { + return row, true, nil + } + if err != pgx.ErrNoRows { + return domain.BotAPIUpdate{}, false, fmt.Errorf("insert bot api update: %w", err) + } + row, err = s.scanBotAPIUpdate(s.db.QueryRow(ctx, ` +SELECT id, bot_user_id, update_kind, peer_type, peer_id, message_id, source_pts, date +FROM bot_api_updates +WHERE bot_user_id = $1 + AND update_kind = $2 + AND peer_type = $3 + AND peer_id = $4 + AND message_id = $5 + AND source_pts = $6 +`, req.BotUserID, string(req.Kind), string(req.Peer.Type), req.Peer.ID, req.MessageID, req.SourcePts)) + if err != nil { + return domain.BotAPIUpdate{}, false, fmt.Errorf("select existing bot api update: %w", err) + } + return row, false, nil +} + +func (s *BotAPIUpdateStore) ListBotAPIUpdates(ctx context.Context, botUserID, fromUpdateID int64, limit int) ([]domain.BotAPIUpdate, error) { + if botUserID == 0 { + return nil, nil + } + if fromUpdateID <= 0 { + fromUpdateID = 1 + } + if limit <= 0 || limit > 100 { + limit = 100 + } + rows, err := s.db.Query(ctx, ` +SELECT id, bot_user_id, update_kind, peer_type, peer_id, message_id, source_pts, date +FROM bot_api_updates +WHERE bot_user_id = $1 AND id >= $2 +ORDER BY id +LIMIT $3 +`, botUserID, fromUpdateID, limit) + if err != nil { + return nil, fmt.Errorf("list bot api updates: %w", err) + } + defer rows.Close() + out := make([]domain.BotAPIUpdate, 0, limit) + for rows.Next() { + item, err := scanBotAPIUpdateRows(rows) + if err != nil { + return nil, err + } + out = append(out, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list bot api updates rows: %w", err) + } + return out, nil +} + +func (s *BotAPIUpdateStore) ConfirmBotAPIUpdates(ctx context.Context, botUserID, confirmedUpdateID int64) error { + if botUserID == 0 || confirmedUpdateID <= 0 { + return nil + } + if _, err := s.db.Exec(ctx, ` +INSERT INTO bot_api_update_states (bot_user_id, confirmed_update_id) +VALUES ($1, $2) +ON CONFLICT (bot_user_id) DO UPDATE +SET confirmed_update_id = GREATEST(bot_api_update_states.confirmed_update_id, EXCLUDED.confirmed_update_id), + updated_at = now() +WHERE bot_api_update_states.confirmed_update_id < EXCLUDED.confirmed_update_id +`, botUserID, confirmedUpdateID); err != nil { + return fmt.Errorf("confirm bot api updates: %w", err) + } + return nil +} + +// DeleteDeliveredOrExpired 回收 Bot API 投递队列的死行（性能审计 H1）： +// 1. 已确认（id <= bot_api_update_states.confirmed_update_id）且入队超过 confirmedGrace 的行—— +// 官方 Bot API 语义下确认即弃，getUpdates 的 fromID 恒 > confirmed，删除不影响任何读路径； +// 宽限仅防御 offset 回拨调试场景。 +// 2. 按消息 date 超过 maxAge 的行（无论确认与否）——对齐官方「updates 服务器最多保留 24 小时」 +// 语义，同时封顶 MTProto-only bot（从不调 getUpdates、无 state 行）成员身份带来的无界增长。 +// +// 与 user_update_events 的「永久保留」约束无关：那是 TDesktop 账号级 differenceTooLong 缺陷所迫， +// Bot API 队列没有该约束。返回两步合计删除行数。 +func (s *BotAPIUpdateStore) DeleteDeliveredOrExpired(ctx context.Context, confirmedGrace, maxAge time.Duration, limit int) (int, error) { + if limit <= 0 { + limit = 10000 + } + if limit > 100000 { + limit = 100000 + } + total := 0 + if confirmedGrace > 0 { + // 从 states 小表出发，每 bot 走 bot_api_updates_bot_scan_idx(bot_user_id, id) 范围扫描。 + tag, err := s.db.Exec(ctx, ` +DELETE FROM bot_api_updates +WHERE id IN ( + SELECT u.id + FROM bot_api_update_states s + JOIN bot_api_updates u ON u.bot_user_id = s.bot_user_id AND u.id <= s.confirmed_update_id + WHERE u.created_at < now() - make_interval(secs => $1) + LIMIT $2 +)`, int64(confirmedGrace/time.Second), limit) + if err != nil { + return total, fmt.Errorf("delete confirmed bot api updates: %w", err) + } + total += int(tag.RowsAffected()) + } + if maxAge > 0 { + cutoff := time.Now().Add(-maxAge).Unix() + // 走 bot_api_updates_retention_idx(date, id)。 + tag, err := s.db.Exec(ctx, ` +DELETE FROM bot_api_updates +WHERE id IN ( + SELECT id + FROM bot_api_updates + WHERE date < $1 + ORDER BY date, id + LIMIT $2 +)`, cutoff, limit) + if err != nil { + return total, fmt.Errorf("delete expired bot api updates: %w", err) + } + total += int(tag.RowsAffected()) + } + return total, nil +} + +func (s *BotAPIUpdateStore) ConfirmedBotAPIUpdateID(ctx context.Context, botUserID int64) (int64, bool, error) { + if botUserID == 0 { + return 0, false, nil + } + var id int64 + if err := s.db.QueryRow(ctx, ` +SELECT confirmed_update_id +FROM bot_api_update_states +WHERE bot_user_id = $1 +`, botUserID).Scan(&id); err != nil { + if err == pgx.ErrNoRows { + return 0, false, nil + } + return 0, false, fmt.Errorf("get bot api update state: %w", err) + } + return id, true, nil +} + +func (s *BotAPIUpdateStore) scanBotAPIUpdate(row pgx.Row) (domain.BotAPIUpdate, error) { + return scanBotAPIUpdateRows(row) +} + +type botAPIUpdateScanner interface { + Scan(dest ...any) error +} + +func scanBotAPIUpdateRows(row botAPIUpdateScanner) (domain.BotAPIUpdate, error) { + var item domain.BotAPIUpdate + var kind, peerType string + if err := row.Scan(&item.ID, &item.BotUserID, &kind, &peerType, &item.Peer.ID, &item.MessageID, &item.SourcePts, &item.Date); err != nil { + return domain.BotAPIUpdate{}, err + } + item.Kind = domain.BotAPIUpdateKind(kind) + item.Peer.Type = domain.PeerType(peerType) + return item, nil +} + +func validateBotAPIUpdateRequest(req domain.EnqueueBotAPIUpdateRequest) error { + if req.BotUserID == 0 || req.MessageID <= 0 { + return fmt.Errorf("invalid bot api update") + } + if req.Kind != domain.BotAPIUpdateMessage && req.Kind != domain.BotAPIUpdateEditedMessage { + return fmt.Errorf("invalid bot api update kind %q", req.Kind) + } + switch req.Peer.Type { + case domain.PeerTypeUser, domain.PeerTypeChannel: + if req.Peer.ID <= 0 { + return fmt.Errorf("invalid bot api update peer") + } + default: + return fmt.Errorf("invalid bot api update peer type %q", req.Peer.Type) + } + return nil +} diff --git a/internal/store/postgres/botapi_update_integration_test.go b/internal/store/postgres/botapi_update_integration_test.go new file mode 100644 index 00000000..67b6dd5d --- /dev/null +++ b/internal/store/postgres/botapi_update_integration_test.go @@ -0,0 +1,122 @@ +package postgres + +import ( + "context" + "testing" + "time" + + "telesrv/internal/domain" +) + +// TestBotAPIUpdateRetention 锁定 H1 场景矩阵： +// - 已确认 + 超宽限 → 删；已确认 + 宽限内 → 留； +// - 未确认 + date 超保留期 → 删（含无 state 行的 MTProto-only bot）； +// - 未确认 + date 在保留期内 → 留； +// - 删除后 getUpdates 读路径（fromID > confirmed）不受影响。 +func TestBotAPIUpdateRetention(t *testing.T) { + pool := testPool(t) + ctx := context.Background() + + suffix := randomSuffix(t) + users := NewUserStore(pool) + newBot := func(phoneTail, name string) int64 { + t.Helper() + u, err := users.Create(ctx, domain.User{ + AccessHash: 920, + Phone: "+1920" + suffix + phoneTail, + FirstName: name, + }) + if err != nil { + t.Fatalf("create bot user %s: %v", name, err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO bots (bot_user_id, owner_user_id, token_secret) +VALUES ($1, $1, 'retention-test-secret') +ON CONFLICT (bot_user_id) DO NOTHING`, u.ID); err != nil { + t.Fatalf("seed bot %s: %v", name, err) + } + return u.ID + } + confirmedBot := newBot("01", "RetentionConfirmedBot") + mtprotoOnlyBot := newBot("02", "RetentionMTOnlyBot") + t.Cleanup(func() { + _, _ = pool.Exec(ctx, "DELETE FROM bot_api_updates WHERE bot_user_id IN ($1, $2)", confirmedBot, mtprotoOnlyBot) + _, _ = pool.Exec(ctx, "DELETE FROM bot_api_update_states WHERE bot_user_id IN ($1, $2)", confirmedBot, mtprotoOnlyBot) + _, _ = pool.Exec(ctx, "DELETE FROM bots WHERE bot_user_id IN ($1, $2)", confirmedBot, mtprotoOnlyBot) + }) + + s := NewBotAPIUpdateStore(pool) + now := time.Now().Unix() + stale := now - int64((48 * time.Hour).Seconds()) + enqueue := func(botID int64, messageID int, date int64) domain.BotAPIUpdate { + t.Helper() + row, created, err := s.EnqueueBotAPIUpdate(ctx, domain.EnqueueBotAPIUpdateRequest{ + BotUserID: botID, + Kind: domain.BotAPIUpdateMessage, + Peer: domain.Peer{Type: domain.PeerTypeUser, ID: 1}, + MessageID: messageID, + SourcePts: messageID, + Date: int(date), + }) + if err != nil || !created { + t.Fatalf("enqueue bot=%d msg=%d: created=%v err=%v", botID, messageID, created, err) + } + return row + } + + confirmedOld := enqueue(confirmedBot, 1, now) // 已确认 + created_at 回拨超宽限 → 删 + confirmedFresh := enqueue(confirmedBot, 2, now) // 已确认 + 宽限内 → 留 + unconfirmedFresh := enqueue(confirmedBot, 3, now) + expiredNoState := enqueue(mtprotoOnlyBot, 4, stale) // 无 state 行 + date 超保留期 → 删 + freshNoState := enqueue(mtprotoOnlyBot, 5, now) + + if err := s.ConfirmBotAPIUpdates(ctx, confirmedBot, confirmedFresh.ID); err != nil { + t.Fatalf("confirm: %v", err) + } + if _, err := pool.Exec(ctx, + "UPDATE bot_api_updates SET created_at = now() - interval '1 hour' WHERE id = $1", confirmedOld.ID); err != nil { + t.Fatalf("backdate confirmed row: %v", err) + } + + deleted, err := s.DeleteDeliveredOrExpired(ctx, 15*time.Minute, 24*time.Hour, 1000) + if err != nil { + t.Fatalf("DeleteDeliveredOrExpired: %v", err) + } + // 共享测试库可能有其它历史行同被回收，只要求至少删掉本测试的 2 行； + // 精确归属由下方 remaining 断言保证。 + if deleted < 2 { + t.Fatalf("deleted = %d, want >= 2 (confirmed+grace expired, date expired)", deleted) + } + + remaining := map[int64]bool{} + rows, err := pool.Query(ctx, "SELECT id FROM bot_api_updates WHERE bot_user_id IN ($1, $2)", confirmedBot, mtprotoOnlyBot) + if err != nil { + t.Fatalf("list remaining: %v", err) + } + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + t.Fatalf("scan remaining: %v", err) + } + remaining[id] = true + } + rows.Close() + if remaining[confirmedOld.ID] { + t.Fatal("confirmed row past grace was not deleted") + } + if remaining[expiredNoState.ID] { + t.Fatal("expired row of state-less bot was not deleted") + } + if !remaining[confirmedFresh.ID] || !remaining[unconfirmedFresh.ID] || !remaining[freshNoState.ID] { + t.Fatalf("fresh rows were deleted, remaining=%v", remaining) + } + + // 读路径回归：确认水位之后的未确认行仍可被 getUpdates 读到。 + items, err := s.ListBotAPIUpdates(ctx, confirmedBot, confirmedFresh.ID+1, 100) + if err != nil { + t.Fatalf("list after retention: %v", err) + } + if len(items) != 1 || items[0].ID != unconfirmedFresh.ID { + t.Fatalf("post-retention list = %+v, want only unconfirmed fresh row %d", items, unconfirmedFresh.ID) + } +} diff --git a/internal/store/postgres/channel_member_cache_test.go b/internal/store/postgres/channel_member_cache_test.go index 5299d8c1..80f1de8a 100644 --- a/internal/store/postgres/channel_member_cache_test.go +++ b/internal/store/postgres/channel_member_cache_test.go @@ -136,6 +136,7 @@ func TestReadModelChangeListenerInvalidatesChannelCaches(t *testing.T) { rows := NewChannelRowCache(16) members := NewChannelMemberCache(16) fullBots := &fakeChannelFullBotReadModelCache{} + botMembers := &fakeChannelBotMemberReadModelCache{} mediaCounts := &fakeChannelMediaCountReadModelCache{} rows.put(domain.Channel{ID: 7, Title: "old"}) members.put(domain.ChannelMember{ChannelID: 7, UserID: 100, Status: domain.ChannelMemberActive}) @@ -146,6 +147,7 @@ func TestReadModelChangeListenerInvalidatesChannelCaches(t *testing.T) { ChannelRows: rows, ChannelMembers: members, ChannelFullBots: fullBots, + ChannelBotMembers: botMembers, ChannelMediaCounts: mediaCounts, }, nil) listener.handlePayload(`{"model":"channel_member","owner_user_id":100,"peer_type":"channel","peer_id":7,"version":2}`) @@ -158,6 +160,9 @@ func TestReadModelChangeListenerInvalidatesChannelCaches(t *testing.T) { if got := fullBots.channelsSnapshot(); len(got) != 1 || got[0] != 7 { t.Fatalf("channel_member 应失效 full bot info: %+v", got) } + if got := botMembers.channelsSnapshot(); len(got) != 1 || got[0] != 7 { + t.Fatalf("channel_member 应失效 bot member ids: %+v", got) + } if got := mediaCounts.viewerSnapshot(); len(got) != 1 || got[0] != [2]int64{100, 7} { t.Fatalf("channel_member 应失效该 viewer 的 media count: %+v", got) } @@ -175,8 +180,16 @@ func TestReadModelChangeListenerInvalidatesChannelCaches(t *testing.T) { if got := fullBots.channelsSnapshot(); len(got) != 2 || got[1] != 7 { t.Fatalf("channel_base 应失效 full bot info: %+v", got) } + if got := botMembers.channelsSnapshot(); len(got) != 2 || got[1] != 7 { + t.Fatalf("channel_base 应失效 bot member ids: %+v", got) + } - listener.handlePayload(`{"model":"channel_media_counts","owner_user_id":0,"peer_type":"channel","peer_id":7,"version":4}`) + listener.handlePayload(`{"model":"channel_participants","owner_user_id":0,"peer_type":"channel","peer_id":7,"version":4}`) + if got := botMembers.channelsSnapshot(); len(got) != 3 || got[2] != 7 { + t.Fatalf("channel_participants 应失效 bot member ids: %+v", got) + } + + listener.handlePayload(`{"model":"channel_media_counts","owner_user_id":0,"peer_type":"channel","peer_id":7,"version":5}`) if got := mediaCounts.channelSnapshot(); len(got) != 1 || got[0] != 7 { t.Fatalf("channel_media_counts 应失效该频道 media count: %+v", got) } @@ -244,6 +257,30 @@ func (f *fakeChannelFullBotReadModelCache) flushCount() int { return f.flushes } +type fakeChannelBotMemberReadModelCache struct { + mu sync.Mutex + channels []int64 + flushes int +} + +func (f *fakeChannelBotMemberReadModelCache) InvalidateActiveBotMemberIDsReadModel(channelID int64) { + f.mu.Lock() + defer f.mu.Unlock() + f.channels = append(f.channels, channelID) +} + +func (f *fakeChannelBotMemberReadModelCache) FlushActiveBotMemberIDsReadModel() { + f.mu.Lock() + defer f.mu.Unlock() + f.flushes++ +} + +func (f *fakeChannelBotMemberReadModelCache) channelsSnapshot() []int64 { + f.mu.Lock() + defer f.mu.Unlock() + return append([]int64(nil), f.channels...) +} + type fakeChannelMediaCountReadModelCache struct { mu sync.Mutex channels []int64 diff --git a/internal/store/postgres/read_model_listener.go b/internal/store/postgres/read_model_listener.go index 52e5988c..268deed9 100644 --- a/internal/store/postgres/read_model_listener.go +++ b/internal/store/postgres/read_model_listener.go @@ -27,6 +27,7 @@ type ReadModelCacheSet struct { ProfilePhotos ProfilePhotoReadModelCache Stories StoryReadModelCache ChannelFullBots ChannelFullBotReadModelCache + ChannelBotMembers ChannelBotMemberReadModelCache ChannelMediaCounts ChannelMediaCountReadModelCache PrivateMediaCounts PrivateMediaCountReadModelCache RPCProjections RPCProjectionReadModelCache @@ -99,6 +100,11 @@ type ChannelFullBotReadModelCache interface { FlushChannelFullBotInfoReadModel() } +type ChannelBotMemberReadModelCache interface { + InvalidateActiveBotMemberIDsReadModel(channelID int64) + FlushActiveBotMemberIDsReadModel() +} + type ChannelMediaCountReadModelCache interface { InvalidateChannelMediaCountReadModel(channelID int64) InvalidateChannelMediaCountReadModelForViewer(userID, channelID int64) @@ -203,6 +209,7 @@ func (l *ReadModelChangeListener) empty() bool { l.caches.ProfilePhotos == nil && l.caches.Stories == nil && l.caches.ChannelFullBots == nil && + l.caches.ChannelBotMembers == nil && l.caches.ChannelMediaCounts == nil && l.caches.PrivateMediaCounts == nil && l.caches.RPCProjections == nil && @@ -260,6 +267,10 @@ func (l *ReadModelChangeListener) flush(reasons ...string) { l.caches.ChannelFullBots.FlushChannelFullBotInfoReadModel() flushed = append(flushed, "channel_full_bots") } + if l.caches.ChannelBotMembers != nil { + l.caches.ChannelBotMembers.FlushActiveBotMemberIDsReadModel() + flushed = append(flushed, "channel_bot_members") + } if l.caches.ChannelMediaCounts != nil { l.caches.ChannelMediaCounts.FlushChannelMediaCountReadModel() flushed = append(flushed, "channel_media_counts") @@ -396,6 +407,9 @@ func (l *ReadModelChangeListener) handlePayload(payload string) { if l.caches.ChannelFullBots != nil { l.caches.ChannelFullBots.InvalidateChannelFullBotInfoReadModel(evt.PeerID) } + if l.caches.ChannelBotMembers != nil { + l.caches.ChannelBotMembers.InvalidateActiveBotMemberIDsReadModel(evt.PeerID) + } if l.caches.RPCProjections != nil { l.caches.RPCProjections.InvalidateRPCProjectionReadModelForChannel(evt.PeerID) } @@ -421,6 +435,9 @@ func (l *ReadModelChangeListener) handlePayload(payload string) { if l.caches.ChannelFullBots != nil { l.caches.ChannelFullBots.InvalidateChannelFullBotInfoReadModel(evt.PeerID) } + if l.caches.ChannelBotMembers != nil { + l.caches.ChannelBotMembers.InvalidateActiveBotMemberIDsReadModel(evt.PeerID) + } if l.caches.RPCProjections != nil { l.caches.RPCProjections.InvalidateRPCProjectionReadModelForChannel(evt.PeerID) l.caches.RPCProjections.InvalidateRPCProjectionReadModelForPeer(evt.OwnerUserID, domain.Peer{Type: domain.PeerTypeChannel, ID: evt.PeerID}) @@ -439,6 +456,9 @@ func (l *ReadModelChangeListener) handlePayload(payload string) { l.caches.RPCProjections.InvalidateRPCProjectionReadModelForPeer(evt.OwnerUserID, domain.Peer{Type: domain.PeerTypeChannel, ID: evt.PeerID}) } case "channel_participants": + if evt.PeerType == "channel" && evt.PeerID != 0 && l.caches.ChannelBotMembers != nil { + l.caches.ChannelBotMembers.InvalidateActiveBotMemberIDsReadModel(evt.PeerID) + } if evt.PeerType == "channel" && evt.PeerID != 0 && l.caches.RPCProjections != nil { l.caches.RPCProjections.InvalidateRPCProjectionReadModelForChannel(evt.PeerID) } diff --git a/internal/web/stickerlinks/server.go b/internal/web/stickerlinks/server.go index 1071712a..df9d8e4e 100644 --- a/internal/web/stickerlinks/server.go +++ b/internal/web/stickerlinks/server.go @@ -21,12 +21,17 @@ import ( type Config struct { Addr string PublicBaseURL string + Users UsernameResolver } type Resolver interface { ResolveStickerSet(ctx context.Context, ref domain.StickerSetRef) (domain.StickerSet, []domain.Document, bool, error) } +type UsernameResolver interface { + ByUsername(ctx context.Context, username string) (domain.User, bool, error) +} + func Start(ctx context.Context, cfg Config, resolver Resolver, logger *zap.Logger) (*http.Server, error) { addr := strings.TrimSpace(cfg.Addr) if addr == "" { @@ -38,7 +43,7 @@ func Start(ctx context.Context, cfg Config, resolver Resolver, logger *zap.Logge if logger == nil { logger = zap.NewNop() } - handler := NewHandler(resolver, cfg.PublicBaseURL) + handler := NewHandlerWithUsers(resolver, cfg.Users, cfg.PublicBaseURL) srv := &http.Server{ Addr: addr, Handler: handler, @@ -64,8 +69,13 @@ func Start(ctx context.Context, cfg Config, resolver Resolver, logger *zap.Logge } func NewHandler(resolver Resolver, publicBaseURL string) http.Handler { + return NewHandlerWithUsers(resolver, nil, publicBaseURL) +} + +func NewHandlerWithUsers(resolver Resolver, users UsernameResolver, publicBaseURL string) http.Handler { h := &handler{ resolver: resolver, + users: users, publicBaseURL: normalizePublicBaseURL(publicBaseURL), } mux := http.NewServeMux() @@ -73,11 +83,13 @@ func NewHandler(resolver Resolver, publicBaseURL string) http.Handler { mux.HandleFunc("GET /addstickers/{shortName}", h.addStickers) mux.HandleFunc("GET /addemoji/{shortName}", h.addEmoji) mux.HandleFunc("GET /addlist/{slug}", h.addList) + mux.HandleFunc("GET /{username}", h.usernameLink) return mux } type handler struct { resolver Resolver + users UsernameResolver publicBaseURL string } @@ -118,6 +130,43 @@ func (h *handler) addList(w http.ResponseWriter, r *http.Request) { } } +func (h *handler) usernameLink(w http.ResponseWriter, r *http.Request) { + username := strings.TrimSpace(r.PathValue("username")) + if h.users == nil || !validUsernamePath(username) { + http.NotFound(w, r) + return + } + u, found, err := h.users.ByUsername(r.Context(), username) + if err != nil { + http.Error(w, "username lookup failed", http.StatusInternalServerError) + return + } + if !found || !u.Bot || strings.TrimSpace(u.Username) == "" { + http.NotFound(w, r) + return + } + title := strings.TrimSpace(u.FirstName) + if title == "" { + title = u.Username + } + app := schemeURL("telesrv", "resolve", "domain", u.Username) + data := pageData{ + Title: title, + KindLabel: "bot", + Subtitle: "@" + u.Username, + Description: "This page opens the app so you can start a chat with this bot.", + CanonicalURL: h.publicUsernameURL(u.Username), + AppURL: template.URL(app), + LegacyTgURL: template.URL(schemeURL("tg", "resolve", "domain", u.Username)), + } + data.AppURLJS = template.JS(strconv.Quote(app)) + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Cache-Control", "public, max-age=60") + if err := landingTemplate.Execute(w, data); err != nil { + http.Error(w, "render bot page failed", http.StatusInternalServerError) + } +} + func (h *handler) serveSet(w http.ResponseWriter, r *http.Request, pathKind string) { shortName := strings.TrimSpace(r.PathValue("shortName")) if !validShortNamePath(shortName) { @@ -167,6 +216,10 @@ func (h *handler) publicURL(kind, value string) string { return h.publicBaseURL + "/" + kind + "/" + url.PathEscape(value) } +func (h *handler) publicUsernameURL(username string) string { + return h.publicBaseURL + "/" + url.PathEscape(username) +} + func normalizePublicBaseURL(raw string) string { u, err := url.Parse(links.NormalizeBaseURL(raw)) if err != nil || u.Scheme == "" || u.Host == "" { @@ -199,6 +252,24 @@ func validSlugPath(slug string) bool { return links.ValidChatlistSlug(slug) } +func validUsernamePath(username string) bool { + if username == "" || len(username) < 5 || len(username) > 32 { + return false + } + for i, r := range username { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z': + case r >= '0' && r <= '9', r == '_': + if i == 0 { + return false + } + default: + return false + } + } + return true +} + func linkKind(set domain.StickerSet) string { if set.Kind == domain.StickerSetKindEmoji || set.Emojis { return "addemoji" diff --git a/internal/web/stickerlinks/server_test.go b/internal/web/stickerlinks/server_test.go index bffc8553..e042a0f2 100644 --- a/internal/web/stickerlinks/server_test.go +++ b/internal/web/stickerlinks/server_test.go @@ -107,6 +107,42 @@ func TestHandlerServesChatlistLandingPage(t *testing.T) { } } +func TestHandlerServesBotUsernameLandingPage(t *testing.T) { + users := fakeUsers{ + "tetrisbot": { + ID: 1001, + Username: "TetrisBot", + FirstName: "Tetris Bot", + Bot: true, + }, + } + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/TetrisBot", nil) + + NewHandlerWithUsers(fakeResolver{}, users, "http://127.0.0.1:2401").ServeHTTP(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rr.Code, rr.Body.String()) + } + body := rr.Body.String() + for _, want := range []string{ + "Tetris Bot", + "bot", + "@TetrisBot", + "http://127.0.0.1:2401/TetrisBot", + "telesrv://resolve?domain=TetrisBot", + "tg://resolve?domain=TetrisBot", + "start a chat with this bot", + } { + if !strings.Contains(body, want) { + t.Fatalf("body missing %q:\n%s", want, body) + } + } + if strings.Contains(body, `window.location.href = "tg://`) { + t.Fatalf("landing page must not auto-open tg:// and steal official Telegram:\n%s", body) + } +} + func TestHandlerRedirectsMismatchedKindToCanonicalURL(t *testing.T) { resolver := fakeResolver{ "emoji_pack": { @@ -131,13 +167,23 @@ func TestHandlerRedirectsMismatchedKindToCanonicalURL(t *testing.T) { } func TestHandlerNotFoundForMissingOrInvalidShortName(t *testing.T) { - handler := NewHandler(fakeResolver{}, "https://telesrv.net") + handler := NewHandlerWithUsers(fakeResolver{}, fakeUsers{ + "alice": { + ID: 2001, + Username: "Alice", + FirstName: "Alice", + }, + }, "https://telesrv.net") for _, path := range []string{ "/addstickers/missing_pack", "/addstickers/bad-name", "/addemoji/%E4%B8%AD%E6%96%87", "/addlist/bad!slug", "/addlist/%E4%B8%AD%E6%96%87", + "/MissingBot", + "/Alice", + "/bad-name-bot", + "/1stBot", } { rr := httptest.NewRecorder() req := httptest.NewRequest(http.MethodGet, path, nil) @@ -175,3 +221,10 @@ type errorResolver struct{} func (errorResolver) ResolveStickerSet(context.Context, domain.StickerSetRef) (domain.StickerSet, []domain.Document, bool, error) { return domain.StickerSet{}, nil, false, errors.New("boom") } + +type fakeUsers map[string]domain.User + +func (f fakeUsers) ByUsername(_ context.Context, username string) (domain.User, bool, error) { + u, ok := f[strings.ToLower(strings.TrimPrefix(username, "@"))] + return u, ok, nil +} diff --git a/scripts/botapi_echo_once.py b/scripts/botapi_echo_once.py new file mode 100644 index 00000000..d1e9b9a3 --- /dev/null +++ b/scripts/botapi_echo_once.py @@ -0,0 +1,63 @@ +import argparse +import asyncio +import sys +import time + +from telegram import Bot + + +async def main() -> int: + parser = argparse.ArgumentParser(description="Listen once via telesrv Bot API and echo the first text message.") + parser.add_argument("--token", required=True) + parser.add_argument("--base-url", default="http://127.0.0.1:8081/bot") + parser.add_argument("--base-file-url", default="http://127.0.0.1:8081/file/bot") + parser.add_argument("--timeout-seconds", type=int, default=180) + parser.add_argument("--prefix", default="echo: ") + args = parser.parse_args() + + bot = Bot(token=args.token, base_url=args.base_url, base_file_url=args.base_file_url) + me = await bot.get_me() + print(f"listening as @{me.username or me.id} ({me.id})", flush=True) + await bot.delete_webhook(drop_pending_updates=False) + + offset = None + stale = await bot.get_updates(timeout=0, allowed_updates=["message", "edited_message"]) + if stale: + offset = max(update.update_id for update in stale) + 1 + await bot.get_updates(offset=offset, timeout=0, allowed_updates=["message", "edited_message"]) + print(f"drained {len(stale)} stale update(s), next offset={offset}", flush=True) + + deadline = time.monotonic() + max(1, args.timeout_seconds) + while time.monotonic() < deadline: + remaining = max(1, min(30, int(deadline - time.monotonic()))) + updates = await bot.get_updates( + offset=offset, + timeout=remaining, + allowed_updates=["message", "edited_message"], + ) + if not updates: + continue + offset = max(update.update_id for update in updates) + 1 + for update in updates: + msg = update.message or update.edited_message + if msg is None or msg.chat_id is None: + continue + text = msg.text or msg.caption or "" + if not text: + continue + reply = args.prefix + text + sent = await bot.send_message(chat_id=msg.chat_id, text=reply) + print( + f"echoed update_id={update.update_id} chat_id={msg.chat_id} " + f"message_id={msg.message_id} sent_message_id={sent.message_id} text={text!r}", + flush=True, + ) + await bot.get_updates(offset=offset, timeout=0, allowed_updates=["message", "edited_message"]) + return 0 + + print("timed out waiting for a text message", file=sys.stderr, flush=True) + return 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/botapi_receive_selftest.py b/scripts/botapi_receive_selftest.py new file mode 100644 index 00000000..bfdcd187 --- /dev/null +++ b/scripts/botapi_receive_selftest.py @@ -0,0 +1,61 @@ +import argparse +import asyncio +import sys + +from telegram import Bot + + +async def main() -> int: + parser = argparse.ArgumentParser(description="Poll telesrv Bot API with python-telegram-bot.") + parser.add_argument("--token", required=True) + parser.add_argument("--base-url", default="http://127.0.0.1:8081/bot") + parser.add_argument("--base-file-url", default="http://127.0.0.1:8081/file/bot") + parser.add_argument("--offset", type=int, default=0) + parser.add_argument("--expect-chat-id", type=int, default=0) + parser.add_argument("--expect-text", default="") + parser.add_argument("--confirm", action="store_true") + args = parser.parse_args() + + bot = Bot(token=args.token, base_url=args.base_url, base_file_url=args.base_file_url) + me = await bot.get_me() + print(f"getMe: id={me.id} username={me.username!r} is_bot={me.is_bot}") + await bot.delete_webhook(drop_pending_updates=False) + + updates = await bot.get_updates( + offset=args.offset or None, + timeout=0, + allowed_updates=["message", "edited_message"], + ) + print(f"getUpdates: count={len(updates)}") + for update in updates: + msg = update.message or update.edited_message + chat_id = msg.chat_id if msg else None + text = msg.text if msg else None + print(f"update_id={update.update_id} chat_id={chat_id} text={text!r}") + + if args.expect_text: + matched = False + for update in updates: + msg = update.message or update.edited_message + if msg is None: + continue + if args.expect_chat_id and msg.chat_id != args.expect_chat_id: + continue + if msg.text == args.expect_text: + matched = True + break + if not matched: + print("expected update not found", file=sys.stderr) + return 1 + + if args.confirm and updates: + next_offset = max(update.update_id for update in updates) + 1 + after = await bot.get_updates(offset=next_offset, timeout=0, allowed_updates=["message", "edited_message"]) + print(f"confirm offset={next_offset}: count={len(after)}") + if after: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main()))