feat: sync Bot API gateway support

This commit is contained in:
A 2026-07-09 13:49:24 +08:00
parent 9a501f900a
commit 4c0cc2b7a7
44 changed files with 4609 additions and 49 deletions

View file

@ -0,0 +1,214 @@
package postgres
import (
"context"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"telesrv/internal/domain"
"telesrv/internal/store/postgres/sqlcgen"
)
// BotAPIUpdateStore persists Bot API getUpdates queues in PostgreSQL.
type BotAPIUpdateStore struct {
db sqlcgen.DBTX
}
func NewBotAPIUpdateStore(db sqlcgen.DBTX) *BotAPIUpdateStore {
return &BotAPIUpdateStore{db: db}
}
func (s *BotAPIUpdateStore) EnqueueBotAPIUpdate(ctx context.Context, req domain.EnqueueBotAPIUpdateRequest) (domain.BotAPIUpdate, bool, error) {
if err := validateBotAPIUpdateRequest(req); err != nil {
return domain.BotAPIUpdate{}, false, err
}
row, err := s.scanBotAPIUpdate(s.db.QueryRow(ctx, `
INSERT INTO bot_api_updates (
bot_user_id, update_kind, peer_type, peer_id, message_id, source_pts, date
) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (bot_user_id, update_kind, peer_type, peer_id, message_id, source_pts) DO NOTHING
RETURNING id, bot_user_id, update_kind, peer_type, peer_id, message_id, source_pts, date
`, req.BotUserID, string(req.Kind), string(req.Peer.Type), req.Peer.ID, req.MessageID, req.SourcePts, req.Date))
if err == nil {
return row, true, nil
}
if err != pgx.ErrNoRows {
return domain.BotAPIUpdate{}, false, fmt.Errorf("insert bot api update: %w", err)
}
row, err = s.scanBotAPIUpdate(s.db.QueryRow(ctx, `
SELECT id, bot_user_id, update_kind, peer_type, peer_id, message_id, source_pts, date
FROM bot_api_updates
WHERE bot_user_id = $1
AND update_kind = $2
AND peer_type = $3
AND peer_id = $4
AND message_id = $5
AND source_pts = $6
`, req.BotUserID, string(req.Kind), string(req.Peer.Type), req.Peer.ID, req.MessageID, req.SourcePts))
if err != nil {
return domain.BotAPIUpdate{}, false, fmt.Errorf("select existing bot api update: %w", err)
}
return row, false, nil
}
func (s *BotAPIUpdateStore) ListBotAPIUpdates(ctx context.Context, botUserID, fromUpdateID int64, limit int) ([]domain.BotAPIUpdate, error) {
if botUserID == 0 {
return nil, nil
}
if fromUpdateID <= 0 {
fromUpdateID = 1
}
if limit <= 0 || limit > 100 {
limit = 100
}
rows, err := s.db.Query(ctx, `
SELECT id, bot_user_id, update_kind, peer_type, peer_id, message_id, source_pts, date
FROM bot_api_updates
WHERE bot_user_id = $1 AND id >= $2
ORDER BY id
LIMIT $3
`, botUserID, fromUpdateID, limit)
if err != nil {
return nil, fmt.Errorf("list bot api updates: %w", err)
}
defer rows.Close()
out := make([]domain.BotAPIUpdate, 0, limit)
for rows.Next() {
item, err := scanBotAPIUpdateRows(rows)
if err != nil {
return nil, err
}
out = append(out, item)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("list bot api updates rows: %w", err)
}
return out, nil
}
func (s *BotAPIUpdateStore) ConfirmBotAPIUpdates(ctx context.Context, botUserID, confirmedUpdateID int64) error {
if botUserID == 0 || confirmedUpdateID <= 0 {
return nil
}
if _, err := s.db.Exec(ctx, `
INSERT INTO bot_api_update_states (bot_user_id, confirmed_update_id)
VALUES ($1, $2)
ON CONFLICT (bot_user_id) DO UPDATE
SET confirmed_update_id = GREATEST(bot_api_update_states.confirmed_update_id, EXCLUDED.confirmed_update_id),
updated_at = now()
WHERE bot_api_update_states.confirmed_update_id < EXCLUDED.confirmed_update_id
`, botUserID, confirmedUpdateID); err != nil {
return fmt.Errorf("confirm bot api updates: %w", err)
}
return nil
}
// DeleteDeliveredOrExpired 回收 Bot API 投递队列的死行(性能审计 H1
// 1. 已确认id <= bot_api_update_states.confirmed_update_id且入队超过 confirmedGrace 的行——
// 官方 Bot API 语义下确认即弃getUpdates 的 fromID 恒 > confirmed删除不影响任何读路径
// 宽限仅防御 offset 回拨调试场景。
// 2. 按消息 date 超过 maxAge 的行无论确认与否——对齐官方「updates 服务器最多保留 24 小时」
// 语义,同时封顶 MTProto-only bot从不调 getUpdates、无 state 行)成员身份带来的无界增长。
//
// 与 user_update_events 的「永久保留」约束无关:那是 TDesktop 账号级 differenceTooLong 缺陷所迫,
// Bot API 队列没有该约束。返回两步合计删除行数。
func (s *BotAPIUpdateStore) DeleteDeliveredOrExpired(ctx context.Context, confirmedGrace, maxAge time.Duration, limit int) (int, error) {
if limit <= 0 {
limit = 10000
}
if limit > 100000 {
limit = 100000
}
total := 0
if confirmedGrace > 0 {
// 从 states 小表出发,每 bot 走 bot_api_updates_bot_scan_idx(bot_user_id, id) 范围扫描。
tag, err := s.db.Exec(ctx, `
DELETE FROM bot_api_updates
WHERE id IN (
SELECT u.id
FROM bot_api_update_states s
JOIN bot_api_updates u ON u.bot_user_id = s.bot_user_id AND u.id <= s.confirmed_update_id
WHERE u.created_at < now() - make_interval(secs => $1)
LIMIT $2
)`, int64(confirmedGrace/time.Second), limit)
if err != nil {
return total, fmt.Errorf("delete confirmed bot api updates: %w", err)
}
total += int(tag.RowsAffected())
}
if maxAge > 0 {
cutoff := time.Now().Add(-maxAge).Unix()
// 走 bot_api_updates_retention_idx(date, id)。
tag, err := s.db.Exec(ctx, `
DELETE FROM bot_api_updates
WHERE id IN (
SELECT id
FROM bot_api_updates
WHERE date < $1
ORDER BY date, id
LIMIT $2
)`, cutoff, limit)
if err != nil {
return total, fmt.Errorf("delete expired bot api updates: %w", err)
}
total += int(tag.RowsAffected())
}
return total, nil
}
func (s *BotAPIUpdateStore) ConfirmedBotAPIUpdateID(ctx context.Context, botUserID int64) (int64, bool, error) {
if botUserID == 0 {
return 0, false, nil
}
var id int64
if err := s.db.QueryRow(ctx, `
SELECT confirmed_update_id
FROM bot_api_update_states
WHERE bot_user_id = $1
`, botUserID).Scan(&id); err != nil {
if err == pgx.ErrNoRows {
return 0, false, nil
}
return 0, false, fmt.Errorf("get bot api update state: %w", err)
}
return id, true, nil
}
func (s *BotAPIUpdateStore) scanBotAPIUpdate(row pgx.Row) (domain.BotAPIUpdate, error) {
return scanBotAPIUpdateRows(row)
}
type botAPIUpdateScanner interface {
Scan(dest ...any) error
}
func scanBotAPIUpdateRows(row botAPIUpdateScanner) (domain.BotAPIUpdate, error) {
var item domain.BotAPIUpdate
var kind, peerType string
if err := row.Scan(&item.ID, &item.BotUserID, &kind, &peerType, &item.Peer.ID, &item.MessageID, &item.SourcePts, &item.Date); err != nil {
return domain.BotAPIUpdate{}, err
}
item.Kind = domain.BotAPIUpdateKind(kind)
item.Peer.Type = domain.PeerType(peerType)
return item, nil
}
func validateBotAPIUpdateRequest(req domain.EnqueueBotAPIUpdateRequest) error {
if req.BotUserID == 0 || req.MessageID <= 0 {
return fmt.Errorf("invalid bot api update")
}
if req.Kind != domain.BotAPIUpdateMessage && req.Kind != domain.BotAPIUpdateEditedMessage {
return fmt.Errorf("invalid bot api update kind %q", req.Kind)
}
switch req.Peer.Type {
case domain.PeerTypeUser, domain.PeerTypeChannel:
if req.Peer.ID <= 0 {
return fmt.Errorf("invalid bot api update peer")
}
default:
return fmt.Errorf("invalid bot api update peer type %q", req.Peer.Type)
}
return nil
}

View file

@ -0,0 +1,122 @@
package postgres
import (
"context"
"testing"
"time"
"telesrv/internal/domain"
)
// TestBotAPIUpdateRetention 锁定 H1 场景矩阵:
// - 已确认 + 超宽限 → 删;已确认 + 宽限内 → 留;
// - 未确认 + date 超保留期 → 删(含无 state 行的 MTProto-only bot
// - 未确认 + date 在保留期内 → 留;
// - 删除后 getUpdates 读路径fromID > confirmed不受影响。
func TestBotAPIUpdateRetention(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
suffix := randomSuffix(t)
users := NewUserStore(pool)
newBot := func(phoneTail, name string) int64 {
t.Helper()
u, err := users.Create(ctx, domain.User{
AccessHash: 920,
Phone: "+1920" + suffix + phoneTail,
FirstName: name,
})
if err != nil {
t.Fatalf("create bot user %s: %v", name, err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO bots (bot_user_id, owner_user_id, token_secret)
VALUES ($1, $1, 'retention-test-secret')
ON CONFLICT (bot_user_id) DO NOTHING`, u.ID); err != nil {
t.Fatalf("seed bot %s: %v", name, err)
}
return u.ID
}
confirmedBot := newBot("01", "RetentionConfirmedBot")
mtprotoOnlyBot := newBot("02", "RetentionMTOnlyBot")
t.Cleanup(func() {
_, _ = pool.Exec(ctx, "DELETE FROM bot_api_updates WHERE bot_user_id IN ($1, $2)", confirmedBot, mtprotoOnlyBot)
_, _ = pool.Exec(ctx, "DELETE FROM bot_api_update_states WHERE bot_user_id IN ($1, $2)", confirmedBot, mtprotoOnlyBot)
_, _ = pool.Exec(ctx, "DELETE FROM bots WHERE bot_user_id IN ($1, $2)", confirmedBot, mtprotoOnlyBot)
})
s := NewBotAPIUpdateStore(pool)
now := time.Now().Unix()
stale := now - int64((48 * time.Hour).Seconds())
enqueue := func(botID int64, messageID int, date int64) domain.BotAPIUpdate {
t.Helper()
row, created, err := s.EnqueueBotAPIUpdate(ctx, domain.EnqueueBotAPIUpdateRequest{
BotUserID: botID,
Kind: domain.BotAPIUpdateMessage,
Peer: domain.Peer{Type: domain.PeerTypeUser, ID: 1},
MessageID: messageID,
SourcePts: messageID,
Date: int(date),
})
if err != nil || !created {
t.Fatalf("enqueue bot=%d msg=%d: created=%v err=%v", botID, messageID, created, err)
}
return row
}
confirmedOld := enqueue(confirmedBot, 1, now) // 已确认 + created_at 回拨超宽限 → 删
confirmedFresh := enqueue(confirmedBot, 2, now) // 已确认 + 宽限内 → 留
unconfirmedFresh := enqueue(confirmedBot, 3, now)
expiredNoState := enqueue(mtprotoOnlyBot, 4, stale) // 无 state 行 + date 超保留期 → 删
freshNoState := enqueue(mtprotoOnlyBot, 5, now)
if err := s.ConfirmBotAPIUpdates(ctx, confirmedBot, confirmedFresh.ID); err != nil {
t.Fatalf("confirm: %v", err)
}
if _, err := pool.Exec(ctx,
"UPDATE bot_api_updates SET created_at = now() - interval '1 hour' WHERE id = $1", confirmedOld.ID); err != nil {
t.Fatalf("backdate confirmed row: %v", err)
}
deleted, err := s.DeleteDeliveredOrExpired(ctx, 15*time.Minute, 24*time.Hour, 1000)
if err != nil {
t.Fatalf("DeleteDeliveredOrExpired: %v", err)
}
// 共享测试库可能有其它历史行同被回收,只要求至少删掉本测试的 2 行;
// 精确归属由下方 remaining 断言保证。
if deleted < 2 {
t.Fatalf("deleted = %d, want >= 2 (confirmed+grace expired, date expired)", deleted)
}
remaining := map[int64]bool{}
rows, err := pool.Query(ctx, "SELECT id FROM bot_api_updates WHERE bot_user_id IN ($1, $2)", confirmedBot, mtprotoOnlyBot)
if err != nil {
t.Fatalf("list remaining: %v", err)
}
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
t.Fatalf("scan remaining: %v", err)
}
remaining[id] = true
}
rows.Close()
if remaining[confirmedOld.ID] {
t.Fatal("confirmed row past grace was not deleted")
}
if remaining[expiredNoState.ID] {
t.Fatal("expired row of state-less bot was not deleted")
}
if !remaining[confirmedFresh.ID] || !remaining[unconfirmedFresh.ID] || !remaining[freshNoState.ID] {
t.Fatalf("fresh rows were deleted, remaining=%v", remaining)
}
// 读路径回归:确认水位之后的未确认行仍可被 getUpdates 读到。
items, err := s.ListBotAPIUpdates(ctx, confirmedBot, confirmedFresh.ID+1, 100)
if err != nil {
t.Fatalf("list after retention: %v", err)
}
if len(items) != 1 || items[0].ID != unconfirmedFresh.ID {
t.Fatalf("post-retention list = %+v, want only unconfirmed fresh row %d", items, unconfirmedFresh.ID)
}
}

View file

@ -136,6 +136,7 @@ func TestReadModelChangeListenerInvalidatesChannelCaches(t *testing.T) {
rows := NewChannelRowCache(16)
members := NewChannelMemberCache(16)
fullBots := &fakeChannelFullBotReadModelCache{}
botMembers := &fakeChannelBotMemberReadModelCache{}
mediaCounts := &fakeChannelMediaCountReadModelCache{}
rows.put(domain.Channel{ID: 7, Title: "old"})
members.put(domain.ChannelMember{ChannelID: 7, UserID: 100, Status: domain.ChannelMemberActive})
@ -146,6 +147,7 @@ func TestReadModelChangeListenerInvalidatesChannelCaches(t *testing.T) {
ChannelRows: rows,
ChannelMembers: members,
ChannelFullBots: fullBots,
ChannelBotMembers: botMembers,
ChannelMediaCounts: mediaCounts,
}, nil)
listener.handlePayload(`{"model":"channel_member","owner_user_id":100,"peer_type":"channel","peer_id":7,"version":2}`)
@ -158,6 +160,9 @@ func TestReadModelChangeListenerInvalidatesChannelCaches(t *testing.T) {
if got := fullBots.channelsSnapshot(); len(got) != 1 || got[0] != 7 {
t.Fatalf("channel_member 应失效 full bot info: %+v", got)
}
if got := botMembers.channelsSnapshot(); len(got) != 1 || got[0] != 7 {
t.Fatalf("channel_member 应失效 bot member ids: %+v", got)
}
if got := mediaCounts.viewerSnapshot(); len(got) != 1 || got[0] != [2]int64{100, 7} {
t.Fatalf("channel_member 应失效该 viewer 的 media count: %+v", got)
}
@ -175,8 +180,16 @@ func TestReadModelChangeListenerInvalidatesChannelCaches(t *testing.T) {
if got := fullBots.channelsSnapshot(); len(got) != 2 || got[1] != 7 {
t.Fatalf("channel_base 应失效 full bot info: %+v", got)
}
if got := botMembers.channelsSnapshot(); len(got) != 2 || got[1] != 7 {
t.Fatalf("channel_base 应失效 bot member ids: %+v", got)
}
listener.handlePayload(`{"model":"channel_media_counts","owner_user_id":0,"peer_type":"channel","peer_id":7,"version":4}`)
listener.handlePayload(`{"model":"channel_participants","owner_user_id":0,"peer_type":"channel","peer_id":7,"version":4}`)
if got := botMembers.channelsSnapshot(); len(got) != 3 || got[2] != 7 {
t.Fatalf("channel_participants 应失效 bot member ids: %+v", got)
}
listener.handlePayload(`{"model":"channel_media_counts","owner_user_id":0,"peer_type":"channel","peer_id":7,"version":5}`)
if got := mediaCounts.channelSnapshot(); len(got) != 1 || got[0] != 7 {
t.Fatalf("channel_media_counts 应失效该频道 media count: %+v", got)
}
@ -244,6 +257,30 @@ func (f *fakeChannelFullBotReadModelCache) flushCount() int {
return f.flushes
}
type fakeChannelBotMemberReadModelCache struct {
mu sync.Mutex
channels []int64
flushes int
}
func (f *fakeChannelBotMemberReadModelCache) InvalidateActiveBotMemberIDsReadModel(channelID int64) {
f.mu.Lock()
defer f.mu.Unlock()
f.channels = append(f.channels, channelID)
}
func (f *fakeChannelBotMemberReadModelCache) FlushActiveBotMemberIDsReadModel() {
f.mu.Lock()
defer f.mu.Unlock()
f.flushes++
}
func (f *fakeChannelBotMemberReadModelCache) channelsSnapshot() []int64 {
f.mu.Lock()
defer f.mu.Unlock()
return append([]int64(nil), f.channels...)
}
type fakeChannelMediaCountReadModelCache struct {
mu sync.Mutex
channels []int64

View file

@ -27,6 +27,7 @@ type ReadModelCacheSet struct {
ProfilePhotos ProfilePhotoReadModelCache
Stories StoryReadModelCache
ChannelFullBots ChannelFullBotReadModelCache
ChannelBotMembers ChannelBotMemberReadModelCache
ChannelMediaCounts ChannelMediaCountReadModelCache
PrivateMediaCounts PrivateMediaCountReadModelCache
RPCProjections RPCProjectionReadModelCache
@ -99,6 +100,11 @@ type ChannelFullBotReadModelCache interface {
FlushChannelFullBotInfoReadModel()
}
type ChannelBotMemberReadModelCache interface {
InvalidateActiveBotMemberIDsReadModel(channelID int64)
FlushActiveBotMemberIDsReadModel()
}
type ChannelMediaCountReadModelCache interface {
InvalidateChannelMediaCountReadModel(channelID int64)
InvalidateChannelMediaCountReadModelForViewer(userID, channelID int64)
@ -203,6 +209,7 @@ func (l *ReadModelChangeListener) empty() bool {
l.caches.ProfilePhotos == nil &&
l.caches.Stories == nil &&
l.caches.ChannelFullBots == nil &&
l.caches.ChannelBotMembers == nil &&
l.caches.ChannelMediaCounts == nil &&
l.caches.PrivateMediaCounts == nil &&
l.caches.RPCProjections == nil &&
@ -260,6 +267,10 @@ func (l *ReadModelChangeListener) flush(reasons ...string) {
l.caches.ChannelFullBots.FlushChannelFullBotInfoReadModel()
flushed = append(flushed, "channel_full_bots")
}
if l.caches.ChannelBotMembers != nil {
l.caches.ChannelBotMembers.FlushActiveBotMemberIDsReadModel()
flushed = append(flushed, "channel_bot_members")
}
if l.caches.ChannelMediaCounts != nil {
l.caches.ChannelMediaCounts.FlushChannelMediaCountReadModel()
flushed = append(flushed, "channel_media_counts")
@ -396,6 +407,9 @@ func (l *ReadModelChangeListener) handlePayload(payload string) {
if l.caches.ChannelFullBots != nil {
l.caches.ChannelFullBots.InvalidateChannelFullBotInfoReadModel(evt.PeerID)
}
if l.caches.ChannelBotMembers != nil {
l.caches.ChannelBotMembers.InvalidateActiveBotMemberIDsReadModel(evt.PeerID)
}
if l.caches.RPCProjections != nil {
l.caches.RPCProjections.InvalidateRPCProjectionReadModelForChannel(evt.PeerID)
}
@ -421,6 +435,9 @@ func (l *ReadModelChangeListener) handlePayload(payload string) {
if l.caches.ChannelFullBots != nil {
l.caches.ChannelFullBots.InvalidateChannelFullBotInfoReadModel(evt.PeerID)
}
if l.caches.ChannelBotMembers != nil {
l.caches.ChannelBotMembers.InvalidateActiveBotMemberIDsReadModel(evt.PeerID)
}
if l.caches.RPCProjections != nil {
l.caches.RPCProjections.InvalidateRPCProjectionReadModelForChannel(evt.PeerID)
l.caches.RPCProjections.InvalidateRPCProjectionReadModelForPeer(evt.OwnerUserID, domain.Peer{Type: domain.PeerTypeChannel, ID: evt.PeerID})
@ -439,6 +456,9 @@ func (l *ReadModelChangeListener) handlePayload(payload string) {
l.caches.RPCProjections.InvalidateRPCProjectionReadModelForPeer(evt.OwnerUserID, domain.Peer{Type: domain.PeerTypeChannel, ID: evt.PeerID})
}
case "channel_participants":
if evt.PeerType == "channel" && evt.PeerID != 0 && l.caches.ChannelBotMembers != nil {
l.caches.ChannelBotMembers.InvalidateActiveBotMemberIDsReadModel(evt.PeerID)
}
if evt.PeerType == "channel" && evt.PeerID != 0 && l.caches.RPCProjections != nil {
l.caches.RPCProjections.InvalidateRPCProjectionReadModelForChannel(evt.PeerID)
}