369 lines
12 KiB
Go
369 lines
12 KiB
Go
package rpc
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/gotd/td/proto"
|
||
"github.com/gotd/td/tg"
|
||
"go.uber.org/zap"
|
||
|
||
"telesrv/internal/domain"
|
||
"telesrv/internal/store"
|
||
)
|
||
|
||
const (
|
||
defaultOutboxBatch = 100
|
||
defaultOutboxInterval = 200 * time.Millisecond
|
||
defaultOutboxWorkers = 2
|
||
)
|
||
|
||
var errMissingOutboxEvent = errors.New("missing outbox update event")
|
||
|
||
// OutboxDispatcher 把 PG transactional outbox 中的 update 批量推给在线 session。
|
||
// 多 worker 并发 claim：ClaimPending 用 FOR UPDATE SKIP LOCKED，worker 间认领不重叠。
|
||
type OutboxDispatcher struct {
|
||
events store.UpdateEventStore
|
||
outbox store.DispatchOutboxStore
|
||
sessions SessionBinder
|
||
log *zap.Logger
|
||
metrics Metrics
|
||
batch int
|
||
interval time.Duration
|
||
workers int
|
||
pushTimeout time.Duration
|
||
}
|
||
|
||
// OutboxOption 调整 OutboxDispatcher 的运行参数。
|
||
type OutboxOption func(*OutboxDispatcher)
|
||
|
||
// WithOutboxBatch 设置每次 claim 的最大条数；<=0 时保持默认。
|
||
func WithOutboxBatch(n int) OutboxOption {
|
||
return func(d *OutboxDispatcher) {
|
||
if n > 0 {
|
||
d.batch = n
|
||
}
|
||
}
|
||
}
|
||
|
||
// WithOutboxInterval 设置两次 claim 之间的轮询间隔；<=0 时保持默认。
|
||
func WithOutboxInterval(interval time.Duration) OutboxOption {
|
||
return func(d *OutboxDispatcher) {
|
||
if interval > 0 {
|
||
d.interval = interval
|
||
}
|
||
}
|
||
}
|
||
|
||
// WithOutboxWorkers 设置并发 claim worker 数；<=0 时保持默认。
|
||
func WithOutboxWorkers(n int) OutboxOption {
|
||
return func(d *OutboxDispatcher) {
|
||
if n > 0 {
|
||
d.workers = n
|
||
}
|
||
}
|
||
}
|
||
|
||
// WithOutboxPushTimeout 设置 updates fanout 入队等待时间；<=0 时使用同步可靠推送。
|
||
func WithOutboxPushTimeout(timeout time.Duration) OutboxOption {
|
||
return func(d *OutboxDispatcher) {
|
||
if timeout > 0 {
|
||
d.pushTimeout = timeout
|
||
}
|
||
}
|
||
}
|
||
|
||
// WithOutboxMetrics 注入指标实现；nil 时保持 NopMetrics。
|
||
func WithOutboxMetrics(m Metrics) OutboxOption {
|
||
return func(d *OutboxDispatcher) {
|
||
if m != nil {
|
||
d.metrics = m
|
||
}
|
||
}
|
||
}
|
||
|
||
// NewOutboxDispatcher 创建在线 update 推送 worker。batch/interval 默认值见
|
||
// defaultOutboxBatch/defaultOutboxInterval，可经 WithOutbox* 选项覆盖（生产由 config 注入）。
|
||
func NewOutboxDispatcher(events store.UpdateEventStore, outbox store.DispatchOutboxStore, sessions SessionBinder, log *zap.Logger, opts ...OutboxOption) *OutboxDispatcher {
|
||
if log == nil {
|
||
log = zap.NewNop()
|
||
}
|
||
d := &OutboxDispatcher{
|
||
events: events,
|
||
outbox: outbox,
|
||
sessions: sessions,
|
||
log: log,
|
||
metrics: NopMetrics{},
|
||
batch: defaultOutboxBatch,
|
||
interval: defaultOutboxInterval,
|
||
workers: defaultOutboxWorkers,
|
||
}
|
||
for _, opt := range opts {
|
||
if opt != nil {
|
||
opt(d)
|
||
}
|
||
}
|
||
return d
|
||
}
|
||
|
||
// Run 启动 workers 个并发 worker 持续 claim pending outbox；ctx 退出时全部停止并等待退出。
|
||
func (d *OutboxDispatcher) Run(ctx context.Context) {
|
||
if d == nil || d.events == nil || d.outbox == nil || d.sessions == nil {
|
||
return
|
||
}
|
||
workers := d.workers
|
||
if workers < 1 {
|
||
workers = 1
|
||
}
|
||
var wg sync.WaitGroup
|
||
wg.Add(workers)
|
||
for i := 0; i < workers; i++ {
|
||
go func() {
|
||
defer wg.Done()
|
||
d.runWorker(ctx)
|
||
}()
|
||
}
|
||
wg.Wait()
|
||
}
|
||
|
||
// runWorker 是单个 claim 循环；多 worker 靠 ClaimPending 的 SKIP LOCKED 互不重叠。
|
||
func (d *OutboxDispatcher) runWorker(ctx context.Context) {
|
||
ticker := time.NewTicker(d.interval)
|
||
defer ticker.Stop()
|
||
for {
|
||
d.DispatchOnce(ctx)
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-ticker.C:
|
||
}
|
||
}
|
||
}
|
||
|
||
// batchEventLoader 是 UpdateEventStore 的可选批量能力：一次取多条 (user,pts) 事件。
|
||
type batchEventLoader interface {
|
||
BatchByCursor(ctx context.Context, cursors []store.EventCursor) ([]domain.UpdateEvent, error)
|
||
}
|
||
|
||
// batchOutboxMarker 是 DispatchOutboxStore 的可选批量能力：一次标记多行 delivered。
|
||
type batchOutboxMarker interface {
|
||
MarkDeliveredBatch(ctx context.Context, items []store.DispatchOutboxItem) error
|
||
}
|
||
|
||
// DispatchOnce claim 一批 outbox 并投递，测试可直接调用。
|
||
// store 同时具备批量取事件 + 批量标记能力时走批量路径（每批 ~3 次 PG 往返），否则逐条回退。
|
||
func (d *OutboxDispatcher) DispatchOnce(ctx context.Context) {
|
||
items, err := d.outbox.ClaimPending(ctx, d.batch)
|
||
if err != nil {
|
||
d.log.Warn("claim dispatch outbox", zap.Error(err))
|
||
return
|
||
}
|
||
if len(items) == 0 {
|
||
return
|
||
}
|
||
d.metrics.OutboxClaimed(len(items))
|
||
if loader, ok := d.events.(batchEventLoader); ok {
|
||
if marker, ok := d.outbox.(batchOutboxMarker); ok {
|
||
d.dispatchBatch(ctx, items, loader, marker)
|
||
return
|
||
}
|
||
}
|
||
for _, item := range items {
|
||
d.dispatchItem(ctx, item)
|
||
}
|
||
}
|
||
|
||
type outboxEventKey struct {
|
||
userID int64
|
||
pts int
|
||
}
|
||
|
||
// dispatchBatch 批量加载已 claim 事件、逐条 push、批量标记 delivered；失败项单独退避重试。
|
||
func (d *OutboxDispatcher) dispatchBatch(ctx context.Context, items []store.DispatchOutboxItem, loader batchEventLoader, marker batchOutboxMarker) {
|
||
cursors := make([]store.EventCursor, len(items))
|
||
for i, item := range items {
|
||
cursors[i] = store.EventCursor{UserID: item.TargetUserID, Pts: item.Pts}
|
||
}
|
||
events, err := loader.BatchByCursor(ctx, cursors)
|
||
if err != nil {
|
||
// 批量取失败则整批回退逐条路径，让每条各自重试/标失败，不丢进度。
|
||
d.log.Warn("batch load dispatch events", zap.Error(err))
|
||
for _, item := range items {
|
||
d.dispatchItem(ctx, item)
|
||
}
|
||
return
|
||
}
|
||
byKey := make(map[outboxEventKey]domain.UpdateEvent, len(events))
|
||
for _, event := range events {
|
||
byKey[outboxEventKey{event.UserID, event.Pts}] = event
|
||
}
|
||
start := time.Now()
|
||
delivered := make([]store.DispatchOutboxItem, 0, len(items))
|
||
for _, item := range items {
|
||
event, ok := byKey[outboxEventKey{item.TargetUserID, item.Pts}]
|
||
if !ok {
|
||
d.markDispatchFailed(ctx, item, errMissingOutboxEvent)
|
||
continue
|
||
}
|
||
update := tgUpdateForOutboxEvent(event)
|
||
if update == nil {
|
||
delivered = append(delivered, item)
|
||
continue
|
||
}
|
||
if _, retriable, err := d.pushOutboxUpdate(ctx, item, update); err != nil {
|
||
if retriable {
|
||
// 出站队列拥塞：留 dispatching 行靠租约过期重投，不计入 attempts 升级。
|
||
// 不加入 delivered，故不会被 MarkDeliveredBatch 删除。
|
||
continue
|
||
}
|
||
d.markDispatchFailed(ctx, item, err)
|
||
continue
|
||
}
|
||
delivered = append(delivered, item)
|
||
}
|
||
if len(delivered) == 0 {
|
||
return
|
||
}
|
||
if err := marker.MarkDeliveredBatch(ctx, delivered); err != nil {
|
||
// 批量标记失败则逐条标记，避免整批已投递却卡在 dispatching 等租约过期重投。
|
||
d.log.Warn("mark dispatch delivered batch", zap.Error(err))
|
||
for _, item := range delivered {
|
||
if markErr := d.outbox.MarkDelivered(ctx, item.TargetUserID, item.ID); markErr != nil {
|
||
d.log.Warn("mark dispatch delivered", zap.Int64("target_user_id", item.TargetUserID), zap.Int64("outbox_id", item.ID), zap.Error(markErr))
|
||
}
|
||
}
|
||
}
|
||
per := time.Since(start) / time.Duration(len(delivered))
|
||
for range delivered {
|
||
d.metrics.OutboxDelivered(per)
|
||
}
|
||
}
|
||
|
||
func (d *OutboxDispatcher) dispatchItem(ctx context.Context, item store.DispatchOutboxItem) {
|
||
start := time.Now()
|
||
events, err := d.events.ListAfter(ctx, item.TargetUserID, item.Pts-1, 1)
|
||
if err != nil {
|
||
d.markDispatchFailed(ctx, item, err)
|
||
return
|
||
}
|
||
if len(events) == 0 || events[0].Pts != item.Pts {
|
||
d.markDispatchFailed(ctx, item, errMissingOutboxEvent)
|
||
return
|
||
}
|
||
update := tgUpdateForOutboxEvent(events[0])
|
||
if update == nil {
|
||
if err := d.outbox.MarkDelivered(ctx, item.TargetUserID, item.ID); err != nil {
|
||
d.log.Warn("mark noop dispatch delivered", zap.Int64("target_user_id", item.TargetUserID), zap.Int64("outbox_id", item.ID), zap.Error(err))
|
||
return
|
||
}
|
||
d.metrics.OutboxDelivered(time.Since(start))
|
||
return
|
||
}
|
||
sent, retriable, err := d.pushOutboxUpdate(ctx, item, update)
|
||
if err != nil {
|
||
if retriable {
|
||
// 出站队列拥塞：保留 dispatching 行，靠租约过期（defaultDispatchLease）重新 claim 重投，
|
||
// 不计入 attempts 升级，避免正常满 fan-out 负载把可靠 update 误打成 failed。
|
||
d.log.Debug("dispatch outbox deferred (push queue full)",
|
||
zap.Int64("target_user_id", item.TargetUserID),
|
||
zap.Int64("outbox_id", item.ID),
|
||
zap.Int("pts", item.Pts),
|
||
)
|
||
return
|
||
}
|
||
d.markDispatchFailed(ctx, item, err)
|
||
return
|
||
}
|
||
if err := d.outbox.MarkDelivered(ctx, item.TargetUserID, item.ID); err != nil {
|
||
d.log.Warn("mark dispatch delivered", zap.Int64("target_user_id", item.TargetUserID), zap.Int64("outbox_id", item.ID), zap.Error(err))
|
||
return
|
||
}
|
||
d.metrics.OutboxDelivered(time.Since(start))
|
||
d.log.Debug("dispatch outbox delivered",
|
||
zap.Int64("target_user_id", item.TargetUserID),
|
||
zap.Int64("outbox_id", item.ID),
|
||
zap.Int("pts", item.Pts),
|
||
zap.Int("sessions", sent),
|
||
)
|
||
}
|
||
|
||
// pushOutboxUpdate 投递一条 outbox update，返回 (送达的在线 session 数, 是否可重试, err)。
|
||
// best-effort 路径（pushTimeout>0）的失败只可能是出站队列拥塞（慢消费者入队超时），属暂时性、
|
||
// 可重试：调用方应保留 dispatching 行靠租约过期重投，而非计入 attempts 升级为 failed。
|
||
// 可靠路径的失败是真实投递错误，retriable=false，按原逻辑退避升级。
|
||
func (d *OutboxDispatcher) pushOutboxUpdate(ctx context.Context, item store.DispatchOutboxItem, update *tg.Updates) (sent int, retriable bool, err error) {
|
||
var zeroAuthKeyID [8]byte
|
||
if d.pushTimeout > 0 {
|
||
if scoped, ok := d.sessions.(ScopedBestEffortSessionBinder); ok && item.ExcludeAuthKeyID != zeroAuthKeyID {
|
||
sent, err = scoped.PushToUserExceptAuthKeySessionBestEffort(ctx, item.TargetUserID, item.ExcludeAuthKeyID, item.ExcludeSessionID, proto.MessageFromServer, update, d.pushTimeout)
|
||
return sent, err != nil, err
|
||
}
|
||
if bestEffort, ok := d.sessions.(BestEffortSessionBinder); ok {
|
||
sent, err = bestEffort.PushToUserExceptSessionBestEffort(ctx, item.TargetUserID, item.ExcludeSessionID, proto.MessageFromServer, update, d.pushTimeout)
|
||
return sent, err != nil, err
|
||
}
|
||
}
|
||
if scoped, ok := d.sessions.(ScopedSessionBinder); ok && item.ExcludeAuthKeyID != zeroAuthKeyID {
|
||
sent, err = scoped.PushToUserExceptAuthKeySession(ctx, item.TargetUserID, item.ExcludeAuthKeyID, item.ExcludeSessionID, proto.MessageFromServer, update)
|
||
return sent, false, err
|
||
}
|
||
sent, err = d.sessions.PushToUserExceptSession(ctx, item.TargetUserID, item.ExcludeSessionID, proto.MessageFromServer, update)
|
||
return sent, false, err
|
||
}
|
||
|
||
func (d *OutboxDispatcher) markDispatchFailed(ctx context.Context, item store.DispatchOutboxItem, err error) {
|
||
if err == nil {
|
||
err = errMissingOutboxEvent
|
||
}
|
||
if markErr := d.outbox.MarkFailed(ctx, item.TargetUserID, item.ID, err.Error()); markErr != nil {
|
||
d.log.Warn("mark dispatch failed",
|
||
zap.Int64("target_user_id", item.TargetUserID),
|
||
zap.Int64("outbox_id", item.ID),
|
||
zap.Error(markErr),
|
||
)
|
||
return
|
||
}
|
||
d.metrics.OutboxFailed(err)
|
||
d.log.Debug("dispatch outbox failed",
|
||
zap.Int64("target_user_id", item.TargetUserID),
|
||
zap.Int64("outbox_id", item.ID),
|
||
zap.Int("pts", item.Pts),
|
||
zap.Error(err),
|
||
)
|
||
}
|
||
|
||
func tgUpdateForOutboxEvent(event domain.UpdateEvent) *tg.Updates {
|
||
switch event.Type {
|
||
case domain.UpdateEventNewMessage:
|
||
return tgPrivateMessageUpdates(event, event.Message, 0, false, tgUsers(event.Users), tgChannels(event.UserID, event.Channels))
|
||
case domain.UpdateEventReadHistoryInbox, domain.UpdateEventReadHistoryOutbox:
|
||
var update tg.UpdateClass
|
||
if event.Type == domain.UpdateEventReadHistoryOutbox {
|
||
update = tgReadHistoryOutboxUpdate(event)
|
||
} else {
|
||
update = tgReadHistoryInboxUpdate(event)
|
||
}
|
||
if update == nil {
|
||
return nil
|
||
}
|
||
return &tg.Updates{
|
||
Updates: []tg.UpdateClass{update},
|
||
Date: event.Date,
|
||
Seq: 0, // 私聊不维护账号级 seq，恒 0
|
||
}
|
||
case domain.UpdateEventNoop:
|
||
return nil
|
||
default:
|
||
update := tgOtherUpdateFromEvent(event)
|
||
if update == nil {
|
||
return nil
|
||
}
|
||
return &tg.Updates{
|
||
Updates: []tg.UpdateClass{update},
|
||
Date: event.Date,
|
||
Seq: 0,
|
||
}
|
||
}
|
||
}
|