perf: sync protocol and core hardening updates

This commit is contained in:
A 2026-07-11 19:48:26 +08:00
parent 152fed3b87
commit 4390ebf5a9
283 changed files with 29231 additions and 2295 deletions

View file

@ -4,7 +4,9 @@ import (
"context"
"errors"
"fmt"
"sort"
"sync"
"sync/atomic"
"time"
"go.uber.org/zap"
@ -36,7 +38,8 @@ const (
// (typing/presence,不写 durable log)经 PushToUserTransient* 在未就绪时直接跳过、不入队,
// 因此本队列被老化/溢出/重试耗尽丢弃时,丢的一定是 durable 条目——getDifference 以
// user_update_events 兜底补齐,丢弃不丢数据。
pendingPushMaxAge = 60 * time.Second
pendingPushMaxAge = 60 * time.Second
defaultPendingPushMaxBytes = int64(256 << 20)
// maxSessionsPerAuthKey:单个 raw auth_key 允许同时在线的 session 上限。telesrv 单 DC,
// 一个客户端的全部连接(主连接 + 并发下载/上传)共享同一 auth_key、各用独立 session_id,
// 故此上限须高于真实客户端单设备的并发连接峰值,否则会误杀活跃下载/主连接:
@ -52,10 +55,51 @@ const (
maxChannelIndexPerSession = 8192
)
// forceCloseBatchTimeout is one deadline for a whole revoke/replace/eviction batch. Conn.Close
// already bounds its inbound-RPC wait, but calling ForceClose serially would multiply that bound
// by the number of sessions. The batch helper starts every close concurrently and waits at most
// this one shared interval.
const forceCloseBatchTimeout = rpcCloseWaitTimeout
// maxForceCloseParallelism caps control-plane close goroutines even if a corrupted/runtime index
// hands a revoke path far more sessions than maxSessionsPerAuthKey. Every Conn's producer/RPC gate
// is closed synchronously before these workers start, so a stuck transport.Close cannot admit more
// memory while the bounded workers continue draining physical sockets in the background.
const maxForceCloseParallelism = 64
type queuedPush struct {
t proto.MessageType
msg bin.Encoder
at time.Time
t proto.MessageType
encoded *encodedOutboundMessage
reservation *pendingPushReservation
at time.Time
}
type pendingPushReservation struct {
budget *outboundTrackedBudget
bytes int
refs atomic.Int32
}
func (r *pendingPushReservation) retain() {
if r == nil {
return
}
if refs := r.refs.Add(1); refs <= 1 {
panic("mtprotoedge: retained released pending push reservation")
}
}
func (r *pendingPushReservation) release() {
if r == nil {
return
}
refs := r.refs.Add(-1)
if refs < 0 {
panic("mtprotoedge: pending push reservation released more than retained")
}
if refs == 0 {
r.budget.release(r.bytes)
}
}
type sessionKey struct {
@ -85,6 +129,7 @@ type SessionManager struct {
bySessionMembers map[sessionKey]map[int64]struct{}
pending map[sessionKey][]queuedPush // updates-ready 前暂存的主动推送
flushing map[sessionKey]bool // 置位时暂存正在排空的 session;排空完成前推送继续进 pending 保序
pendingBudget *outboundTrackedBudget // 未就绪 session 暂存 encoded body 的进程级上限
lifecycle SessionLifecycleObserver
log *zap.Logger
@ -107,6 +152,7 @@ func NewSessionManager(log *zap.Logger) *SessionManager {
bySessionMembers: make(map[sessionKey]map[int64]struct{}),
pending: make(map[sessionKey][]queuedPush),
flushing: make(map[sessionKey]bool),
pendingBudget: newOutboundTrackedBudget(defaultPendingPushMaxBytes),
log: log,
}
}
@ -161,11 +207,13 @@ func (m *SessionManager) Register(c *Conn) {
)
m.mu.Unlock()
if replaced != nil {
replaced.Close()
}
if evicted != nil {
evicted.Close()
// 同 identity 的新物理连接已经原子接管索引;立即关闭旧 transport,不能只停
// actor 后让旧 FD/read goroutine 滞留到 read timeout。replacement 与 cap eviction
// 共用一个并发关闭批次,不能把每条 Conn 的 RPC 等待上界串行相加。
if replaced != nil || evicted != nil {
if !forceCloseConnBatch([]*Conn{replaced, evicted}, forceCloseBatchTimeout) {
m.log.Warn("Session replacement/eviction close exceeded shared deadline")
}
}
}
@ -217,7 +265,12 @@ func (m *SessionManager) DestroySession(sessionID int64) bool {
zap.Int("online", len(m.bySession)),
)
m.mu.Unlock()
c.Close()
if !forceCloseConnBatch([]*Conn{c}, forceCloseBatchTimeout) {
m.log.Warn("Destroyed session close exceeded shared deadline",
zap.String("auth_key_id", sessionKeyLog(key.authKeyID)),
zap.Int64("session_id", sessionID),
)
}
if observer != nil && offlineUser != 0 {
observer.SessionOffline(key.authKeyID, sessionID, offlineUser, lastForUser)
}
@ -230,7 +283,7 @@ func (m *SessionManager) DestroySessionForAuthKey(authKeyID [8]byte, sessionID i
key := sessionKey{authKeyID: authKeyID, sessionID: sessionID}
c, ok := m.bySession[key]
if !ok {
delete(m.pending, key)
m.deletePendingLocked(key)
m.mu.Unlock()
return false
}
@ -243,7 +296,12 @@ func (m *SessionManager) DestroySessionForAuthKey(authKeyID [8]byte, sessionID i
zap.Int("online", len(m.bySession)),
)
m.mu.Unlock()
c.Close()
if !forceCloseConnBatch([]*Conn{c}, forceCloseBatchTimeout) {
m.log.Warn("Destroyed session close exceeded shared deadline",
zap.String("auth_key_id", sessionKeyLog(authKeyID)),
zap.Int64("session_id", sessionID),
)
}
if observer != nil && offlineUser != 0 {
observer.SessionOffline(authKeyID, sessionID, offlineUser, lastForUser)
}
@ -287,7 +345,7 @@ func (m *SessionManager) bindUserLocked(c *Conn, key sessionKey, userID int64) {
c.membershipsSynced.Store(false)
// 身份变化即丢弃暂存推送:它们属于前一个账号,flush 给新账号是跨账号泄露。
// 同时取消进行中的排空(runFlush 还另有 owner 校验做批内兜底)。
delete(m.pending, key)
m.deletePendingLocked(key)
delete(m.flushing, key)
}
}
@ -298,7 +356,7 @@ func (m *SessionManager) bindUserLocked(c *Conn, key sessionKey, userID int64) {
m.clearChannelInterestsLocked(key)
m.clearChannelMembershipsLocked(c, key)
c.membershipsSynced.Store(false)
delete(m.pending, key)
m.deletePendingLocked(key)
delete(m.flushing, key)
}
}
@ -399,7 +457,7 @@ func (m *SessionManager) bindAuthKeyLocked(c *Conn, key sessionKey, authKeyID [8
m.clearChannelInterestsLocked(key)
m.clearChannelMembershipsLocked(c, key)
c.membershipsSynced.Store(false)
delete(m.pending, key)
m.deletePendingLocked(key)
delete(m.flushing, key)
c.userID.Store(0)
c.userIDResolved.Store(false)
@ -459,8 +517,11 @@ func (m *SessionManager) CloseSessionsForBusinessAuthKey(authKeyID [8]byte) int
)
}
m.mu.Unlock()
for _, c := range conns {
c.ForceClose()
if !forceCloseConnBatch(conns, forceCloseBatchTimeout) {
m.log.Warn("Revoked auth-key session close exceeded shared deadline",
zap.String("auth_key_id", sessionKeyLog(authKeyID)),
zap.Int("sessions", len(conns)),
)
}
if observer != nil {
for _, e := range events {
@ -493,8 +554,11 @@ func (m *SessionManager) CloseSessionsForRawAuthKeyExcept(authKeyID [8]byte, exc
}
observer := m.lifecycle
m.mu.Unlock()
for _, c := range conns {
c.ForceClose()
if !forceCloseConnBatch(conns, forceCloseBatchTimeout) {
m.log.Warn("Raw auth-key session close exceeded shared deadline",
zap.String("auth_key_id", sessionKeyLog(authKeyID)),
zap.Int("sessions", len(conns)),
)
}
if observer != nil {
for _, e := range events {
@ -504,6 +568,99 @@ func (m *SessionManager) CloseSessionsForRawAuthKeyExcept(authKeyID [8]byte, exc
return len(conns)
}
// forceCloseConnBatch closes every producer/RPC gate first, then closes physical transports with a
// bounded worker set. Physical close and actor/RPC convergence share one batch deadline; the wait is
// never multiplied by the number of sessions. Workers may finish physical closes after the caller's
// deadline, but no timed-out Conn can enqueue more work in that interval. Nil/duplicate entries are
// removed so Register's replacement/eviction slots cannot close the same Conn twice.
func forceCloseConnBatch(conns []*Conn, timeout time.Duration) bool {
if len(conns) == 0 {
return true
}
unique := make([]*Conn, 0, len(conns))
seen := make(map[*Conn]struct{}, len(conns))
for _, c := range conns {
if c == nil {
continue
}
if _, ok := seen[c]; ok {
continue
}
seen[c] = struct{}{}
unique = append(unique, c)
}
if len(unique) == 0 {
return true
}
// This phase is non-blocking and must precede transport.Close: it is the safety boundary if
// an implementation of transport.Conn.Close itself blocks past the batch deadline.
for _, c := range unique {
c.beginTerminalShutdown()
}
workers := min(len(unique), maxForceCloseParallelism)
jobs := make(chan *Conn, len(unique))
for _, c := range unique {
jobs <- c
}
close(jobs)
var closeWG sync.WaitGroup
closeWG.Add(workers)
for range workers {
go func() {
defer closeWG.Done()
for c := range jobs {
c.closeTransport()
}
}()
}
physicalDone := make(chan struct{})
go func() {
closeWG.Wait()
close(physicalDone)
}()
if timeout <= 0 {
return false
}
deadline := time.Now().Add(timeout)
timer := time.NewTimer(time.Until(deadline))
defer timer.Stop()
select {
case <-physicalDone:
case <-timer.C:
return false
}
// All physical close calls returned. Wait for memory-owning actor/RPC work using the same
// deadline; the first genuinely stuck Conn consumes the remaining allowance, not a fresh 5s.
for _, c := range unique {
remaining := time.Until(deadline)
if remaining <= 0 {
return false
}
if c.rpcScheduler != nil && !c.waitInboundShutdown(remaining) {
return false
}
if c.outboundDone == nil {
continue
}
remaining = time.Until(deadline)
if remaining <= 0 {
return false
}
wait := time.NewTimer(remaining)
select {
case <-c.outboundDone:
wait.Stop()
case <-wait.C:
return false
}
}
return true
}
// UnbindAuthKey 清理某业务 auth_key 下所有活跃连接的登录用户缓存。
func (m *SessionManager) UnbindAuthKey(authKeyID [8]byte) int {
m.mu.Lock()
@ -520,7 +677,7 @@ func (m *SessionManager) UnbindAuthKey(authKeyID [8]byte) int {
m.clearChannelMembershipsLocked(c, key)
c.membershipsSynced.Store(false)
// 授权解除后暂存推送属于已登出的账号,不能等下一个登录者置位时 flush 出去。
delete(m.pending, key)
m.deletePendingLocked(key)
delete(m.flushing, key)
c.userIDResolved.Store(true)
count++
@ -595,7 +752,7 @@ func (m *SessionManager) runFlush(c *Conn, key sessionKey, owner int64, attempt
}
if c.userID.Load() != owner {
// 排空期间发生登出/换号:剩余暂存属于旧账号,丢弃且不得发给新账号。
delete(m.pending, key)
m.deletePendingLocked(key)
delete(m.flushing, key)
m.mu.Unlock()
return
@ -613,38 +770,47 @@ func (m *SessionManager) runFlush(c *Conn, key sessionKey, owner int64, attempt
// 每条发送前复查身份:登出/换号后 batch 的剩余条目不能继续发到已易主的连接。
if c.userID.Load() != owner {
m.mu.Lock()
delete(m.pending, key)
m.deletePendingLocked(key)
delete(m.flushing, key)
m.mu.Unlock()
releaseQueuedPushes(batch[i:])
return
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
err := c.Send(ctx, item.t, item.msg)
// Pending entries are durable account updates. Shared body-budget pressure is not
// evidence that this socket is corrupt, so use the non-terminal enqueue path; after
// bounded retries, getDifference is the authoritative recovery path.
err := c.SendBestEffortEncoded(ctx, item.t, item.encoded, 5*time.Second)
cancel()
if err == nil {
item.release()
continue
}
m.mu.Lock()
if cur, ok := m.bySession[key]; !ok || cur != c || !m.flushing[key] || c.userID.Load() != owner {
// 连接换代/取消/易主:剩余 batch 不属于当前连接当前账号,丢弃。
if c.userID.Load() != owner {
delete(m.pending, key)
m.deletePendingLocked(key)
delete(m.flushing, key)
}
m.mu.Unlock()
releaseQueuedPushes(batch[i:])
return
}
rest := append(append([]queuedPush(nil), batch[i:]...), m.pending[key]...)
if len(rest) > maxPendingPushesPerSession {
// 与 queueLocked 溢出策略一致:丢最旧留最新,让 pts 空洞集中在最前端,
// flush 首条即触发客户端 gap 检测,恢复路径最短。
rest = rest[len(rest)-maxPendingPushesPerSession:]
dropped := len(rest) - maxPendingPushesPerSession
releaseQueuedPushes(rest[:dropped])
rest = rest[dropped:]
}
m.pending[key] = rest
if attempt+1 >= maxFlushAttempts {
// 重试用尽:置位激活避免 idle 客户端永久断流;剩余暂存中的 durable 更新
// 由客户端后续 pts 空洞触发 getDifference 补齐。
c.receivesUpdates.Store(true)
m.deletePendingLocked(key)
delete(m.flushing, key)
m.mu.Unlock()
m.log.Debug("Flush gave up after retries; activated with getDifference fallback",
@ -717,41 +883,61 @@ func (m *SessionManager) SetReceivesUpdatesForAuthKey(authKeyID [8]byte, session
// PushToSession 向指定 session 推送一条消息。
func (m *SessionManager) PushToSession(ctx context.Context, sessionID int64, t proto.MessageType, msg bin.Encoder) error {
m.mu.Lock()
m.mu.RLock()
c, key, ok, ambiguous := m.uniqueSessionLocked(sessionID)
if ambiguous {
m.mu.Unlock()
m.mu.RUnlock()
return ErrSessionAmbiguous
}
if !ok {
m.mu.Unlock()
m.mu.RUnlock()
return ErrSessionNotFound
}
if !c.receivesUpdates.Load() {
m.queueLocked(key, t, msg)
m.mu.Unlock()
return nil
ready := c.receivesUpdates.Load()
m.mu.RUnlock()
if ready {
return c.Send(ctx, t, msg)
}
m.mu.Unlock()
return c.Send(ctx, t, msg)
return m.queueOrSendPrepared(ctx, key, t, msg)
}
// PushToSessionForAuthKey 向指定 raw auth_key_id + session_id 推送一条消息。
func (m *SessionManager) PushToSessionForAuthKey(ctx context.Context, authKeyID [8]byte, sessionID int64, t proto.MessageType, msg bin.Encoder) error {
m.mu.Lock()
m.mu.RLock()
key := sessionKey{authKeyID: authKeyID, sessionID: sessionID}
c, ok := m.bySession[key]
if !ok {
m.mu.RUnlock()
return ErrSessionNotFound
}
ready := c.receivesUpdates.Load()
m.mu.RUnlock()
if ready {
return c.Send(ctx, t, msg)
}
return m.queueOrSendPrepared(ctx, key, t, msg)
}
func (m *SessionManager) queueOrSendPrepared(ctx context.Context, key sessionKey, t proto.MessageType, msg bin.Encoder) error {
encoded, reservation, err := m.preparePendingPush(ctx, msg)
if err != nil {
return err
}
defer reservation.release()
m.mu.Lock()
c, ok := m.bySession[key]
if !ok {
m.mu.Unlock()
return ErrSessionNotFound
}
if !c.receivesUpdates.Load() {
m.queueLocked(key, t, msg)
_ = m.queuePreparedLocked(key, t, encoded, reservation)
m.mu.Unlock()
return nil
}
m.mu.Unlock()
return c.Send(ctx, t, msg)
return c.SendEncoded(ctx, t, encoded)
}
// PushToSessionForAuthKeyImmediate 向指定 raw auth_key_id + session_id 立即推送一条消息。
@ -782,7 +968,7 @@ func (m *SessionManager) PushToUserExceptSession(ctx context.Context, userID, ex
return m.pushToUser(ctx, userID, nil, excludeSessionID, t, msg)
}
// PushToUserExceptAuthKeySession 向某 user 所有活跃连接推送,跳过指定业务 auth_key + session。
// PushToUserExceptAuthKeySession 向某 user 所有活跃连接推送,跳过指定 raw auth_key + session。
func (m *SessionManager) PushToUserExceptAuthKeySession(ctx context.Context, userID int64, excludeAuthKeyID [8]byte, excludeSessionID int64, t proto.MessageType, msg bin.Encoder) (int, error) {
return m.pushToUser(ctx, userID, &excludeAuthKeyID, excludeSessionID, t, msg)
}
@ -793,23 +979,28 @@ func (m *SessionManager) PushToUserExceptAuthKeySession(ctx context.Context, use
// 漏 temp-key 设备)。未就绪连接跳过、不进 pending——密聊消息 durable 在 qts 队列,
// 离线设备靠 getDifference 补回(在线推送只是加速器)。c.userID 复查防跨账号泄露。
func (m *SessionManager) PushToUserAuthKey(ctx context.Context, userID int64, businessAuthKeyID [8]byte, t proto.MessageType, msg bin.Encoder) (int, error) {
getEncoded := onceEncodedOutbound(msg)
return m.pushToBusinessAuthKey(ctx, userID, businessAuthKeyID, false, func(c *Conn) error {
if c.outbound == nil || c.outboundControl == nil {
return ErrConnClosed
}
encoded, err := getEncoded()
if err != nil {
return err
}
return c.SendEncoded(ctx, t, encoded)
})
// Secret-chat qts is the durable source of truth, so online delivery is an accelerator just
// like account pts fan-out. Do not synchronously wait for every PFS/raw connection's socket.
return m.pushToBusinessAuthKeyBestEffort(ctx, userID, businessAuthKeyID, t, msg, 2*time.Second)
}
// PushToUserAuthKeyTransient 是 PushToUserAuthKey 的 transient(typing)best-effort 版本。
func (m *SessionManager) PushToUserAuthKeyTransient(ctx context.Context, userID int64, businessAuthKeyID [8]byte, t proto.MessageType, msg bin.Encoder, timeout time.Duration) (int, error) {
getEncoded := onceEncodedOutbound(msg)
return m.pushToBusinessAuthKey(ctx, userID, businessAuthKeyID, true, func(c *Conn) error {
return m.pushToBusinessAuthKeyBestEffort(ctx, userID, businessAuthKeyID, t, msg, timeout)
}
func (m *SessionManager) pushToBusinessAuthKeyBestEffort(ctx context.Context, userID int64, businessAuthKeyID [8]byte, t proto.MessageType, msg bin.Encoder, timeout time.Duration) (int, error) {
getEncoded := onceEncodedOutbound(ctx, msg)
var deadline time.Time
if timeout > 0 {
deadline = time.Now().Add(timeout)
}
if ctx != nil {
if ctxDeadline, ok := ctx.Deadline(); ok && (deadline.IsZero() || ctxDeadline.Before(deadline)) {
deadline = ctxDeadline
}
}
return m.pushToBusinessAuthKey(ctx, userID, businessAuthKeyID, func(c *Conn) error {
if c.outbound == nil || c.outboundControl == nil {
return ErrConnClosed
}
@ -817,11 +1008,18 @@ func (m *SessionManager) PushToUserAuthKeyTransient(ctx context.Context, userID
if err != nil {
return err
}
return c.SendBestEffortEncoded(ctx, t, encoded, timeout)
remaining := timeout
if !deadline.IsZero() {
remaining = time.Until(deadline)
if remaining < 0 {
remaining = 0
}
}
return c.SendBestEffortEncoded(ctx, t, encoded, remaining)
})
}
func (m *SessionManager) pushToBusinessAuthKey(ctx context.Context, userID int64, businessAuthKeyID [8]byte, transient bool, send func(*Conn) error) (int, error) {
func (m *SessionManager) pushToBusinessAuthKey(ctx context.Context, userID int64, businessAuthKeyID [8]byte, send func(*Conn) error) (int, error) {
m.mu.Lock()
candidates := m.businessAuthKeyCandidatesLocked(businessAuthKeyID)
conns := make([]*Conn, 0, len(candidates))
@ -836,7 +1034,6 @@ func (m *SessionManager) pushToBusinessAuthKey(ctx context.Context, userID int64
conns = append(conns, c)
}
m.mu.Unlock()
_ = transient
var firstErr error
sent := 0
for _, c := range conns {
@ -845,6 +1042,18 @@ func (m *SessionManager) pushToBusinessAuthKey(ctx context.Context, userID int64
continue
}
if err := send(c); err != nil {
if errors.Is(err, ErrOutboundTrackedBudget) {
// Shared process pressure is not evidence that this particular socket is
// slow. Skip this online accelerator; durable qts/difference is the truth.
continue
}
if errors.Is(err, ErrOutboundQueueFull) {
c.dropSlowConsumer()
continue
}
if errors.Is(err, ErrConnClosed) {
continue
}
if firstErr == nil {
firstErr = err
}
@ -856,7 +1065,7 @@ func (m *SessionManager) pushToBusinessAuthKey(ctx context.Context, userID int64
}
func (m *SessionManager) pushToUser(ctx context.Context, userID int64, excludeAuthKeyID *[8]byte, excludeSessionID int64, t proto.MessageType, msg bin.Encoder) (int, error) {
getEncoded := onceEncodedOutbound(msg)
getEncoded := onceEncodedOutbound(ctx, msg)
return m.pushToUserWithSender(ctx, userID, excludeAuthKeyID, excludeSessionID, t, msg, true, func(c *Conn) error {
if c.outbound == nil || c.outboundControl == nil {
return ErrConnClosed
@ -875,7 +1084,7 @@ func (m *SessionManager) pushToUser(ctx context.Context, userID int64, excludeAu
// 下一次状态变化重建,囤积过期 transient 既无意义又会被 pending 的老化/溢出/重试耗尽误当
// 「durable 兜底」丢弃。走 best-effort 发送,不阻塞调用方。
func (m *SessionManager) PushToUserTransientExceptAuthKeySession(ctx context.Context, userID int64, excludeAuthKeyID [8]byte, excludeSessionID int64, t proto.MessageType, msg bin.Encoder, timeout time.Duration) (int, error) {
getEncoded := onceEncodedOutbound(msg)
getEncoded := onceEncodedOutbound(ctx, msg)
return m.pushToUserWithSender(ctx, userID, &excludeAuthKeyID, excludeSessionID, t, msg, false, func(c *Conn) error {
if c.outbound == nil || c.outboundControl == nil {
return ErrConnClosed
@ -897,7 +1106,19 @@ func (m *SessionManager) PushToUserExceptAuthKeySessionBestEffort(ctx context.Co
}
func (m *SessionManager) pushToUserBestEffort(ctx context.Context, userID int64, excludeAuthKeyID *[8]byte, excludeSessionID int64, t proto.MessageType, msg bin.Encoder, timeout time.Duration) (int, error) {
getEncoded := onceEncodedOutbound(msg)
getEncoded := onceEncodedOutbound(ctx, msg)
// timeout 是整次 fan-out 的等待预算,不是每个 session 各自一份。健康连接始终先走
// SendBestEffortEncoded 的非阻塞快路径;预算耗尽后 remaining=0,仍会尝试快路径,
// 但不会再为后续慢连接串行等待。
var deadline time.Time
if timeout > 0 {
deadline = time.Now().Add(timeout)
}
if ctx != nil {
if ctxDeadline, ok := ctx.Deadline(); ok && (deadline.IsZero() || ctxDeadline.Before(deadline)) {
deadline = ctxDeadline
}
}
return m.pushToUserWithSender(ctx, userID, excludeAuthKeyID, excludeSessionID, t, msg, true, func(c *Conn) error {
if c.outbound == nil || c.outboundControl == nil {
return ErrConnClosed
@ -906,18 +1127,25 @@ func (m *SessionManager) pushToUserBestEffort(ctx context.Context, userID int64,
if err != nil {
return err
}
return c.SendBestEffortEncoded(ctx, t, encoded, timeout)
remaining := timeout
if !deadline.IsZero() {
remaining = time.Until(deadline)
if remaining < 0 {
remaining = 0
}
}
return c.SendBestEffortEncoded(ctx, t, encoded, remaining)
})
}
func onceEncodedOutbound(msg bin.Encoder) func() (*encodedOutboundMessage, error) {
func onceEncodedOutbound(ctx context.Context, msg bin.Encoder) func() (*encodedOutboundMessage, error) {
var (
encoded *encodedOutboundMessage
err error
)
return func() (*encodedOutboundMessage, error) {
if encoded == nil && err == nil {
encoded, err = encodeOutboundMessage(msg)
encoded, err = encodeOutboundMessageContext(ctx, msg)
}
return encoded, err
}
@ -957,6 +1185,10 @@ func (m *SessionManager) pushToUserWithSender(ctx context.Context, userID int64,
}
m.mu.RUnlock()
if needQueue {
// TL encoding and the process-wide pending-byte reservation may be expensive or
// briefly block on the global encode gate. Do both before taking SessionManager.mu,
// then share the immutable body across every not-ready session found by the re-scan.
pendingEncoded, pendingReservation, pendingErr := m.preparePendingPush(ctx, msg)
// 写锁下完整重扫(读锁释放到此之间状态可能变化,以重扫结果为准)。
conns = conns[:0]
queued, dropped, excluded, skipped = 0, 0, 0, 0
@ -972,7 +1204,7 @@ func (m *SessionManager) pushToUserWithSender(ctx context.Context, userID int64,
skipped++
continue
}
if m.queueLocked(key, t, msg) {
if pendingErr == nil && m.queuePreparedLocked(key, t, pendingEncoded, pendingReservation) {
queued++
if debug {
m.log.Debug("Push queued (session not updates-ready)",
@ -996,6 +1228,15 @@ func (m *SessionManager) pushToUserWithSender(ctx context.Context, userID int64,
conns = append(conns, c)
}
m.mu.Unlock()
if pendingReservation != nil {
pendingReservation.release() // drop producer ref; queued entries own the body now.
}
if pendingErr != nil && debug {
m.log.Debug("Drop pending pushes outside byte budget",
zap.Int64("user_id", userID),
zap.Error(pendingErr),
)
}
}
var firstErr error
@ -1008,6 +1249,29 @@ func (m *SessionManager) pushToUserWithSender(ctx context.Context, userID int64,
continue
}
if err := send(c); err != nil {
if errors.Is(err, ErrOutboundTrackedBudget) {
// Do not turn pressure owned by other sockets into a reconnect storm on
// healthy recipients. The durable event remains recoverable by difference.
dropped++
continue
}
// 对 durable/best-effort fan-out,队列满意味着该 socket 已成为慢消费者。
// 立即摘除并把它视为离线:不能让其错误把已经投递给健康 session 的 outbox
// 行整体重试。该 session 的 durable gap 由 getDifference 恢复。
if errors.Is(err, ErrOutboundQueueFull) {
c.dropSlowConsumer()
if debug {
m.log.Debug("Drop slow outbound consumer",
zap.Int64("user_id", userID),
zap.String("auth_key_id", sessionKeyLog(c.authKeyID)),
zap.Int64("session_id", c.sessionID),
)
}
continue
}
if errors.Is(err, ErrConnClosed) {
continue
}
if firstErr == nil {
firstErr = err
}
@ -1055,6 +1319,24 @@ func (m *SessionManager) Online() int {
return len(m.bySession)
}
// ActiveRawAuthKeyIDs 返回当前物理连接实际使用的 raw auth_key_id 去重快照。
// maintenance 用它保护“已建 key 但尚未登录”的长连接不被 orphan GC 删除;不能用
// business/temp→perm key 替代,否则活跃 temp 连接仍可能误删。
func (m *SessionManager) ActiveRawAuthKeyIDs() [][8]byte {
m.mu.RLock()
defer m.mu.RUnlock()
seen := make(map[[8]byte]struct{}, len(m.bySession))
out := make([][8]byte, 0, len(m.byAuthKey))
for key := range m.bySession {
if _, ok := seen[key.authKeyID]; ok {
continue
}
seen[key.authKeyID] = struct{}{}
out = append(out, key.authKeyID)
}
return out
}
// IsUserOnline returns whether userID has at least one active connection.
func (m *SessionManager) IsUserOnline(userID int64) bool {
if userID == 0 {
@ -1268,6 +1550,54 @@ func (m *SessionManager) OnlineChannelMemberUserIDsExcluding(channelID int64, ex
return out
}
// OnlineChannelIDsSnapshot returns every channel with at least one live joined-member session in
// strictly ascending order. The global SessionManager lock is held only while copying map keys;
// sorting and all recovery database work happen after unlock. The fixed saturation-recovery actor
// is the sole caller, so its exceptional-path temporary memory is one int64 slice (peak about 8*C
// bytes) rather than repeated O(C) scans under the connection/membership lock.
func (m *SessionManager) OnlineChannelIDsSnapshot() []int64 {
m.mu.RLock()
out := make([]int64, 0, len(m.byMemberChannel))
for channelID, sessions := range m.byMemberChannel {
if channelID <= 0 || len(sessions) == 0 {
continue
}
live := false
for key := range sessions {
if _, ok := m.bySession[key]; ok {
live = true
break
}
}
if !live {
continue
}
out = append(out, channelID)
}
m.mu.RUnlock()
sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
return out
}
// OnlineChannelIDsAfter is retained for bounded diagnostics/tests. Production recovery takes one
// OnlineChannelIDsSnapshot per generation and slices it into pages, avoiding repeated full scans.
func (m *SessionManager) OnlineChannelIDsAfter(afterChannelID int64, limit int) []int64 {
if limit <= 0 {
return nil
}
const maxRecoveryPage = 4096
if limit > maxRecoveryPage {
limit = maxRecoveryPage
}
all := m.OnlineChannelIDsSnapshot()
start := sort.Search(len(all), func(i int) bool { return all[i] > afterChannelID })
end := start + limit
if end > len(all) {
end = len(all)
}
return all[start:end]
}
func (m *SessionManager) onlineChannelUsers(index map[int64]map[sessionKey]int64, channelID int64, limit int) []int64 {
if channelID == 0 {
return nil
@ -1314,7 +1644,7 @@ func (m *SessionManager) removeLocked(c *Conn, dropPending bool) int64 {
m.clearChannelInterestsLocked(key)
m.clearChannelMembershipsLocked(c, key)
if dropPending {
delete(m.pending, key)
m.deletePendingLocked(key)
}
delete(m.flushing, key)
return uid
@ -1430,8 +1760,11 @@ func (m *SessionManager) takePendingLocked(key sessionKey, ready bool) []queuedP
now := time.Now()
pending := make([]queuedPush, 0, len(q))
dropped := 0
for _, item := range q {
for i := range q {
item := q[i]
q[i] = queuedPush{}
if now.Sub(item.at) > pendingPushMaxAge {
item.release()
dropped++
continue
}
@ -1447,9 +1780,43 @@ func (m *SessionManager) takePendingLocked(key sessionKey, ready bool) []queuedP
return pending
}
// queueLocked 暂存一条主动推送,返回是否实际入队——stale 丢批分支会连同当前
// 这条一起丢弃,调用方据此区分 queued/dropped 计数,避免投递日志失真。
func (m *SessionManager) queueLocked(key sessionKey, t proto.MessageType, msg bin.Encoder) bool {
// preparePendingPush encodes outside SessionManager.mu and reserves the one physical body before
// releasing the process-wide encode slot. Multiple not-ready sessions may then share this
// immutable body via reservation refs instead of encoding/copying it once per session.
func (m *SessionManager) preparePendingPush(ctx context.Context, msg bin.Encoder) (*encodedOutboundMessage, *pendingPushReservation, error) {
var (
encoded *encodedOutboundMessage
bytes int
)
err := withOutboundEncodeSlot(ctx, nil, func() error {
var err error
encoded, err = encodeOutboundMessageWithoutSlot(msg)
if err != nil {
return err
}
if encoded == nil {
return errors.New("nil encoded pending push")
}
bytes = len(encoded.body)
if bytes > maxOutboundBodyBytes {
return fmt.Errorf("%w: body=%d limit=%d", ErrOutboundMessageTooLarge, bytes, maxOutboundBodyBytes)
}
if !m.pendingBudget.reserve(bytes) {
return ErrOutboundTrackedBudget
}
return nil
})
if err != nil {
return nil, nil, err
}
reservation := &pendingPushReservation{budget: m.pendingBudget, bytes: bytes}
reservation.refs.Store(1) // producer ownership; queue entries retain below.
return encoded, reservation, nil
}
// queuePreparedLocked 暂存一条已编码的主动推送,返回是否实际入队。
// 调用方必须在锁外保持 reservation 的 producer ref,并在全部入队完成后 release。
func (m *SessionManager) queuePreparedLocked(key sessionKey, t proto.MessageType, encoded *encodedOutboundMessage, reservation *pendingPushReservation) bool {
q := m.pending[key]
// 过期保护:最早一条暂存已超过 pendingPushMaxAge(session 迟迟未 ready)时,丢整批并
// 不再囤这条,记 trace。避免「登录后从不 getState」的连接长期占用 pending 内存。
@ -1459,11 +1826,21 @@ func (m *SessionManager) queueLocked(key sessionKey, t proto.MessageType, msg bi
zap.Int64("session_id", key.sessionID),
zap.Int("dropped", len(q)),
)
delete(m.pending, key)
m.deletePendingLocked(key)
return false
}
push := queuedPush{t: t, msg: msg, at: time.Now()}
if encoded == nil || reservation == nil {
return false
}
reservation.retain()
push := queuedPush{
t: t,
encoded: encoded,
reservation: reservation,
at: time.Now(),
}
if len(q) >= maxPendingPushesPerSession {
q[0].release()
copy(q, q[1:])
q[len(q)-1] = push
m.pending[key] = q
@ -1473,6 +1850,22 @@ func (m *SessionManager) queueLocked(key sessionKey, t proto.MessageType, msg bi
return true
}
// queueLocked remains as a test/internal single-target convenience. Production fan-out prepares
// outside m.mu and calls queuePreparedLocked so TL encoding never serializes the session registry.
func (m *SessionManager) queueLocked(key sessionKey, t proto.MessageType, msg bin.Encoder) bool {
encoded, reservation, err := m.preparePendingPush(context.Background(), msg)
if err != nil {
m.log.Debug("Drop pending push outside byte budget",
zap.String("auth_key_id", sessionKeyLog(key.authKeyID)),
zap.Int64("session_id", key.sessionID),
zap.Error(err),
)
return false
}
defer reservation.release()
return m.queuePreparedLocked(key, t, encoded, reservation)
}
func (m *SessionManager) uniqueSessionLocked(sessionID int64) (*Conn, sessionKey, bool, bool) {
set := m.bySessionID[sessionID]
if len(set) == 0 {
@ -1490,7 +1883,7 @@ func (m *SessionManager) uniqueSessionLocked(sessionID int64) (*Conn, sessionKey
func (m *SessionManager) dropPendingBySessionLocked(sessionID int64) {
for key := range m.pending {
if key.sessionID == sessionID {
delete(m.pending, key)
m.deletePendingLocked(key)
}
}
}
@ -1528,7 +1921,7 @@ func (m *SessionManager) sweepStalePending() {
if len(q) == 0 || now.Sub(q[0].at) <= pendingPushMaxAge {
continue
}
delete(m.pending, key)
m.deletePendingLocked(key)
dropped++
}
if dropped > 0 {
@ -1536,6 +1929,27 @@ func (m *SessionManager) sweepStalePending() {
}
}
func (q *queuedPush) release() {
if q == nil {
return
}
reservation := q.reservation
*q = queuedPush{}
reservation.release()
}
func releaseQueuedPushes(q []queuedPush) {
for i := range q {
q[i].release()
}
}
func (m *SessionManager) deletePendingLocked(key sessionKey) {
q := m.pending[key]
delete(m.pending, key)
releaseQueuedPushes(q)
}
func addConnIndex[K comparable](idx map[K]map[int64]*Conn, key K, sessionID int64, c *Conn) {
set := idx[key]
if set == nil {
@ -1630,7 +2044,7 @@ func shouldExcludeSession(c *Conn, excludeAuthKeyID *[8]byte, excludeSessionID i
if excludeAuthKeyID == nil || *excludeAuthKeyID == ([8]byte{}) {
return true
}
return connUsesBusinessAuthKey(c, *excludeAuthKeyID)
return c.authKeyID == *excludeAuthKeyID
}
func sessionKeyLog(id [8]byte) string {