perf: sync protocol and core hardening updates
This commit is contained in:
parent
152fed3b87
commit
4390ebf5a9
283 changed files with 29231 additions and 2295 deletions
|
|
@ -4,7 +4,9 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
|
@ -36,7 +38,8 @@ const (
|
|||
// (typing/presence,不写 durable log)经 PushToUserTransient* 在未就绪时直接跳过、不入队,
|
||||
// 因此本队列被老化/溢出/重试耗尽丢弃时,丢的一定是 durable 条目——getDifference 以
|
||||
// user_update_events 兜底补齐,丢弃不丢数据。
|
||||
pendingPushMaxAge = 60 * time.Second
|
||||
pendingPushMaxAge = 60 * time.Second
|
||||
defaultPendingPushMaxBytes = int64(256 << 20)
|
||||
// maxSessionsPerAuthKey:单个 raw auth_key 允许同时在线的 session 上限。telesrv 单 DC,
|
||||
// 一个客户端的全部连接(主连接 + 并发下载/上传)共享同一 auth_key、各用独立 session_id,
|
||||
// 故此上限须高于真实客户端单设备的并发连接峰值,否则会误杀活跃下载/主连接:
|
||||
|
|
@ -52,10 +55,51 @@ const (
|
|||
maxChannelIndexPerSession = 8192
|
||||
)
|
||||
|
||||
// forceCloseBatchTimeout is one deadline for a whole revoke/replace/eviction batch. Conn.Close
|
||||
// already bounds its inbound-RPC wait, but calling ForceClose serially would multiply that bound
|
||||
// by the number of sessions. The batch helper starts every close concurrently and waits at most
|
||||
// this one shared interval.
|
||||
const forceCloseBatchTimeout = rpcCloseWaitTimeout
|
||||
|
||||
// maxForceCloseParallelism caps control-plane close goroutines even if a corrupted/runtime index
|
||||
// hands a revoke path far more sessions than maxSessionsPerAuthKey. Every Conn's producer/RPC gate
|
||||
// is closed synchronously before these workers start, so a stuck transport.Close cannot admit more
|
||||
// memory while the bounded workers continue draining physical sockets in the background.
|
||||
const maxForceCloseParallelism = 64
|
||||
|
||||
type queuedPush struct {
|
||||
t proto.MessageType
|
||||
msg bin.Encoder
|
||||
at time.Time
|
||||
t proto.MessageType
|
||||
encoded *encodedOutboundMessage
|
||||
reservation *pendingPushReservation
|
||||
at time.Time
|
||||
}
|
||||
|
||||
type pendingPushReservation struct {
|
||||
budget *outboundTrackedBudget
|
||||
bytes int
|
||||
refs atomic.Int32
|
||||
}
|
||||
|
||||
func (r *pendingPushReservation) retain() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
if refs := r.refs.Add(1); refs <= 1 {
|
||||
panic("mtprotoedge: retained released pending push reservation")
|
||||
}
|
||||
}
|
||||
|
||||
func (r *pendingPushReservation) release() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
refs := r.refs.Add(-1)
|
||||
if refs < 0 {
|
||||
panic("mtprotoedge: pending push reservation released more than retained")
|
||||
}
|
||||
if refs == 0 {
|
||||
r.budget.release(r.bytes)
|
||||
}
|
||||
}
|
||||
|
||||
type sessionKey struct {
|
||||
|
|
@ -85,6 +129,7 @@ type SessionManager struct {
|
|||
bySessionMembers map[sessionKey]map[int64]struct{}
|
||||
pending map[sessionKey][]queuedPush // updates-ready 前暂存的主动推送
|
||||
flushing map[sessionKey]bool // 置位时暂存正在排空的 session;排空完成前推送继续进 pending 保序
|
||||
pendingBudget *outboundTrackedBudget // 未就绪 session 暂存 encoded body 的进程级上限
|
||||
|
||||
lifecycle SessionLifecycleObserver
|
||||
log *zap.Logger
|
||||
|
|
@ -107,6 +152,7 @@ func NewSessionManager(log *zap.Logger) *SessionManager {
|
|||
bySessionMembers: make(map[sessionKey]map[int64]struct{}),
|
||||
pending: make(map[sessionKey][]queuedPush),
|
||||
flushing: make(map[sessionKey]bool),
|
||||
pendingBudget: newOutboundTrackedBudget(defaultPendingPushMaxBytes),
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
|
@ -161,11 +207,13 @@ func (m *SessionManager) Register(c *Conn) {
|
|||
)
|
||||
m.mu.Unlock()
|
||||
|
||||
if replaced != nil {
|
||||
replaced.Close()
|
||||
}
|
||||
if evicted != nil {
|
||||
evicted.Close()
|
||||
// 同 identity 的新物理连接已经原子接管索引;立即关闭旧 transport,不能只停
|
||||
// actor 后让旧 FD/read goroutine 滞留到 read timeout。replacement 与 cap eviction
|
||||
// 共用一个并发关闭批次,不能把每条 Conn 的 RPC 等待上界串行相加。
|
||||
if replaced != nil || evicted != nil {
|
||||
if !forceCloseConnBatch([]*Conn{replaced, evicted}, forceCloseBatchTimeout) {
|
||||
m.log.Warn("Session replacement/eviction close exceeded shared deadline")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -217,7 +265,12 @@ func (m *SessionManager) DestroySession(sessionID int64) bool {
|
|||
zap.Int("online", len(m.bySession)),
|
||||
)
|
||||
m.mu.Unlock()
|
||||
c.Close()
|
||||
if !forceCloseConnBatch([]*Conn{c}, forceCloseBatchTimeout) {
|
||||
m.log.Warn("Destroyed session close exceeded shared deadline",
|
||||
zap.String("auth_key_id", sessionKeyLog(key.authKeyID)),
|
||||
zap.Int64("session_id", sessionID),
|
||||
)
|
||||
}
|
||||
if observer != nil && offlineUser != 0 {
|
||||
observer.SessionOffline(key.authKeyID, sessionID, offlineUser, lastForUser)
|
||||
}
|
||||
|
|
@ -230,7 +283,7 @@ func (m *SessionManager) DestroySessionForAuthKey(authKeyID [8]byte, sessionID i
|
|||
key := sessionKey{authKeyID: authKeyID, sessionID: sessionID}
|
||||
c, ok := m.bySession[key]
|
||||
if !ok {
|
||||
delete(m.pending, key)
|
||||
m.deletePendingLocked(key)
|
||||
m.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
|
|
@ -243,7 +296,12 @@ func (m *SessionManager) DestroySessionForAuthKey(authKeyID [8]byte, sessionID i
|
|||
zap.Int("online", len(m.bySession)),
|
||||
)
|
||||
m.mu.Unlock()
|
||||
c.Close()
|
||||
if !forceCloseConnBatch([]*Conn{c}, forceCloseBatchTimeout) {
|
||||
m.log.Warn("Destroyed session close exceeded shared deadline",
|
||||
zap.String("auth_key_id", sessionKeyLog(authKeyID)),
|
||||
zap.Int64("session_id", sessionID),
|
||||
)
|
||||
}
|
||||
if observer != nil && offlineUser != 0 {
|
||||
observer.SessionOffline(authKeyID, sessionID, offlineUser, lastForUser)
|
||||
}
|
||||
|
|
@ -287,7 +345,7 @@ func (m *SessionManager) bindUserLocked(c *Conn, key sessionKey, userID int64) {
|
|||
c.membershipsSynced.Store(false)
|
||||
// 身份变化即丢弃暂存推送:它们属于前一个账号,flush 给新账号是跨账号泄露。
|
||||
// 同时取消进行中的排空(runFlush 还另有 owner 校验做批内兜底)。
|
||||
delete(m.pending, key)
|
||||
m.deletePendingLocked(key)
|
||||
delete(m.flushing, key)
|
||||
}
|
||||
}
|
||||
|
|
@ -298,7 +356,7 @@ func (m *SessionManager) bindUserLocked(c *Conn, key sessionKey, userID int64) {
|
|||
m.clearChannelInterestsLocked(key)
|
||||
m.clearChannelMembershipsLocked(c, key)
|
||||
c.membershipsSynced.Store(false)
|
||||
delete(m.pending, key)
|
||||
m.deletePendingLocked(key)
|
||||
delete(m.flushing, key)
|
||||
}
|
||||
}
|
||||
|
|
@ -399,7 +457,7 @@ func (m *SessionManager) bindAuthKeyLocked(c *Conn, key sessionKey, authKeyID [8
|
|||
m.clearChannelInterestsLocked(key)
|
||||
m.clearChannelMembershipsLocked(c, key)
|
||||
c.membershipsSynced.Store(false)
|
||||
delete(m.pending, key)
|
||||
m.deletePendingLocked(key)
|
||||
delete(m.flushing, key)
|
||||
c.userID.Store(0)
|
||||
c.userIDResolved.Store(false)
|
||||
|
|
@ -459,8 +517,11 @@ func (m *SessionManager) CloseSessionsForBusinessAuthKey(authKeyID [8]byte) int
|
|||
)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
for _, c := range conns {
|
||||
c.ForceClose()
|
||||
if !forceCloseConnBatch(conns, forceCloseBatchTimeout) {
|
||||
m.log.Warn("Revoked auth-key session close exceeded shared deadline",
|
||||
zap.String("auth_key_id", sessionKeyLog(authKeyID)),
|
||||
zap.Int("sessions", len(conns)),
|
||||
)
|
||||
}
|
||||
if observer != nil {
|
||||
for _, e := range events {
|
||||
|
|
@ -493,8 +554,11 @@ func (m *SessionManager) CloseSessionsForRawAuthKeyExcept(authKeyID [8]byte, exc
|
|||
}
|
||||
observer := m.lifecycle
|
||||
m.mu.Unlock()
|
||||
for _, c := range conns {
|
||||
c.ForceClose()
|
||||
if !forceCloseConnBatch(conns, forceCloseBatchTimeout) {
|
||||
m.log.Warn("Raw auth-key session close exceeded shared deadline",
|
||||
zap.String("auth_key_id", sessionKeyLog(authKeyID)),
|
||||
zap.Int("sessions", len(conns)),
|
||||
)
|
||||
}
|
||||
if observer != nil {
|
||||
for _, e := range events {
|
||||
|
|
@ -504,6 +568,99 @@ func (m *SessionManager) CloseSessionsForRawAuthKeyExcept(authKeyID [8]byte, exc
|
|||
return len(conns)
|
||||
}
|
||||
|
||||
// forceCloseConnBatch closes every producer/RPC gate first, then closes physical transports with a
|
||||
// bounded worker set. Physical close and actor/RPC convergence share one batch deadline; the wait is
|
||||
// never multiplied by the number of sessions. Workers may finish physical closes after the caller's
|
||||
// deadline, but no timed-out Conn can enqueue more work in that interval. Nil/duplicate entries are
|
||||
// removed so Register's replacement/eviction slots cannot close the same Conn twice.
|
||||
func forceCloseConnBatch(conns []*Conn, timeout time.Duration) bool {
|
||||
if len(conns) == 0 {
|
||||
return true
|
||||
}
|
||||
unique := make([]*Conn, 0, len(conns))
|
||||
seen := make(map[*Conn]struct{}, len(conns))
|
||||
for _, c := range conns {
|
||||
if c == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[c]; ok {
|
||||
continue
|
||||
}
|
||||
seen[c] = struct{}{}
|
||||
unique = append(unique, c)
|
||||
}
|
||||
if len(unique) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
// This phase is non-blocking and must precede transport.Close: it is the safety boundary if
|
||||
// an implementation of transport.Conn.Close itself blocks past the batch deadline.
|
||||
for _, c := range unique {
|
||||
c.beginTerminalShutdown()
|
||||
}
|
||||
|
||||
workers := min(len(unique), maxForceCloseParallelism)
|
||||
jobs := make(chan *Conn, len(unique))
|
||||
for _, c := range unique {
|
||||
jobs <- c
|
||||
}
|
||||
close(jobs)
|
||||
var closeWG sync.WaitGroup
|
||||
closeWG.Add(workers)
|
||||
for range workers {
|
||||
go func() {
|
||||
defer closeWG.Done()
|
||||
for c := range jobs {
|
||||
c.closeTransport()
|
||||
}
|
||||
}()
|
||||
}
|
||||
physicalDone := make(chan struct{})
|
||||
go func() {
|
||||
closeWG.Wait()
|
||||
close(physicalDone)
|
||||
}()
|
||||
|
||||
if timeout <= 0 {
|
||||
return false
|
||||
}
|
||||
deadline := time.Now().Add(timeout)
|
||||
timer := time.NewTimer(time.Until(deadline))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-physicalDone:
|
||||
case <-timer.C:
|
||||
return false
|
||||
}
|
||||
|
||||
// All physical close calls returned. Wait for memory-owning actor/RPC work using the same
|
||||
// deadline; the first genuinely stuck Conn consumes the remaining allowance, not a fresh 5s.
|
||||
for _, c := range unique {
|
||||
remaining := time.Until(deadline)
|
||||
if remaining <= 0 {
|
||||
return false
|
||||
}
|
||||
if c.rpcScheduler != nil && !c.waitInboundShutdown(remaining) {
|
||||
return false
|
||||
}
|
||||
if c.outboundDone == nil {
|
||||
continue
|
||||
}
|
||||
remaining = time.Until(deadline)
|
||||
if remaining <= 0 {
|
||||
return false
|
||||
}
|
||||
wait := time.NewTimer(remaining)
|
||||
select {
|
||||
case <-c.outboundDone:
|
||||
wait.Stop()
|
||||
case <-wait.C:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// UnbindAuthKey 清理某业务 auth_key 下所有活跃连接的登录用户缓存。
|
||||
func (m *SessionManager) UnbindAuthKey(authKeyID [8]byte) int {
|
||||
m.mu.Lock()
|
||||
|
|
@ -520,7 +677,7 @@ func (m *SessionManager) UnbindAuthKey(authKeyID [8]byte) int {
|
|||
m.clearChannelMembershipsLocked(c, key)
|
||||
c.membershipsSynced.Store(false)
|
||||
// 授权解除后暂存推送属于已登出的账号,不能等下一个登录者置位时 flush 出去。
|
||||
delete(m.pending, key)
|
||||
m.deletePendingLocked(key)
|
||||
delete(m.flushing, key)
|
||||
c.userIDResolved.Store(true)
|
||||
count++
|
||||
|
|
@ -595,7 +752,7 @@ func (m *SessionManager) runFlush(c *Conn, key sessionKey, owner int64, attempt
|
|||
}
|
||||
if c.userID.Load() != owner {
|
||||
// 排空期间发生登出/换号:剩余暂存属于旧账号,丢弃且不得发给新账号。
|
||||
delete(m.pending, key)
|
||||
m.deletePendingLocked(key)
|
||||
delete(m.flushing, key)
|
||||
m.mu.Unlock()
|
||||
return
|
||||
|
|
@ -613,38 +770,47 @@ func (m *SessionManager) runFlush(c *Conn, key sessionKey, owner int64, attempt
|
|||
// 每条发送前复查身份:登出/换号后 batch 的剩余条目不能继续发到已易主的连接。
|
||||
if c.userID.Load() != owner {
|
||||
m.mu.Lock()
|
||||
delete(m.pending, key)
|
||||
m.deletePendingLocked(key)
|
||||
delete(m.flushing, key)
|
||||
m.mu.Unlock()
|
||||
releaseQueuedPushes(batch[i:])
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
err := c.Send(ctx, item.t, item.msg)
|
||||
// Pending entries are durable account updates. Shared body-budget pressure is not
|
||||
// evidence that this socket is corrupt, so use the non-terminal enqueue path; after
|
||||
// bounded retries, getDifference is the authoritative recovery path.
|
||||
err := c.SendBestEffortEncoded(ctx, item.t, item.encoded, 5*time.Second)
|
||||
cancel()
|
||||
if err == nil {
|
||||
item.release()
|
||||
continue
|
||||
}
|
||||
m.mu.Lock()
|
||||
if cur, ok := m.bySession[key]; !ok || cur != c || !m.flushing[key] || c.userID.Load() != owner {
|
||||
// 连接换代/取消/易主:剩余 batch 不属于当前连接当前账号,丢弃。
|
||||
if c.userID.Load() != owner {
|
||||
delete(m.pending, key)
|
||||
m.deletePendingLocked(key)
|
||||
delete(m.flushing, key)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
releaseQueuedPushes(batch[i:])
|
||||
return
|
||||
}
|
||||
rest := append(append([]queuedPush(nil), batch[i:]...), m.pending[key]...)
|
||||
if len(rest) > maxPendingPushesPerSession {
|
||||
// 与 queueLocked 溢出策略一致:丢最旧留最新,让 pts 空洞集中在最前端,
|
||||
// flush 首条即触发客户端 gap 检测,恢复路径最短。
|
||||
rest = rest[len(rest)-maxPendingPushesPerSession:]
|
||||
dropped := len(rest) - maxPendingPushesPerSession
|
||||
releaseQueuedPushes(rest[:dropped])
|
||||
rest = rest[dropped:]
|
||||
}
|
||||
m.pending[key] = rest
|
||||
if attempt+1 >= maxFlushAttempts {
|
||||
// 重试用尽:置位激活避免 idle 客户端永久断流;剩余暂存中的 durable 更新
|
||||
// 由客户端后续 pts 空洞触发 getDifference 补齐。
|
||||
c.receivesUpdates.Store(true)
|
||||
m.deletePendingLocked(key)
|
||||
delete(m.flushing, key)
|
||||
m.mu.Unlock()
|
||||
m.log.Debug("Flush gave up after retries; activated with getDifference fallback",
|
||||
|
|
@ -717,41 +883,61 @@ func (m *SessionManager) SetReceivesUpdatesForAuthKey(authKeyID [8]byte, session
|
|||
|
||||
// PushToSession 向指定 session 推送一条消息。
|
||||
func (m *SessionManager) PushToSession(ctx context.Context, sessionID int64, t proto.MessageType, msg bin.Encoder) error {
|
||||
m.mu.Lock()
|
||||
m.mu.RLock()
|
||||
c, key, ok, ambiguous := m.uniqueSessionLocked(sessionID)
|
||||
if ambiguous {
|
||||
m.mu.Unlock()
|
||||
m.mu.RUnlock()
|
||||
return ErrSessionAmbiguous
|
||||
}
|
||||
if !ok {
|
||||
m.mu.Unlock()
|
||||
m.mu.RUnlock()
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
if !c.receivesUpdates.Load() {
|
||||
m.queueLocked(key, t, msg)
|
||||
m.mu.Unlock()
|
||||
return nil
|
||||
ready := c.receivesUpdates.Load()
|
||||
m.mu.RUnlock()
|
||||
if ready {
|
||||
return c.Send(ctx, t, msg)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
return c.Send(ctx, t, msg)
|
||||
return m.queueOrSendPrepared(ctx, key, t, msg)
|
||||
}
|
||||
|
||||
// PushToSessionForAuthKey 向指定 raw auth_key_id + session_id 推送一条消息。
|
||||
func (m *SessionManager) PushToSessionForAuthKey(ctx context.Context, authKeyID [8]byte, sessionID int64, t proto.MessageType, msg bin.Encoder) error {
|
||||
m.mu.Lock()
|
||||
m.mu.RLock()
|
||||
key := sessionKey{authKeyID: authKeyID, sessionID: sessionID}
|
||||
c, ok := m.bySession[key]
|
||||
if !ok {
|
||||
m.mu.RUnlock()
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
ready := c.receivesUpdates.Load()
|
||||
m.mu.RUnlock()
|
||||
if ready {
|
||||
return c.Send(ctx, t, msg)
|
||||
}
|
||||
return m.queueOrSendPrepared(ctx, key, t, msg)
|
||||
}
|
||||
|
||||
func (m *SessionManager) queueOrSendPrepared(ctx context.Context, key sessionKey, t proto.MessageType, msg bin.Encoder) error {
|
||||
encoded, reservation, err := m.preparePendingPush(ctx, msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer reservation.release()
|
||||
|
||||
m.mu.Lock()
|
||||
c, ok := m.bySession[key]
|
||||
if !ok {
|
||||
m.mu.Unlock()
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
if !c.receivesUpdates.Load() {
|
||||
m.queueLocked(key, t, msg)
|
||||
_ = m.queuePreparedLocked(key, t, encoded, reservation)
|
||||
m.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
m.mu.Unlock()
|
||||
return c.Send(ctx, t, msg)
|
||||
return c.SendEncoded(ctx, t, encoded)
|
||||
}
|
||||
|
||||
// PushToSessionForAuthKeyImmediate 向指定 raw auth_key_id + session_id 立即推送一条消息。
|
||||
|
|
@ -782,7 +968,7 @@ func (m *SessionManager) PushToUserExceptSession(ctx context.Context, userID, ex
|
|||
return m.pushToUser(ctx, userID, nil, excludeSessionID, t, msg)
|
||||
}
|
||||
|
||||
// PushToUserExceptAuthKeySession 向某 user 所有活跃连接推送,跳过指定业务 auth_key + session。
|
||||
// PushToUserExceptAuthKeySession 向某 user 所有活跃连接推送,跳过指定 raw auth_key + session。
|
||||
func (m *SessionManager) PushToUserExceptAuthKeySession(ctx context.Context, userID int64, excludeAuthKeyID [8]byte, excludeSessionID int64, t proto.MessageType, msg bin.Encoder) (int, error) {
|
||||
return m.pushToUser(ctx, userID, &excludeAuthKeyID, excludeSessionID, t, msg)
|
||||
}
|
||||
|
|
@ -793,23 +979,28 @@ func (m *SessionManager) PushToUserExceptAuthKeySession(ctx context.Context, use
|
|||
// 漏 temp-key 设备)。未就绪连接跳过、不进 pending——密聊消息 durable 在 qts 队列,
|
||||
// 离线设备靠 getDifference 补回(在线推送只是加速器)。c.userID 复查防跨账号泄露。
|
||||
func (m *SessionManager) PushToUserAuthKey(ctx context.Context, userID int64, businessAuthKeyID [8]byte, t proto.MessageType, msg bin.Encoder) (int, error) {
|
||||
getEncoded := onceEncodedOutbound(msg)
|
||||
return m.pushToBusinessAuthKey(ctx, userID, businessAuthKeyID, false, func(c *Conn) error {
|
||||
if c.outbound == nil || c.outboundControl == nil {
|
||||
return ErrConnClosed
|
||||
}
|
||||
encoded, err := getEncoded()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.SendEncoded(ctx, t, encoded)
|
||||
})
|
||||
// Secret-chat qts is the durable source of truth, so online delivery is an accelerator just
|
||||
// like account pts fan-out. Do not synchronously wait for every PFS/raw connection's socket.
|
||||
return m.pushToBusinessAuthKeyBestEffort(ctx, userID, businessAuthKeyID, t, msg, 2*time.Second)
|
||||
}
|
||||
|
||||
// PushToUserAuthKeyTransient 是 PushToUserAuthKey 的 transient(typing)best-effort 版本。
|
||||
func (m *SessionManager) PushToUserAuthKeyTransient(ctx context.Context, userID int64, businessAuthKeyID [8]byte, t proto.MessageType, msg bin.Encoder, timeout time.Duration) (int, error) {
|
||||
getEncoded := onceEncodedOutbound(msg)
|
||||
return m.pushToBusinessAuthKey(ctx, userID, businessAuthKeyID, true, func(c *Conn) error {
|
||||
return m.pushToBusinessAuthKeyBestEffort(ctx, userID, businessAuthKeyID, t, msg, timeout)
|
||||
}
|
||||
|
||||
func (m *SessionManager) pushToBusinessAuthKeyBestEffort(ctx context.Context, userID int64, businessAuthKeyID [8]byte, t proto.MessageType, msg bin.Encoder, timeout time.Duration) (int, error) {
|
||||
getEncoded := onceEncodedOutbound(ctx, msg)
|
||||
var deadline time.Time
|
||||
if timeout > 0 {
|
||||
deadline = time.Now().Add(timeout)
|
||||
}
|
||||
if ctx != nil {
|
||||
if ctxDeadline, ok := ctx.Deadline(); ok && (deadline.IsZero() || ctxDeadline.Before(deadline)) {
|
||||
deadline = ctxDeadline
|
||||
}
|
||||
}
|
||||
return m.pushToBusinessAuthKey(ctx, userID, businessAuthKeyID, func(c *Conn) error {
|
||||
if c.outbound == nil || c.outboundControl == nil {
|
||||
return ErrConnClosed
|
||||
}
|
||||
|
|
@ -817,11 +1008,18 @@ func (m *SessionManager) PushToUserAuthKeyTransient(ctx context.Context, userID
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.SendBestEffortEncoded(ctx, t, encoded, timeout)
|
||||
remaining := timeout
|
||||
if !deadline.IsZero() {
|
||||
remaining = time.Until(deadline)
|
||||
if remaining < 0 {
|
||||
remaining = 0
|
||||
}
|
||||
}
|
||||
return c.SendBestEffortEncoded(ctx, t, encoded, remaining)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *SessionManager) pushToBusinessAuthKey(ctx context.Context, userID int64, businessAuthKeyID [8]byte, transient bool, send func(*Conn) error) (int, error) {
|
||||
func (m *SessionManager) pushToBusinessAuthKey(ctx context.Context, userID int64, businessAuthKeyID [8]byte, send func(*Conn) error) (int, error) {
|
||||
m.mu.Lock()
|
||||
candidates := m.businessAuthKeyCandidatesLocked(businessAuthKeyID)
|
||||
conns := make([]*Conn, 0, len(candidates))
|
||||
|
|
@ -836,7 +1034,6 @@ func (m *SessionManager) pushToBusinessAuthKey(ctx context.Context, userID int64
|
|||
conns = append(conns, c)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
_ = transient
|
||||
var firstErr error
|
||||
sent := 0
|
||||
for _, c := range conns {
|
||||
|
|
@ -845,6 +1042,18 @@ func (m *SessionManager) pushToBusinessAuthKey(ctx context.Context, userID int64
|
|||
continue
|
||||
}
|
||||
if err := send(c); err != nil {
|
||||
if errors.Is(err, ErrOutboundTrackedBudget) {
|
||||
// Shared process pressure is not evidence that this particular socket is
|
||||
// slow. Skip this online accelerator; durable qts/difference is the truth.
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, ErrOutboundQueueFull) {
|
||||
c.dropSlowConsumer()
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, ErrConnClosed) {
|
||||
continue
|
||||
}
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
|
|
@ -856,7 +1065,7 @@ func (m *SessionManager) pushToBusinessAuthKey(ctx context.Context, userID int64
|
|||
}
|
||||
|
||||
func (m *SessionManager) pushToUser(ctx context.Context, userID int64, excludeAuthKeyID *[8]byte, excludeSessionID int64, t proto.MessageType, msg bin.Encoder) (int, error) {
|
||||
getEncoded := onceEncodedOutbound(msg)
|
||||
getEncoded := onceEncodedOutbound(ctx, msg)
|
||||
return m.pushToUserWithSender(ctx, userID, excludeAuthKeyID, excludeSessionID, t, msg, true, func(c *Conn) error {
|
||||
if c.outbound == nil || c.outboundControl == nil {
|
||||
return ErrConnClosed
|
||||
|
|
@ -875,7 +1084,7 @@ func (m *SessionManager) pushToUser(ctx context.Context, userID int64, excludeAu
|
|||
// 下一次状态变化重建,囤积过期 transient 既无意义又会被 pending 的老化/溢出/重试耗尽误当
|
||||
// 「durable 兜底」丢弃。走 best-effort 发送,不阻塞调用方。
|
||||
func (m *SessionManager) PushToUserTransientExceptAuthKeySession(ctx context.Context, userID int64, excludeAuthKeyID [8]byte, excludeSessionID int64, t proto.MessageType, msg bin.Encoder, timeout time.Duration) (int, error) {
|
||||
getEncoded := onceEncodedOutbound(msg)
|
||||
getEncoded := onceEncodedOutbound(ctx, msg)
|
||||
return m.pushToUserWithSender(ctx, userID, &excludeAuthKeyID, excludeSessionID, t, msg, false, func(c *Conn) error {
|
||||
if c.outbound == nil || c.outboundControl == nil {
|
||||
return ErrConnClosed
|
||||
|
|
@ -897,7 +1106,19 @@ func (m *SessionManager) PushToUserExceptAuthKeySessionBestEffort(ctx context.Co
|
|||
}
|
||||
|
||||
func (m *SessionManager) pushToUserBestEffort(ctx context.Context, userID int64, excludeAuthKeyID *[8]byte, excludeSessionID int64, t proto.MessageType, msg bin.Encoder, timeout time.Duration) (int, error) {
|
||||
getEncoded := onceEncodedOutbound(msg)
|
||||
getEncoded := onceEncodedOutbound(ctx, msg)
|
||||
// timeout 是整次 fan-out 的等待预算,不是每个 session 各自一份。健康连接始终先走
|
||||
// SendBestEffortEncoded 的非阻塞快路径;预算耗尽后 remaining=0,仍会尝试快路径,
|
||||
// 但不会再为后续慢连接串行等待。
|
||||
var deadline time.Time
|
||||
if timeout > 0 {
|
||||
deadline = time.Now().Add(timeout)
|
||||
}
|
||||
if ctx != nil {
|
||||
if ctxDeadline, ok := ctx.Deadline(); ok && (deadline.IsZero() || ctxDeadline.Before(deadline)) {
|
||||
deadline = ctxDeadline
|
||||
}
|
||||
}
|
||||
return m.pushToUserWithSender(ctx, userID, excludeAuthKeyID, excludeSessionID, t, msg, true, func(c *Conn) error {
|
||||
if c.outbound == nil || c.outboundControl == nil {
|
||||
return ErrConnClosed
|
||||
|
|
@ -906,18 +1127,25 @@ func (m *SessionManager) pushToUserBestEffort(ctx context.Context, userID int64,
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.SendBestEffortEncoded(ctx, t, encoded, timeout)
|
||||
remaining := timeout
|
||||
if !deadline.IsZero() {
|
||||
remaining = time.Until(deadline)
|
||||
if remaining < 0 {
|
||||
remaining = 0
|
||||
}
|
||||
}
|
||||
return c.SendBestEffortEncoded(ctx, t, encoded, remaining)
|
||||
})
|
||||
}
|
||||
|
||||
func onceEncodedOutbound(msg bin.Encoder) func() (*encodedOutboundMessage, error) {
|
||||
func onceEncodedOutbound(ctx context.Context, msg bin.Encoder) func() (*encodedOutboundMessage, error) {
|
||||
var (
|
||||
encoded *encodedOutboundMessage
|
||||
err error
|
||||
)
|
||||
return func() (*encodedOutboundMessage, error) {
|
||||
if encoded == nil && err == nil {
|
||||
encoded, err = encodeOutboundMessage(msg)
|
||||
encoded, err = encodeOutboundMessageContext(ctx, msg)
|
||||
}
|
||||
return encoded, err
|
||||
}
|
||||
|
|
@ -957,6 +1185,10 @@ func (m *SessionManager) pushToUserWithSender(ctx context.Context, userID int64,
|
|||
}
|
||||
m.mu.RUnlock()
|
||||
if needQueue {
|
||||
// TL encoding and the process-wide pending-byte reservation may be expensive or
|
||||
// briefly block on the global encode gate. Do both before taking SessionManager.mu,
|
||||
// then share the immutable body across every not-ready session found by the re-scan.
|
||||
pendingEncoded, pendingReservation, pendingErr := m.preparePendingPush(ctx, msg)
|
||||
// 写锁下完整重扫(读锁释放到此之间状态可能变化,以重扫结果为准)。
|
||||
conns = conns[:0]
|
||||
queued, dropped, excluded, skipped = 0, 0, 0, 0
|
||||
|
|
@ -972,7 +1204,7 @@ func (m *SessionManager) pushToUserWithSender(ctx context.Context, userID int64,
|
|||
skipped++
|
||||
continue
|
||||
}
|
||||
if m.queueLocked(key, t, msg) {
|
||||
if pendingErr == nil && m.queuePreparedLocked(key, t, pendingEncoded, pendingReservation) {
|
||||
queued++
|
||||
if debug {
|
||||
m.log.Debug("Push queued (session not updates-ready)",
|
||||
|
|
@ -996,6 +1228,15 @@ func (m *SessionManager) pushToUserWithSender(ctx context.Context, userID int64,
|
|||
conns = append(conns, c)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
if pendingReservation != nil {
|
||||
pendingReservation.release() // drop producer ref; queued entries own the body now.
|
||||
}
|
||||
if pendingErr != nil && debug {
|
||||
m.log.Debug("Drop pending pushes outside byte budget",
|
||||
zap.Int64("user_id", userID),
|
||||
zap.Error(pendingErr),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
var firstErr error
|
||||
|
|
@ -1008,6 +1249,29 @@ func (m *SessionManager) pushToUserWithSender(ctx context.Context, userID int64,
|
|||
continue
|
||||
}
|
||||
if err := send(c); err != nil {
|
||||
if errors.Is(err, ErrOutboundTrackedBudget) {
|
||||
// Do not turn pressure owned by other sockets into a reconnect storm on
|
||||
// healthy recipients. The durable event remains recoverable by difference.
|
||||
dropped++
|
||||
continue
|
||||
}
|
||||
// 对 durable/best-effort fan-out,队列满意味着该 socket 已成为慢消费者。
|
||||
// 立即摘除并把它视为离线:不能让其错误把已经投递给健康 session 的 outbox
|
||||
// 行整体重试。该 session 的 durable gap 由 getDifference 恢复。
|
||||
if errors.Is(err, ErrOutboundQueueFull) {
|
||||
c.dropSlowConsumer()
|
||||
if debug {
|
||||
m.log.Debug("Drop slow outbound consumer",
|
||||
zap.Int64("user_id", userID),
|
||||
zap.String("auth_key_id", sessionKeyLog(c.authKeyID)),
|
||||
zap.Int64("session_id", c.sessionID),
|
||||
)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, ErrConnClosed) {
|
||||
continue
|
||||
}
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
|
|
@ -1055,6 +1319,24 @@ func (m *SessionManager) Online() int {
|
|||
return len(m.bySession)
|
||||
}
|
||||
|
||||
// ActiveRawAuthKeyIDs 返回当前物理连接实际使用的 raw auth_key_id 去重快照。
|
||||
// maintenance 用它保护“已建 key 但尚未登录”的长连接不被 orphan GC 删除;不能用
|
||||
// business/temp→perm key 替代,否则活跃 temp 连接仍可能误删。
|
||||
func (m *SessionManager) ActiveRawAuthKeyIDs() [][8]byte {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
seen := make(map[[8]byte]struct{}, len(m.bySession))
|
||||
out := make([][8]byte, 0, len(m.byAuthKey))
|
||||
for key := range m.bySession {
|
||||
if _, ok := seen[key.authKeyID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key.authKeyID] = struct{}{}
|
||||
out = append(out, key.authKeyID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// IsUserOnline returns whether userID has at least one active connection.
|
||||
func (m *SessionManager) IsUserOnline(userID int64) bool {
|
||||
if userID == 0 {
|
||||
|
|
@ -1268,6 +1550,54 @@ func (m *SessionManager) OnlineChannelMemberUserIDsExcluding(channelID int64, ex
|
|||
return out
|
||||
}
|
||||
|
||||
// OnlineChannelIDsSnapshot returns every channel with at least one live joined-member session in
|
||||
// strictly ascending order. The global SessionManager lock is held only while copying map keys;
|
||||
// sorting and all recovery database work happen after unlock. The fixed saturation-recovery actor
|
||||
// is the sole caller, so its exceptional-path temporary memory is one int64 slice (peak about 8*C
|
||||
// bytes) rather than repeated O(C) scans under the connection/membership lock.
|
||||
func (m *SessionManager) OnlineChannelIDsSnapshot() []int64 {
|
||||
m.mu.RLock()
|
||||
out := make([]int64, 0, len(m.byMemberChannel))
|
||||
for channelID, sessions := range m.byMemberChannel {
|
||||
if channelID <= 0 || len(sessions) == 0 {
|
||||
continue
|
||||
}
|
||||
live := false
|
||||
for key := range sessions {
|
||||
if _, ok := m.bySession[key]; ok {
|
||||
live = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !live {
|
||||
continue
|
||||
}
|
||||
out = append(out, channelID)
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
|
||||
return out
|
||||
}
|
||||
|
||||
// OnlineChannelIDsAfter is retained for bounded diagnostics/tests. Production recovery takes one
|
||||
// OnlineChannelIDsSnapshot per generation and slices it into pages, avoiding repeated full scans.
|
||||
func (m *SessionManager) OnlineChannelIDsAfter(afterChannelID int64, limit int) []int64 {
|
||||
if limit <= 0 {
|
||||
return nil
|
||||
}
|
||||
const maxRecoveryPage = 4096
|
||||
if limit > maxRecoveryPage {
|
||||
limit = maxRecoveryPage
|
||||
}
|
||||
all := m.OnlineChannelIDsSnapshot()
|
||||
start := sort.Search(len(all), func(i int) bool { return all[i] > afterChannelID })
|
||||
end := start + limit
|
||||
if end > len(all) {
|
||||
end = len(all)
|
||||
}
|
||||
return all[start:end]
|
||||
}
|
||||
|
||||
func (m *SessionManager) onlineChannelUsers(index map[int64]map[sessionKey]int64, channelID int64, limit int) []int64 {
|
||||
if channelID == 0 {
|
||||
return nil
|
||||
|
|
@ -1314,7 +1644,7 @@ func (m *SessionManager) removeLocked(c *Conn, dropPending bool) int64 {
|
|||
m.clearChannelInterestsLocked(key)
|
||||
m.clearChannelMembershipsLocked(c, key)
|
||||
if dropPending {
|
||||
delete(m.pending, key)
|
||||
m.deletePendingLocked(key)
|
||||
}
|
||||
delete(m.flushing, key)
|
||||
return uid
|
||||
|
|
@ -1430,8 +1760,11 @@ func (m *SessionManager) takePendingLocked(key sessionKey, ready bool) []queuedP
|
|||
now := time.Now()
|
||||
pending := make([]queuedPush, 0, len(q))
|
||||
dropped := 0
|
||||
for _, item := range q {
|
||||
for i := range q {
|
||||
item := q[i]
|
||||
q[i] = queuedPush{}
|
||||
if now.Sub(item.at) > pendingPushMaxAge {
|
||||
item.release()
|
||||
dropped++
|
||||
continue
|
||||
}
|
||||
|
|
@ -1447,9 +1780,43 @@ func (m *SessionManager) takePendingLocked(key sessionKey, ready bool) []queuedP
|
|||
return pending
|
||||
}
|
||||
|
||||
// queueLocked 暂存一条主动推送,返回是否实际入队——stale 丢批分支会连同当前
|
||||
// 这条一起丢弃,调用方据此区分 queued/dropped 计数,避免投递日志失真。
|
||||
func (m *SessionManager) queueLocked(key sessionKey, t proto.MessageType, msg bin.Encoder) bool {
|
||||
// preparePendingPush encodes outside SessionManager.mu and reserves the one physical body before
|
||||
// releasing the process-wide encode slot. Multiple not-ready sessions may then share this
|
||||
// immutable body via reservation refs instead of encoding/copying it once per session.
|
||||
func (m *SessionManager) preparePendingPush(ctx context.Context, msg bin.Encoder) (*encodedOutboundMessage, *pendingPushReservation, error) {
|
||||
var (
|
||||
encoded *encodedOutboundMessage
|
||||
bytes int
|
||||
)
|
||||
err := withOutboundEncodeSlot(ctx, nil, func() error {
|
||||
var err error
|
||||
encoded, err = encodeOutboundMessageWithoutSlot(msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if encoded == nil {
|
||||
return errors.New("nil encoded pending push")
|
||||
}
|
||||
bytes = len(encoded.body)
|
||||
if bytes > maxOutboundBodyBytes {
|
||||
return fmt.Errorf("%w: body=%d limit=%d", ErrOutboundMessageTooLarge, bytes, maxOutboundBodyBytes)
|
||||
}
|
||||
if !m.pendingBudget.reserve(bytes) {
|
||||
return ErrOutboundTrackedBudget
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
reservation := &pendingPushReservation{budget: m.pendingBudget, bytes: bytes}
|
||||
reservation.refs.Store(1) // producer ownership; queue entries retain below.
|
||||
return encoded, reservation, nil
|
||||
}
|
||||
|
||||
// queuePreparedLocked 暂存一条已编码的主动推送,返回是否实际入队。
|
||||
// 调用方必须在锁外保持 reservation 的 producer ref,并在全部入队完成后 release。
|
||||
func (m *SessionManager) queuePreparedLocked(key sessionKey, t proto.MessageType, encoded *encodedOutboundMessage, reservation *pendingPushReservation) bool {
|
||||
q := m.pending[key]
|
||||
// 过期保护:最早一条暂存已超过 pendingPushMaxAge(session 迟迟未 ready)时,丢整批并
|
||||
// 不再囤这条,记 trace。避免「登录后从不 getState」的连接长期占用 pending 内存。
|
||||
|
|
@ -1459,11 +1826,21 @@ func (m *SessionManager) queueLocked(key sessionKey, t proto.MessageType, msg bi
|
|||
zap.Int64("session_id", key.sessionID),
|
||||
zap.Int("dropped", len(q)),
|
||||
)
|
||||
delete(m.pending, key)
|
||||
m.deletePendingLocked(key)
|
||||
return false
|
||||
}
|
||||
push := queuedPush{t: t, msg: msg, at: time.Now()}
|
||||
if encoded == nil || reservation == nil {
|
||||
return false
|
||||
}
|
||||
reservation.retain()
|
||||
push := queuedPush{
|
||||
t: t,
|
||||
encoded: encoded,
|
||||
reservation: reservation,
|
||||
at: time.Now(),
|
||||
}
|
||||
if len(q) >= maxPendingPushesPerSession {
|
||||
q[0].release()
|
||||
copy(q, q[1:])
|
||||
q[len(q)-1] = push
|
||||
m.pending[key] = q
|
||||
|
|
@ -1473,6 +1850,22 @@ func (m *SessionManager) queueLocked(key sessionKey, t proto.MessageType, msg bi
|
|||
return true
|
||||
}
|
||||
|
||||
// queueLocked remains as a test/internal single-target convenience. Production fan-out prepares
|
||||
// outside m.mu and calls queuePreparedLocked so TL encoding never serializes the session registry.
|
||||
func (m *SessionManager) queueLocked(key sessionKey, t proto.MessageType, msg bin.Encoder) bool {
|
||||
encoded, reservation, err := m.preparePendingPush(context.Background(), msg)
|
||||
if err != nil {
|
||||
m.log.Debug("Drop pending push outside byte budget",
|
||||
zap.String("auth_key_id", sessionKeyLog(key.authKeyID)),
|
||||
zap.Int64("session_id", key.sessionID),
|
||||
zap.Error(err),
|
||||
)
|
||||
return false
|
||||
}
|
||||
defer reservation.release()
|
||||
return m.queuePreparedLocked(key, t, encoded, reservation)
|
||||
}
|
||||
|
||||
func (m *SessionManager) uniqueSessionLocked(sessionID int64) (*Conn, sessionKey, bool, bool) {
|
||||
set := m.bySessionID[sessionID]
|
||||
if len(set) == 0 {
|
||||
|
|
@ -1490,7 +1883,7 @@ func (m *SessionManager) uniqueSessionLocked(sessionID int64) (*Conn, sessionKey
|
|||
func (m *SessionManager) dropPendingBySessionLocked(sessionID int64) {
|
||||
for key := range m.pending {
|
||||
if key.sessionID == sessionID {
|
||||
delete(m.pending, key)
|
||||
m.deletePendingLocked(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1528,7 +1921,7 @@ func (m *SessionManager) sweepStalePending() {
|
|||
if len(q) == 0 || now.Sub(q[0].at) <= pendingPushMaxAge {
|
||||
continue
|
||||
}
|
||||
delete(m.pending, key)
|
||||
m.deletePendingLocked(key)
|
||||
dropped++
|
||||
}
|
||||
if dropped > 0 {
|
||||
|
|
@ -1536,6 +1929,27 @@ func (m *SessionManager) sweepStalePending() {
|
|||
}
|
||||
}
|
||||
|
||||
func (q *queuedPush) release() {
|
||||
if q == nil {
|
||||
return
|
||||
}
|
||||
reservation := q.reservation
|
||||
*q = queuedPush{}
|
||||
reservation.release()
|
||||
}
|
||||
|
||||
func releaseQueuedPushes(q []queuedPush) {
|
||||
for i := range q {
|
||||
q[i].release()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *SessionManager) deletePendingLocked(key sessionKey) {
|
||||
q := m.pending[key]
|
||||
delete(m.pending, key)
|
||||
releaseQueuedPushes(q)
|
||||
}
|
||||
|
||||
func addConnIndex[K comparable](idx map[K]map[int64]*Conn, key K, sessionID int64, c *Conn) {
|
||||
set := idx[key]
|
||||
if set == nil {
|
||||
|
|
@ -1630,7 +2044,7 @@ func shouldExcludeSession(c *Conn, excludeAuthKeyID *[8]byte, excludeSessionID i
|
|||
if excludeAuthKeyID == nil || *excludeAuthKeyID == ([8]byte{}) {
|
||||
return true
|
||||
}
|
||||
return connUsesBusinessAuthKey(c, *excludeAuthKeyID)
|
||||
return c.authKeyID == *excludeAuthKeyID
|
||||
}
|
||||
|
||||
func sessionKeyLog(id [8]byte) string {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue