fix: sync protocol and discussion stability fixes
This commit is contained in:
parent
9f73dc20da
commit
aa21bd04e1
43 changed files with 7258 additions and 503 deletions
|
|
@ -40,6 +40,11 @@ const (
|
|||
defaultOutboundControlQueueSize = 32
|
||||
defaultOutboundTrackedMaxBytes = int64(512 << 20) // 512 MiB / Server
|
||||
defaultOutboundControlMaxBytes = int64(64 << 20) // ack/state/resend vectors / Server
|
||||
// requiredControlMaxWait bounds protocol barriers such as new_session_created from
|
||||
// the beginning of encoding through the completed physical write. These frames gate
|
||||
// subsequent session state transitions, so timing out must close the connection instead
|
||||
// of degrading to the best-effort control path.
|
||||
requiredControlMaxWait = 5 * time.Second
|
||||
|
||||
maxTrackedServerMsgIDs = 4096
|
||||
maxTrackedAckedMsgIDs = 1024
|
||||
|
|
@ -332,6 +337,9 @@ func (c *Conn) Close() {
|
|||
// transport.Close call, so a timed-out batch close cannot keep accepting memory/work.
|
||||
func (c *Conn) beginTerminalShutdown() {
|
||||
c.terminal.Store(true)
|
||||
// Retirement is irreversible. SessionManager activation only uses CAS from
|
||||
// provisional/claiming, so a stale goroutine cannot publish this Conn again.
|
||||
c.lifecycle.Store(uint32(connLifecycleRetired))
|
||||
c.signalOutboundStop()
|
||||
c.beginCloseInboundRPCScheduler()
|
||||
}
|
||||
|
|
@ -342,6 +350,23 @@ func (c *Conn) waitOutboundShutdown() {
|
|||
}
|
||||
}
|
||||
|
||||
func (c *Conn) waitOutboundShutdownUntil(timeout time.Duration) bool {
|
||||
if c == nil || c.outboundDone == nil {
|
||||
return true
|
||||
}
|
||||
if timeout <= 0 {
|
||||
return false
|
||||
}
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-c.outboundDone:
|
||||
return true
|
||||
case <-timer.C:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// ForceClose 停止连接并关闭底层 transport。
|
||||
// 仅用于授权撤销 / destroy_auth_key 这类“必须让对端立即断线”的路径;普通生命周期仍由
|
||||
// serveConn 统一关闭 transport,避免正常 push/索引清理把长连接误伤成硬断。
|
||||
|
|
@ -355,6 +380,13 @@ func (c *Conn) ForceClose() {
|
|||
// closeTransport 只关闭物理 transport,不等待 outbound actor。写失败路径运行在
|
||||
// actor 自身 goroutine 中,若在这里调用 Close 会等待 outboundDone 而自锁。
|
||||
func (c *Conn) closeTransport() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
if c.transportLease != nil {
|
||||
_ = c.transportLease.Close()
|
||||
return
|
||||
}
|
||||
c.transportClose.Do(func() {
|
||||
if c.transport != nil {
|
||||
_ = c.transport.Close()
|
||||
|
|
@ -372,6 +404,38 @@ func (c *Conn) failTransport() {
|
|||
c.closeTransport()
|
||||
}
|
||||
|
||||
// fenceUndeliveredRPCResult is the no-reentry terminal path used from a task's
|
||||
// release callback. That callback may itself run while rpcClose.Do is draining
|
||||
// queued tasks, so calling beginCloseInboundRPCScheduler again would deadlock on
|
||||
// sync.Once. Closing the socket wakes serveConn, whose ordinary defer completes
|
||||
// scheduler/index cleanup; when shutdown already owns the callback, that cleanup
|
||||
// is already in progress.
|
||||
func (c *Conn) fenceUndeliveredRPCResult() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
// A replacement/shutdown that already published terminal owns physical
|
||||
// lifecycle cleanup (and may intentionally transfer the lease). Only the
|
||||
// resultless task that wins false->true is allowed to close this generation.
|
||||
if !c.terminal.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
c.lifecycle.Store(uint32(connLifecycleRetired))
|
||||
c.signalOutboundStop()
|
||||
if c.transportLease != nil {
|
||||
c.transportLease.startCloseAlreadyFenced()
|
||||
return
|
||||
}
|
||||
// Legacy construction-only Conns have no owner callback graph, so their
|
||||
// exact transport close cannot re-enter logical lifecycle cleanup. Keep a
|
||||
// pathological Close outside the shared RPC worker just like the lease path.
|
||||
go c.transportClose.Do(func() {
|
||||
if c.transport != nil {
|
||||
_ = c.transport.Close()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// dropSlowConsumer 把出站队列持续拥塞的连接降级为离线连接。它不能等待 outbound
|
||||
// actor:调用方位于 fan-out 热路径,等待单个慢 socket 会把同一用户的健康设备和
|
||||
// transactional outbox lane 一起拖住。关闭 transport 会打断可能阻塞的写;serveConn
|
||||
|
|
@ -402,6 +466,41 @@ func (c *Conn) SendPriority(ctx context.Context, t proto.MessageType, msg bin.En
|
|||
return c.send(ctx, t, msg, true)
|
||||
}
|
||||
|
||||
// SendRequiredControl writes a protocol-critical control message before the caller commits
|
||||
// the state transition guarded by that message. One absolute deadline covers encode admission,
|
||||
// body-budget reservation, control-queue admission and the physical transport write. A failure
|
||||
// is terminal: continuing on the same connection could expose state whose required notification
|
||||
// never reached the client.
|
||||
//
|
||||
// Success only confirms the physical write; it does not wait for the client's msgs_ack.
|
||||
func (c *Conn) SendRequiredControl(ctx context.Context, t proto.MessageType, msg bin.Encoder) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
now := time.Now()
|
||||
deadline := now.Add(requiredControlMaxWait)
|
||||
if c.writeTimeout > 0 {
|
||||
if writeDeadline := now.Add(c.writeTimeout); writeDeadline.Before(deadline) {
|
||||
deadline = writeDeadline
|
||||
}
|
||||
}
|
||||
if parentDeadline, ok := ctx.Deadline(); ok && parentDeadline.Before(deadline) {
|
||||
deadline = parentDeadline
|
||||
}
|
||||
requiredCtx, cancel := context.WithDeadline(ctx, deadline)
|
||||
defer cancel()
|
||||
if err := requiredCtx.Err(); err != nil {
|
||||
c.failTransport()
|
||||
return err
|
||||
}
|
||||
|
||||
err := c.sendOutbound(requiredCtx, t, msg, nil, true)
|
||||
if err != nil {
|
||||
c.failTransport()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// SendBestEffort 只等待消息进入普通 outbound 队列,不等待网络写完成。
|
||||
// 用于 updates fanout:队列拥塞时返回 ErrOutboundQueueFull,durable outbox/getDifference 负责兜底。
|
||||
func (c *Conn) SendBestEffort(ctx context.Context, t proto.MessageType, msg bin.Encoder, timeout time.Duration) error {
|
||||
|
|
@ -519,14 +618,27 @@ func (c *Conn) sendOutbound(ctx context.Context, t proto.MessageType, msg bin.En
|
|||
case res := <-op.done:
|
||||
return res.err
|
||||
case <-ctx.Done():
|
||||
// A physical write can complete at the same instant as the caller's
|
||||
// deadline. Prefer the actor's terminal result when it is already
|
||||
// available so required-control callers do not poison a healthy Conn.
|
||||
select {
|
||||
case res := <-op.done:
|
||||
return res.err
|
||||
default:
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-c.outboundStop:
|
||||
select {
|
||||
case res := <-op.done:
|
||||
return res.err
|
||||
default:
|
||||
}
|
||||
return ErrConnClosed
|
||||
}
|
||||
}
|
||||
|
||||
// SendAsync 入队一条 server 消息但不等待发送结果(fire-and-forget),用于读循环里的控制消息
|
||||
// (ack/pong/new_session_created/bad_msg/future_salts/state_info):避免读循环被 outbound 写
|
||||
// (ack/pong/bad_msg/future_salts/state_info):避免读循环被 outbound 写
|
||||
// 阻塞而连带卡死。走优先(control)队列保证不被普通 push 拖后;队列满时丢弃并记 metrics——此时
|
||||
// 连接多已严重拥塞,控制消息丢失由客户端重传 / 读写超时兜底。返回非 nil 仅表示连接已关闭。
|
||||
func (c *Conn) SendAsync(ctx context.Context, t proto.MessageType, msg bin.Encoder) error {
|
||||
|
|
@ -1038,7 +1150,10 @@ func (c *Conn) failOutboundBudget(err error) {
|
|||
if c.metrics != nil {
|
||||
c.metrics.OutboundDropped("tracked_global_byte_budget")
|
||||
}
|
||||
c.failTransport()
|
||||
// No socket bytes exist yet. If an intentional session handoff already won
|
||||
// the terminal CAS, it owns close/transfer and this old producer must not close
|
||||
// the still-current lease. A live connection still gets fenced and closed.
|
||||
c.fenceUndeliveredRPCResult()
|
||||
}
|
||||
|
||||
func (c *Conn) ensureOutboundTrackedBudget() *outboundTrackedBudget {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue