fix: sync protocol and discussion stability fixes
This commit is contained in:
A
parent
9f73dc20da
commit
aa21bd04e1
43 changed files with 7258 additions and 503 deletions
|
|
@ -30,8 +30,13 @@ type inboundRPC struct {
|
|||
size int
|
||||
run func(context.Context) error
|
||||
onTimeout func()
|
||||
budget *inboundRPCGlobalReservation
|
||||
ticket *inboundRPCTicket
|
||||
// release drops request-scoped ownership which is independent from the body
|
||||
// budget (for example a cross-connection in-flight RPC claim). It runs once
|
||||
// after the request has either published a terminal result or become
|
||||
// impossible to run; callers should still make the callback idempotent.
|
||||
release func()
|
||||
budget *inboundRPCGlobalReservation
|
||||
ticket *inboundRPCTicket
|
||||
}
|
||||
|
||||
const (
|
||||
|
|
@ -76,7 +81,7 @@ type inboundRPCScheduler struct {
|
|||
type inboundRPCGlobalReservation struct {
|
||||
scheduler *inboundRPCScheduler
|
||||
size int64
|
||||
once sync.Once
|
||||
released atomic.Bool
|
||||
}
|
||||
|
||||
// inboundRPCReservation 同时持有全局和单连接的“Copy 前”预算。commit/abort 只能成功一次；
|
||||
|
|
@ -92,6 +97,32 @@ type inboundRPCReservation struct {
|
|||
once sync.Once
|
||||
}
|
||||
|
||||
// inboundRPCSpec 是 container preflight 与 RPC scheduler 之间的有界 admission 描述。
|
||||
// method 仅用于 metrics，size 是在 Copy 之前必须预留的 request body 字节数。
|
||||
type inboundRPCSpec struct {
|
||||
method string
|
||||
size int
|
||||
}
|
||||
|
||||
type inboundRPCBatchEntry struct {
|
||||
global *inboundRPCGlobalReservation
|
||||
method string
|
||||
size int
|
||||
}
|
||||
|
||||
// inboundRPCBatchReservation 把一个 container 内的 RPC 视为一个 admission 单元。
|
||||
// reserve 一次预留整批的全局/单连接条数和字节预算；commit 一次 append
|
||||
// 全部任务；abort 一次归还全部预算。这防止 container 只执行前半批。
|
||||
type inboundRPCBatchReservation struct {
|
||||
conn *Conn
|
||||
ctx context.Context
|
||||
entries []inboundRPCBatchEntry
|
||||
totalSize int64
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
var errInboundRPCBatchTaskCount = errors.New("inbound rpc batch task count mismatch")
|
||||
|
||||
func newInboundRPCScheduler(workers, maxTasks int, maxBytes int64) *inboundRPCScheduler {
|
||||
if workers <= 0 {
|
||||
workers = 1
|
||||
|
|
@ -196,17 +227,89 @@ func (s *inboundRPCScheduler) reserveGlobal(size int) (*inboundRPCGlobalReservat
|
|||
return &inboundRPCGlobalReservation{scheduler: s, size: size64}, "", nil
|
||||
}
|
||||
|
||||
// reserveGlobalBatch 在一次 budgetMu 临界区内检查并预留整批条数/字节。
|
||||
// 返回的每个 reservation 仍由对应 task 单独归还，避免一个慢 RPC 持有
|
||||
// 整个 container 已完成任务的预算。
|
||||
func (s *inboundRPCScheduler) reserveGlobalBatch(sizes []int) ([]*inboundRPCGlobalReservation, string, error) {
|
||||
reservations := make([]*inboundRPCGlobalReservation, len(sizes))
|
||||
s.budgetMu.Lock()
|
||||
defer s.budgetMu.Unlock()
|
||||
|
||||
select {
|
||||
case <-s.stopCh:
|
||||
return nil, "scheduler_closed", ErrConnClosed
|
||||
default:
|
||||
}
|
||||
if len(sizes) > s.maxTasks-s.tasks {
|
||||
return nil, "global_task_budget", ErrInboundRPCQueueFull
|
||||
}
|
||||
// 逐项从剩余预算中减，避免 total 和 s.bytes+total 溢出。
|
||||
remaining := s.maxBytes - s.bytes
|
||||
var total int64
|
||||
for i, size := range sizes {
|
||||
if size < 0 {
|
||||
size = 0
|
||||
}
|
||||
size64 := int64(size)
|
||||
if size64 > remaining-total {
|
||||
return nil, "global_byte_budget", ErrInboundRPCQueueFull
|
||||
}
|
||||
total += size64
|
||||
reservations[i] = &inboundRPCGlobalReservation{scheduler: s, size: size64}
|
||||
}
|
||||
s.tasks += len(sizes)
|
||||
s.bytes += total
|
||||
return reservations, "", nil
|
||||
}
|
||||
|
||||
func (r *inboundRPCGlobalReservation) release() {
|
||||
if r == nil || r.scheduler == nil {
|
||||
return
|
||||
}
|
||||
r.once.Do(func() {
|
||||
s := r.scheduler
|
||||
s.budgetMu.Lock()
|
||||
s.tasks--
|
||||
s.bytes -= r.size
|
||||
s.budgetMu.Unlock()
|
||||
})
|
||||
if !r.released.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
s := r.scheduler
|
||||
s.budgetMu.Lock()
|
||||
s.tasks--
|
||||
s.bytes -= r.size
|
||||
s.budgetMu.Unlock()
|
||||
}
|
||||
|
||||
// releaseInboundRPCGlobalBatch 使 batch abort/commit-failure 在一次全局锁内
|
||||
// 归还它仍持有的全部预算。每张 ticket 的 CAS 保证与任何并发释放幂等。
|
||||
func releaseInboundRPCGlobalBatch(reservations []*inboundRPCGlobalReservation) {
|
||||
var (
|
||||
scheduler *inboundRPCScheduler
|
||||
tasks int
|
||||
bytes int64
|
||||
)
|
||||
for _, reservation := range reservations {
|
||||
if reservation == nil || reservation.scheduler == nil || !reservation.released.CompareAndSwap(false, true) {
|
||||
continue
|
||||
}
|
||||
if scheduler == nil {
|
||||
scheduler = reservation.scheduler
|
||||
}
|
||||
// 一个 batch 只能由同一 scheduler 创建。若未来出现混合调用，
|
||||
// 仍通过单张 release 正确归还，而不会扣错 scheduler。
|
||||
if reservation.scheduler != scheduler {
|
||||
reservation.scheduler.budgetMu.Lock()
|
||||
reservation.scheduler.tasks--
|
||||
reservation.scheduler.bytes -= reservation.size
|
||||
reservation.scheduler.budgetMu.Unlock()
|
||||
continue
|
||||
}
|
||||
tasks++
|
||||
bytes += reservation.size
|
||||
}
|
||||
if scheduler == nil || tasks == 0 {
|
||||
return
|
||||
}
|
||||
scheduler.budgetMu.Lock()
|
||||
scheduler.tasks -= tasks
|
||||
scheduler.bytes -= bytes
|
||||
scheduler.budgetMu.Unlock()
|
||||
}
|
||||
|
||||
func (s *inboundRPCScheduler) budgetSnapshot() (tasks int, bytes int64) {
|
||||
|
|
@ -363,6 +466,10 @@ func (c *Conn) reserveInboundRPC(ctx context.Context, method string, size int) (
|
|||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
if c.terminal.Load() {
|
||||
c.metrics.InboundRPCDropped(method, "scheduler_closed")
|
||||
return nil, ErrConnClosed
|
||||
}
|
||||
if c.rpcScheduler == nil {
|
||||
c.metrics.InboundRPCDropped(method, "scheduler_closed")
|
||||
return nil, ErrConnClosed
|
||||
|
|
@ -392,7 +499,7 @@ func (c *Conn) reserveInboundRPC(ctx context.Context, method string, size int) (
|
|||
c.metrics.InboundRPCDropped(method, "context_done")
|
||||
return nil, err
|
||||
}
|
||||
if c.rpcClosed {
|
||||
if c.rpcClosed || c.terminal.Load() {
|
||||
c.rpcMu.Unlock()
|
||||
global.release()
|
||||
c.metrics.InboundRPCDropped(method, "scheduler_closed")
|
||||
|
|
@ -427,6 +534,102 @@ func (c *Conn) reserveInboundRPC(ctx context.Context, method string, size int) (
|
|||
}, nil
|
||||
}
|
||||
|
||||
// reserveInboundRPCBatch 必须在 container 内任何 request body Copy 前调用。
|
||||
// 全局预算只锁一次，单连接预算也只锁一次；任一限制不满足时
|
||||
// 整批失败，不会留下部分 task/字节 reservation。
|
||||
func (c *Conn) reserveInboundRPCBatch(ctx context.Context, specs []inboundRPCSpec) (*inboundRPCBatchReservation, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
c.dropInboundRPCSpecs(specs, "context_done")
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
if c.terminal.Load() {
|
||||
c.dropInboundRPCSpecs(specs, "scheduler_closed")
|
||||
return nil, ErrConnClosed
|
||||
}
|
||||
if c.rpcScheduler == nil {
|
||||
c.dropInboundRPCSpecs(specs, "scheduler_closed")
|
||||
return nil, ErrConnClosed
|
||||
}
|
||||
|
||||
normalized := make([]inboundRPCSpec, len(specs))
|
||||
sizes := make([]int, len(specs))
|
||||
for i, spec := range specs {
|
||||
if spec.size < 0 {
|
||||
spec.size = 0
|
||||
}
|
||||
normalized[i] = spec
|
||||
sizes[i] = spec.size
|
||||
}
|
||||
globals, reason, err := c.rpcScheduler.reserveGlobalBatch(sizes)
|
||||
if err != nil {
|
||||
c.dropInboundRPCSpecs(normalized, reason)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entries := make([]inboundRPCBatchEntry, len(normalized))
|
||||
var totalSize int64
|
||||
for i, spec := range normalized {
|
||||
entries[i] = inboundRPCBatchEntry{
|
||||
global: globals[i],
|
||||
method: spec.method,
|
||||
size: spec.size,
|
||||
}
|
||||
totalSize += int64(spec.size)
|
||||
}
|
||||
|
||||
c.rpcMu.Lock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
c.rpcMu.Unlock()
|
||||
releaseInboundRPCGlobalBatch(globals)
|
||||
c.dropInboundRPCSpecs(normalized, "context_done")
|
||||
return nil, err
|
||||
}
|
||||
if c.rpcClosed || c.terminal.Load() {
|
||||
c.rpcMu.Unlock()
|
||||
releaseInboundRPCGlobalBatch(globals)
|
||||
c.dropInboundRPCSpecs(normalized, "scheduler_closed")
|
||||
return nil, ErrConnClosed
|
||||
}
|
||||
// 用减法比较，避免对抗性 batch 的 len 加法溢出。
|
||||
availableSlots := c.rpcQueueSize - c.rpcReserved - len(c.rpcQueue)
|
||||
if len(entries) > availableSlots {
|
||||
c.rpcMu.Unlock()
|
||||
releaseInboundRPCGlobalBatch(globals)
|
||||
c.dropInboundRPCSpecs(normalized, "queue_full")
|
||||
return nil, ErrInboundRPCQueueFull
|
||||
}
|
||||
if totalSize > maxInflightRPCBytes-c.inflightRPCBytes.Load() {
|
||||
c.rpcMu.Unlock()
|
||||
releaseInboundRPCGlobalBatch(globals)
|
||||
c.dropInboundRPCSpecs(normalized, "byte_budget")
|
||||
return nil, ErrInboundRPCQueueFull
|
||||
}
|
||||
c.rpcReserved += len(entries)
|
||||
c.inflightRPCBytes.Add(totalSize)
|
||||
// Add 与 close 的 Wait 由 rpcMu 排序：close 置 rpcClosed 后不会再发生 Add。
|
||||
// 整批 reservation 只需一个 waiter；commit/abort 也只会完成一次。
|
||||
c.rpcReservationWG.Add(1)
|
||||
c.rpcMu.Unlock()
|
||||
|
||||
return &inboundRPCBatchReservation{
|
||||
conn: c,
|
||||
ctx: ctx,
|
||||
entries: entries,
|
||||
totalSize: totalSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Conn) dropInboundRPCSpecs(specs []inboundRPCSpec, reason string) {
|
||||
for _, spec := range specs {
|
||||
c.metrics.InboundRPCDropped(spec.method, reason)
|
||||
}
|
||||
}
|
||||
|
||||
// enqueueInboundRPC 是测试和已持有独立 body 的便捷入口。生产收包路径使用
|
||||
// reserveInboundRPC -> Copy -> commit，保证真正的 Copy 前预算。
|
||||
func (c *Conn) enqueueInboundRPC(ctx context.Context, task inboundRPC) error {
|
||||
|
|
@ -450,7 +653,7 @@ func (r *inboundRPCReservation) commit(task inboundRPC) error {
|
|||
c := r.conn
|
||||
c.rpcMu.Lock()
|
||||
c.rpcReserved--
|
||||
if c.rpcClosed {
|
||||
if c.rpcClosed || c.terminal.Load() {
|
||||
c.inflightRPCBytes.Add(-int64(r.size))
|
||||
} else {
|
||||
// The request deadline starts when admission succeeds, not when a worker
|
||||
|
|
@ -525,6 +728,143 @@ func (r *inboundRPCReservation) abort() {
|
|||
})
|
||||
}
|
||||
|
||||
// commit 在一次 rpcMu 临界区内把整批 task append 到队列。
|
||||
// deferSchedule=false 保持旧的立即调度语义；true 则返回一个幂等 activate
|
||||
// 函数，让调用方先完成 new_session_created 等协议 barrier 再启动 worker。
|
||||
//
|
||||
// 延迟调度只能延迟本次 commit 新产生的 ready token；调用方应在连接的
|
||||
// 首个 admission batch 使用它，不得把它当作已有 worker 的全局暂停锁。
|
||||
func (r *inboundRPCBatchReservation) commit(tasks []inboundRPC, deferSchedule bool) (activate func(), result error) {
|
||||
if r == nil {
|
||||
return nil, ErrConnClosed
|
||||
}
|
||||
result = ErrConnClosed
|
||||
var (
|
||||
committed bool
|
||||
reschedule bool
|
||||
firstQueueLen int
|
||||
queueCap int
|
||||
globals []*inboundRPCGlobalReservation
|
||||
)
|
||||
r.once.Do(func() {
|
||||
c := r.conn
|
||||
// A container reservation may deliberately span protocol-critical barriers
|
||||
// (session ownership claim and new_session_created's physical write). Queue
|
||||
// and execution latency starts only when the fully admitted batch becomes
|
||||
// runnable, not while it is waiting behind those independent barriers.
|
||||
enqueuedAt := time.Now()
|
||||
deadline := time.Time{}
|
||||
if c.rpcTimeout > 0 {
|
||||
deadline = enqueuedAt.Add(c.rpcTimeout)
|
||||
}
|
||||
if ctxDeadline, ok := r.ctx.Deadline(); ok && (deadline.IsZero() || ctxDeadline.Before(deadline)) {
|
||||
deadline = ctxDeadline
|
||||
}
|
||||
globals = make([]*inboundRPCGlobalReservation, len(r.entries))
|
||||
for i := range r.entries {
|
||||
globals[i] = r.entries[i].global
|
||||
}
|
||||
c.rpcMu.Lock()
|
||||
c.rpcReserved -= len(r.entries)
|
||||
if len(tasks) != len(r.entries) {
|
||||
c.inflightRPCBytes.Add(-r.totalSize)
|
||||
result = errInboundRPCBatchTaskCount
|
||||
} else if c.rpcClosed || c.terminal.Load() {
|
||||
c.inflightRPCBytes.Add(-r.totalSize)
|
||||
} else {
|
||||
prepared := make([]inboundRPC, len(tasks))
|
||||
// The request deadline starts when admission succeeds, not when a worker
|
||||
// eventually dequeues the request. This bounds total queue + execution
|
||||
// latency and lets a queued request emit its explicit timeout on time.
|
||||
for i := range tasks {
|
||||
task := tasks[i]
|
||||
entry := r.entries[i]
|
||||
if deadline.IsZero() {
|
||||
task.ctx, task.cancel = context.WithCancel(r.ctx)
|
||||
} else {
|
||||
task.ctx, task.cancel = context.WithDeadline(r.ctx, deadline)
|
||||
}
|
||||
task.stopRoot = context.AfterFunc(c.rpcRootCtx, task.cancel)
|
||||
task.method = entry.method
|
||||
task.enqueuedAt = enqueuedAt
|
||||
task.deadline = deadline
|
||||
task.size = entry.size
|
||||
task.budget = entry.global
|
||||
ticket := &inboundRPCTicket{}
|
||||
if task.onTimeout != nil {
|
||||
onTimeout := task.onTimeout
|
||||
var timeoutOnce sync.Once
|
||||
ticket.onTimeout = func() {
|
||||
timeoutOnce.Do(onTimeout)
|
||||
}
|
||||
task.onTimeout = ticket.onTimeout
|
||||
}
|
||||
task.ticket = ticket
|
||||
if task.onTimeout != nil && !task.deadline.IsZero() {
|
||||
taskCtx := task.ctx
|
||||
task.stopTimeout = context.AfterFunc(taskCtx, func() {
|
||||
if errors.Is(taskCtx.Err(), context.DeadlineExceeded) {
|
||||
c.expireInboundRPCTicket(ticket)
|
||||
}
|
||||
})
|
||||
}
|
||||
prepared[i] = task
|
||||
}
|
||||
firstQueueLen = len(c.rpcQueue) + 1
|
||||
c.rpcQueue = append(c.rpcQueue, prepared...)
|
||||
queueCap = c.rpcQueueSize
|
||||
if len(prepared) > 0 && c.rpcRunning < c.rpcMaxInflight && !c.rpcReady {
|
||||
c.rpcReady = true
|
||||
reschedule = true
|
||||
}
|
||||
committed = true
|
||||
result = nil
|
||||
}
|
||||
c.rpcMu.Unlock()
|
||||
c.rpcReservationWG.Done()
|
||||
if !committed {
|
||||
releaseInboundRPCGlobalBatch(globals)
|
||||
}
|
||||
})
|
||||
if committed {
|
||||
for i, entry := range r.entries {
|
||||
r.conn.metrics.InboundRPCQueued(entry.method, firstQueueLen+i, queueCap)
|
||||
}
|
||||
if reschedule {
|
||||
var once sync.Once
|
||||
activate = func() {
|
||||
once.Do(func() {
|
||||
r.conn.rpcScheduler.schedule(r.conn)
|
||||
})
|
||||
}
|
||||
if !deferSchedule {
|
||||
activate()
|
||||
activate = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return activate, result
|
||||
}
|
||||
|
||||
func (r *inboundRPCBatchReservation) abort() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.once.Do(func() {
|
||||
c := r.conn
|
||||
c.rpcMu.Lock()
|
||||
c.rpcReserved -= len(r.entries)
|
||||
c.inflightRPCBytes.Add(-r.totalSize)
|
||||
c.rpcMu.Unlock()
|
||||
c.rpcReservationWG.Done()
|
||||
globals := make([]*inboundRPCGlobalReservation, len(r.entries))
|
||||
for i := range r.entries {
|
||||
globals[i] = r.entries[i].global
|
||||
}
|
||||
releaseInboundRPCGlobalBatch(globals)
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Conn) takeInboundRPC() (task inboundRPC, ok, reschedule bool) {
|
||||
c.rpcMu.Lock()
|
||||
defer c.rpcMu.Unlock()
|
||||
|
|
@ -583,7 +923,7 @@ func (c *Conn) finishInboundRPC(task inboundRPC) {
|
|||
if task.ticket != nil {
|
||||
task.ticket.state.Store(inboundRPCTicketDone)
|
||||
}
|
||||
stopInboundRPCTask(task)
|
||||
timeoutHandoff := stopInboundRPCTask(task)
|
||||
var reschedule bool
|
||||
c.rpcMu.Lock()
|
||||
c.rpcRunning--
|
||||
|
|
@ -594,11 +934,21 @@ func (c *Conn) finishInboundRPC(task inboundRPC) {
|
|||
}
|
||||
c.rpcMu.Unlock()
|
||||
reservation := task.budget
|
||||
release := task.release
|
||||
// The scheduler budget may be reused immediately after release. Clear request-owned
|
||||
// closures/context references first so slow metrics/rescheduling cannot overlap the old body
|
||||
// with a newly admitted body under the same byte accounting.
|
||||
task = inboundRPC{}
|
||||
reservation.release()
|
||||
if timeoutHandoff != nil {
|
||||
// stopTimeout(false) means the deadline callback may already have read
|
||||
// Running but not yet entered ticket.onTimeout. Calling the sync.Once wrapper
|
||||
// here either performs or joins that response before owner release/Abort.
|
||||
timeoutHandoff()
|
||||
}
|
||||
if release != nil {
|
||||
release()
|
||||
}
|
||||
c.rpcWG.Done()
|
||||
if reschedule {
|
||||
c.rpcScheduler.schedule(c)
|
||||
|
|
@ -648,7 +998,8 @@ func (c *Conn) expireInboundRPCTicket(ticket *inboundRPCTicket) {
|
|||
if found {
|
||||
method := task.method
|
||||
reservation := task.budget
|
||||
stopInboundRPCTask(task)
|
||||
release := task.release
|
||||
_ = stopInboundRPCTask(task)
|
||||
// Drop the run/context closures before returning the byte reservation. Otherwise an
|
||||
// onTimeout callback that blocks or performs a slow write can keep the copied request body
|
||||
// reachable after the global scheduler has advertised those bytes as available again.
|
||||
|
|
@ -658,6 +1009,9 @@ func (c *Conn) expireInboundRPCTicket(ticket *inboundRPCTicket) {
|
|||
if ticket.onTimeout != nil {
|
||||
ticket.onTimeout()
|
||||
}
|
||||
if release != nil {
|
||||
release()
|
||||
}
|
||||
return
|
||||
}
|
||||
if ticket.state.Load() == inboundRPCTicketRunning && ticket.onTimeout != nil {
|
||||
|
|
@ -667,11 +1021,17 @@ func (c *Conn) expireInboundRPCTicket(ticket *inboundRPCTicket) {
|
|||
|
||||
// stopInboundRPCTask disarms callbacks before canceling the context so a normal
|
||||
// completion or connection close cannot manufacture an RPC_TIMEOUT response.
|
||||
// A deadline callback already in flight is harmless because enqueueRPC's response
|
||||
// gate makes timeout and normal rpc_result mutually exclusive.
|
||||
func stopInboundRPCTask(task inboundRPC) {
|
||||
// If the runtime already started a deadline callback, the returned sync.Once
|
||||
// wrapper is a mandatory handoff: callers invoke it before owner release so a
|
||||
// callback paused between ticket-state inspection and response-gate claim cannot
|
||||
// publish into a later flight generation.
|
||||
func stopInboundRPCTask(task inboundRPC) (timeoutHandoff func()) {
|
||||
if task.stopTimeout != nil {
|
||||
task.stopTimeout()
|
||||
stopped := task.stopTimeout()
|
||||
if !stopped && task.ctx != nil && errors.Is(task.ctx.Err(), context.DeadlineExceeded) &&
|
||||
task.ticket != nil && task.ticket.onTimeout != nil {
|
||||
timeoutHandoff = task.ticket.onTimeout
|
||||
}
|
||||
}
|
||||
if task.stopRoot != nil {
|
||||
task.stopRoot()
|
||||
|
|
@ -679,6 +1039,7 @@ func stopInboundRPCTask(task inboundRPC) {
|
|||
if task.cancel != nil {
|
||||
task.cancel()
|
||||
}
|
||||
return timeoutHandoff
|
||||
}
|
||||
|
||||
func (c *Conn) closeInboundRPCScheduler() {
|
||||
|
|
@ -722,9 +1083,16 @@ func (c *Conn) beginCloseInboundRPCScheduler() {
|
|||
}
|
||||
method := task.method
|
||||
reservation := task.budget
|
||||
stopInboundRPCTask(task)
|
||||
release := task.release
|
||||
timeoutHandoff := stopInboundRPCTask(task)
|
||||
task = inboundRPC{}
|
||||
reservation.release()
|
||||
if timeoutHandoff != nil {
|
||||
timeoutHandoff()
|
||||
}
|
||||
if release != nil {
|
||||
release()
|
||||
}
|
||||
c.metrics.InboundRPCDropped(method, "connection_closed")
|
||||
}
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue