refactor(mtproto): sync replace result cache with execution ledger
This commit is contained in:
parent
141f2f20c4
commit
c1597696af
39 changed files with 1621 additions and 2320 deletions
|
|
@ -33,10 +33,10 @@ TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS=8192
|
|||
TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES=536870912
|
||||
# Metadata-only rpc_result receipt budgets: global >= auth >= session. ACK deletes immediately;
|
||||
# 331s is only the no-ACK horizon. Payloads live solely in the logical-session outbound budget.
|
||||
TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES=262144
|
||||
TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES=32768
|
||||
TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES=16384
|
||||
TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH=2048
|
||||
TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES=262144
|
||||
TELESRV_MTPROTO_RPC_EXECUTION_AUTH_MAX_ENTRIES=32768
|
||||
TELESRV_MTPROTO_RPC_EXECUTION_SESSION_MAX_ENTRIES=16384
|
||||
TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH=2048
|
||||
# Process-wide in-flight transport wire + decrypted plaintext reservation.
|
||||
TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES=536870912
|
||||
# Per-connection outbound mailboxes (normal/control) and process-wide resend pending bodies.
|
||||
|
|
|
|||
|
|
@ -309,10 +309,11 @@ func mtprotoRuntimeGaugeSamples(snapshot mtprotoedge.RuntimeSnapshot) []obsmetri
|
|||
{Name: "telesrv_mtproto_outbound_tracked_byte_limit", Labels: []obsmetrics.Label{{Name: "kind", Value: "control"}}, Value: float64(snapshot.OutboundControlMaxBytes)},
|
||||
{Name: "telesrv_mtproto_outbound_write_bytes", Value: float64(snapshot.OutboundWriteBytes)},
|
||||
{Name: "telesrv_mtproto_outbound_write_byte_limit", Value: float64(snapshot.OutboundWriteMaxBytes)},
|
||||
{Name: "telesrv_mtproto_rpc_result_owners", Value: float64(snapshot.RPCResultOwners)},
|
||||
{Name: "telesrv_mtproto_rpc_result_receipts", Value: float64(snapshot.RPCResultReceipts)},
|
||||
{Name: "telesrv_mtproto_rpc_result_receipt_bytes", Value: float64(snapshot.RPCResultReceiptBytes)},
|
||||
{Name: "telesrv_mtproto_rpc_result_subscribers", Value: float64(snapshot.RPCResultSubscribers)},
|
||||
{Name: "telesrv_mtproto_rpc_execution_owners", Value: float64(snapshot.RPCExecutionOwners)},
|
||||
{Name: "telesrv_mtproto_rpc_execution_reserved_entries", Value: float64(snapshot.RPCExecutionReservedEntries)},
|
||||
{Name: "telesrv_mtproto_rpc_execution_receipts", Value: float64(snapshot.RPCExecutionReceipts)},
|
||||
{Name: "telesrv_mtproto_rpc_execution_receipt_budget_bytes", Value: float64(snapshot.RPCExecutionReceiptBudgetBytes)},
|
||||
{Name: "telesrv_mtproto_rpc_execution_subscribers", Value: float64(snapshot.RPCExecutionSubscribers)},
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1500,10 +1501,10 @@ func run(logger *zap.Logger) error {
|
|||
RPCGlobalWorkers: cfg.MTProtoRPCGlobalWorkers,
|
||||
RPCGlobalMaxTasks: cfg.MTProtoRPCGlobalMaxTasks,
|
||||
RPCGlobalMaxBytes: cfg.MTProtoRPCGlobalMaxBytes,
|
||||
RPCResultCacheMaxEntries: cfg.MTProtoRPCResultCacheMaxEntries,
|
||||
RPCResultCacheAuthMaxEntries: cfg.MTProtoRPCResultCacheAuthMaxEntries,
|
||||
RPCResultCacheSessionMaxEntries: cfg.MTProtoRPCResultCacheSessionMaxEntries,
|
||||
RPCResultPendingPerAuth: cfg.MTProtoRPCResultPendingPerAuth,
|
||||
RPCExecutionMaxEntries: cfg.MTProtoRPCExecutionMaxEntries,
|
||||
RPCExecutionAuthMaxEntries: cfg.MTProtoRPCExecutionAuthMaxEntries,
|
||||
RPCExecutionSessionMaxEntries: cfg.MTProtoRPCExecutionSessionMaxEntries,
|
||||
RPCExecutionPendingPerAuth: cfg.MTProtoRPCExecutionPendingPerAuth,
|
||||
InboundFrameGlobalMaxBytes: cfg.MTProtoInboundFrameGlobalMaxBytes,
|
||||
OutboundQueueSize: cfg.MTProtoOutboundQueueSize,
|
||||
OutboundControlQueueSize: cfg.MTProtoOutboundControlQueueSize,
|
||||
|
|
|
|||
|
|
@ -35,10 +35,10 @@ This document describes every setting loaded by `internal/config`. Defaults and
|
|||
| `TELESRV_MTPROTO_RPC_GLOBAL_WORKERS` | int / `256` | Shared fair-scheduler worker count. |
|
||||
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS` | int / `8192` | Process-wide scheduled/in-flight RPC task cap. |
|
||||
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES` | int64 charge bytes / `536870912` | Process-wide reserved/queued/in-flight RPC memory charge. Exact admission reserves a conservative typed-materialization charge from wire size and grows it atomically before a nested-gzip-expanded graph is decoded; grow failure rejects the complete candidate batch. This is not an equal amount of concurrently receivable wire bytes. |
|
||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES` | int / `262144` | Global cap for pending owners and compact unacknowledged-result receipts. Receipts retain request identity, execution outcome, and Layer admission metadata only—never TL bodies. `msgs_ack` removes them immediately; 331 seconds is only the no-ACK safety horizon. |
|
||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES` | int / `32768` | Per-raw-auth owner/receipt cap; limits satisfy `global >= auth >= session`. |
|
||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES` | int / `16384` | Per `raw auth key + session_id` owner/receipt cap. |
|
||||
| `TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH` | int / `2048` | Additional active-owner cap per raw auth key; no greater than global pending tasks or auth entries. |
|
||||
| `TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES` | int / `262144` | Global cap for pending owners and compact unacknowledged execution receipts. Receipts retain request identity, execution outcome, and Layer admission metadata only—never TL bodies. `msgs_ack` removes them immediately; 331 seconds is only the no-ACK safety horizon. |
|
||||
| `TELESRV_MTPROTO_RPC_EXECUTION_AUTH_MAX_ENTRIES` | int / `32768` | Per-raw-auth owner/receipt cap; limits satisfy `global >= auth >= session`. |
|
||||
| `TELESRV_MTPROTO_RPC_EXECUTION_SESSION_MAX_ENTRIES` | int / `16384` | Per `raw auth key + session_id` owner/receipt cap. |
|
||||
| `TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH` | int / `2048` | Additional active-owner cap per raw auth key; no greater than global pending tasks or auth entries. |
|
||||
| `TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | Process-wide reservation for transport wire bytes, maximum decrypted plaintext, and every live outer/nested gzip expansion, acquired before the corresponding payload allocation. |
|
||||
| `TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE` | int / `128` | Per-connection normal outbound mailbox capacity. |
|
||||
| `TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE` | int / `32` | Per-connection control-message mailbox capacity. |
|
||||
|
|
|
|||
|
|
@ -35,10 +35,10 @@
|
|||
| `TELESRV_MTPROTO_RPC_GLOBAL_WORKERS` | int / `256` | 共享公平调度器 worker 数。 |
|
||||
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS` | int / `8192` | 进程级排队与执行中的 RPC task 上限。 |
|
||||
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES` | int64 charge bytes / `536870912` | 进程级已预留/排队/执行中 RPC 内存 charge 预算;legacy 等于 copied body,exact 是 typed decode 前按 wire 与生成对象放大计算的保守 materialization charge。nested gzip 展开后会在 decoder 分配 typed graph 前原子增长该 charge,grow 失败原子拒绝整批候选 RPC;该值不代表可并发接收同等大小的 wire body。 |
|
||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES` | int / `262144` | pending owner 与未 ACK 的轻量结果收据全局上限。收据只存请求身份、执行结果和 Layer admission 元数据,不存 TL body;收到 `msgs_ack` 立即删除,331 秒仅是无 ACK 时的安全上限。 |
|
||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES` | int / `32768` | 单 raw auth key 的 owner/收据条目上限;必须满足 `global >= auth >= session`。 |
|
||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES` | int / `16384` | 单 `raw auth key + session_id` 的 owner/收据条目上限。 |
|
||||
| `TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH` | int / `2048` | 单 raw auth key 的 active pending owner 附加上限;必须不大于 `RPC_GLOBAL_MAX_TASKS` 和 auth entry 上限。 |
|
||||
| `TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES` | int / `262144` | pending owner 与未 ACK execution receipt 的全局上限。receipt 只存请求身份、执行结果和 Layer admission 元数据,不存 TL body;收到 `msgs_ack` 立即删除,331 秒仅是无 ACK 时的安全上限。 |
|
||||
| `TELESRV_MTPROTO_RPC_EXECUTION_AUTH_MAX_ENTRIES` | int / `32768` | 单 raw auth key 的 owner/receipt 条目上限;必须满足 `global >= auth >= session`。 |
|
||||
| `TELESRV_MTPROTO_RPC_EXECUTION_SESSION_MAX_ENTRIES` | int / `16384` | 单 `raw auth key + session_id` 的 owner/receipt 条目上限。 |
|
||||
| `TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH` | int / `2048` | 单 raw auth key 的 active pending owner 附加上限;必须不大于 `RPC_GLOBAL_MAX_TASKS` 和 auth entry 上限。 |
|
||||
| `TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | transport wire、最大解密明文以及每个 live outer/nested gzip 输出的进程级在途预算,均在对应 payload 分配前预留。 |
|
||||
| `TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE` | int / `128` | 单连接普通 outbound mailbox 容量。 |
|
||||
| `TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE` | int / `32` | 单连接控制消息 mailbox 容量。 |
|
||||
|
|
|
|||
|
|
@ -63,10 +63,10 @@ type Config struct {
|
|||
// Pending ownership and compact completed receipts share three-level
|
||||
// global/raw-auth/session entry accounting. Result bodies are never cached
|
||||
// here; the logical-session outbox owns unacknowledged wire bytes.
|
||||
MTProtoRPCResultCacheMaxEntries int
|
||||
MTProtoRPCResultCacheAuthMaxEntries int
|
||||
MTProtoRPCResultCacheSessionMaxEntries int
|
||||
MTProtoRPCResultPendingPerAuth int
|
||||
MTProtoRPCExecutionMaxEntries int
|
||||
MTProtoRPCExecutionAuthMaxEntries int
|
||||
MTProtoRPCExecutionSessionMaxEntries int
|
||||
MTProtoRPCExecutionPendingPerAuth int
|
||||
// MTProtoInboundFrameGlobalMaxBytes 是 transport wire + 最大解密 plaintext 的
|
||||
// 进程级在途预算;frame 长度读出后、payload 分配前预留。
|
||||
MTProtoInboundFrameGlobalMaxBytes int64
|
||||
|
|
@ -632,12 +632,12 @@ func Load() (Config, error) {
|
|||
MTProtoRPCGlobalWorkers: envIntOr("TELESRV_MTPROTO_RPC_GLOBAL_WORKERS", 256),
|
||||
MTProtoRPCGlobalMaxTasks: envIntOr("TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS", 8192),
|
||||
MTProtoRPCGlobalMaxBytes: envInt64Or("TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES", 512<<20),
|
||||
MTProtoRPCResultCacheMaxEntries: envIntOr("TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", 1<<18),
|
||||
MTProtoRPCResultCacheAuthMaxEntries: envIntOr("TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES", 1<<15),
|
||||
MTProtoRPCResultCacheSessionMaxEntries: envIntOr(
|
||||
"TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES", 1<<14,
|
||||
MTProtoRPCExecutionMaxEntries: envIntOr("TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES", 1<<18),
|
||||
MTProtoRPCExecutionAuthMaxEntries: envIntOr("TELESRV_MTPROTO_RPC_EXECUTION_AUTH_MAX_ENTRIES", 1<<15),
|
||||
MTProtoRPCExecutionSessionMaxEntries: envIntOr(
|
||||
"TELESRV_MTPROTO_RPC_EXECUTION_SESSION_MAX_ENTRIES", 1<<14,
|
||||
),
|
||||
MTProtoRPCResultPendingPerAuth: envIntOr("TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH", 1<<11),
|
||||
MTProtoRPCExecutionPendingPerAuth: envIntOr("TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH", 1<<11),
|
||||
MTProtoInboundFrameGlobalMaxBytes: envInt64Or("TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES", 512<<20),
|
||||
MTProtoOutboundQueueSize: envIntOr("TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE", 128),
|
||||
MTProtoOutboundControlQueueSize: envIntOr("TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE", 32),
|
||||
|
|
@ -863,7 +863,7 @@ func Load() (Config, error) {
|
|||
if err := validateLoginEmailConfig(cfg); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if err := validateRPCResultCacheConfig(cfg); err != nil {
|
||||
if err := validateRPCExecutionConfig(cfg); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
if err := validateStarGiftConfig(cfg); err != nil {
|
||||
|
|
@ -1222,21 +1222,21 @@ func validateCollectibleUsernameConfig(cfg Config) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func validateRPCResultCacheConfig(cfg Config) error {
|
||||
if cfg.MTProtoRPCResultCacheMaxEntries <= 0 || cfg.MTProtoRPCResultCacheAuthMaxEntries <= 0 ||
|
||||
cfg.MTProtoRPCResultCacheSessionMaxEntries <= 0 {
|
||||
return fmt.Errorf("MTProto rpc_result entry limits must be positive")
|
||||
func validateRPCExecutionConfig(cfg Config) error {
|
||||
if cfg.MTProtoRPCExecutionMaxEntries <= 0 || cfg.MTProtoRPCExecutionAuthMaxEntries <= 0 ||
|
||||
cfg.MTProtoRPCExecutionSessionMaxEntries <= 0 {
|
||||
return fmt.Errorf("MTProto rpc execution entry limits must be positive")
|
||||
}
|
||||
if cfg.MTProtoRPCResultCacheMaxEntries < cfg.MTProtoRPCResultCacheAuthMaxEntries ||
|
||||
cfg.MTProtoRPCResultCacheAuthMaxEntries < cfg.MTProtoRPCResultCacheSessionMaxEntries {
|
||||
return fmt.Errorf("MTProto rpc_result entry hierarchy must satisfy global >= auth >= session: %d/%d/%d",
|
||||
cfg.MTProtoRPCResultCacheMaxEntries, cfg.MTProtoRPCResultCacheAuthMaxEntries, cfg.MTProtoRPCResultCacheSessionMaxEntries)
|
||||
if cfg.MTProtoRPCExecutionMaxEntries < cfg.MTProtoRPCExecutionAuthMaxEntries ||
|
||||
cfg.MTProtoRPCExecutionAuthMaxEntries < cfg.MTProtoRPCExecutionSessionMaxEntries {
|
||||
return fmt.Errorf("MTProto rpc execution entry hierarchy must satisfy global >= auth >= session: %d/%d/%d",
|
||||
cfg.MTProtoRPCExecutionMaxEntries, cfg.MTProtoRPCExecutionAuthMaxEntries, cfg.MTProtoRPCExecutionSessionMaxEntries)
|
||||
}
|
||||
if cfg.MTProtoRPCGlobalMaxTasks <= 0 || cfg.MTProtoRPCResultPendingPerAuth <= 0 ||
|
||||
cfg.MTProtoRPCResultPendingPerAuth > cfg.MTProtoRPCGlobalMaxTasks ||
|
||||
cfg.MTProtoRPCResultPendingPerAuth > cfg.MTProtoRPCResultCacheAuthMaxEntries {
|
||||
return fmt.Errorf("MTProto rpc_result pending-per-auth %d must be positive and <= global pending %d and auth entries %d",
|
||||
cfg.MTProtoRPCResultPendingPerAuth, cfg.MTProtoRPCGlobalMaxTasks, cfg.MTProtoRPCResultCacheAuthMaxEntries)
|
||||
if cfg.MTProtoRPCGlobalMaxTasks <= 0 || cfg.MTProtoRPCExecutionPendingPerAuth <= 0 ||
|
||||
cfg.MTProtoRPCExecutionPendingPerAuth > cfg.MTProtoRPCGlobalMaxTasks ||
|
||||
cfg.MTProtoRPCExecutionPendingPerAuth > cfg.MTProtoRPCExecutionAuthMaxEntries {
|
||||
return fmt.Errorf("MTProto rpc execution pending-per-auth %d must be positive and <= global pending %d and auth entries %d",
|
||||
cfg.MTProtoRPCExecutionPendingPerAuth, cfg.MTProtoRPCGlobalMaxTasks, cfg.MTProtoRPCExecutionAuthMaxEntries)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1531,10 +1531,10 @@ func validateStrictMTProtoCapacityEnv(e envSource) error {
|
|||
"TELESRV_MTPROTO_RPC_QUEUE_SIZE",
|
||||
"TELESRV_MTPROTO_RPC_GLOBAL_WORKERS",
|
||||
"TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS",
|
||||
"TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES",
|
||||
"TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES",
|
||||
"TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES",
|
||||
"TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH",
|
||||
"TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES",
|
||||
"TELESRV_MTPROTO_RPC_EXECUTION_AUTH_MAX_ENTRIES",
|
||||
"TELESRV_MTPROTO_RPC_EXECUTION_SESSION_MAX_ENTRIES",
|
||||
"TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH",
|
||||
"TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE",
|
||||
"TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE",
|
||||
} {
|
||||
|
|
|
|||
|
|
@ -176,10 +176,10 @@ func TestLoadMTProtoAdmissionAndRPCBudgets(t *testing.T) {
|
|||
t.Setenv("TELESRV_MTPROTO_RPC_GLOBAL_WORKERS", "33")
|
||||
t.Setenv("TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS", "444")
|
||||
t.Setenv("TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES", "555555")
|
||||
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", "555")
|
||||
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES", "444")
|
||||
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES", "333")
|
||||
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH", "222")
|
||||
t.Setenv("TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES", "555")
|
||||
t.Setenv("TELESRV_MTPROTO_RPC_EXECUTION_AUTH_MAX_ENTRIES", "444")
|
||||
t.Setenv("TELESRV_MTPROTO_RPC_EXECUTION_SESSION_MAX_ENTRIES", "333")
|
||||
t.Setenv("TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH", "222")
|
||||
t.Setenv("TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES", "777777")
|
||||
t.Setenv("TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE", "88")
|
||||
t.Setenv("TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE", "22")
|
||||
|
|
@ -200,15 +200,15 @@ func TestLoadMTProtoAdmissionAndRPCBudgets(t *testing.T) {
|
|||
cfg.MTProtoRPCGlobalWorkers != 33 || cfg.MTProtoRPCGlobalMaxTasks != 444 || cfg.MTProtoRPCGlobalMaxBytes != 555555 {
|
||||
t.Fatalf("rpc budget config = %d/%d/%v/%d/%d/%d", cfg.MTProtoRPCMaxInflight, cfg.MTProtoRPCQueueSize, cfg.MTProtoRPCTimeout, cfg.MTProtoRPCGlobalWorkers, cfg.MTProtoRPCGlobalMaxTasks, cfg.MTProtoRPCGlobalMaxBytes)
|
||||
}
|
||||
if cfg.MTProtoRPCResultCacheMaxEntries != 555 ||
|
||||
cfg.MTProtoRPCResultCacheAuthMaxEntries != 444 ||
|
||||
cfg.MTProtoRPCResultCacheSessionMaxEntries != 333 ||
|
||||
cfg.MTProtoRPCResultPendingPerAuth != 222 {
|
||||
t.Fatalf("rpc result receipt config = global:%d auth:%d session:%d pending/auth:%d",
|
||||
cfg.MTProtoRPCResultCacheMaxEntries,
|
||||
cfg.MTProtoRPCResultCacheAuthMaxEntries,
|
||||
cfg.MTProtoRPCResultCacheSessionMaxEntries,
|
||||
cfg.MTProtoRPCResultPendingPerAuth)
|
||||
if cfg.MTProtoRPCExecutionMaxEntries != 555 ||
|
||||
cfg.MTProtoRPCExecutionAuthMaxEntries != 444 ||
|
||||
cfg.MTProtoRPCExecutionSessionMaxEntries != 333 ||
|
||||
cfg.MTProtoRPCExecutionPendingPerAuth != 222 {
|
||||
t.Fatalf("rpc execution ledger config = global:%d auth:%d session:%d pending/auth:%d",
|
||||
cfg.MTProtoRPCExecutionMaxEntries,
|
||||
cfg.MTProtoRPCExecutionAuthMaxEntries,
|
||||
cfg.MTProtoRPCExecutionSessionMaxEntries,
|
||||
cfg.MTProtoRPCExecutionPendingPerAuth)
|
||||
}
|
||||
if cfg.MTProtoInboundFrameGlobalMaxBytes != 777777 {
|
||||
t.Fatalf("inbound frame budget config = %d", cfg.MTProtoInboundFrameGlobalMaxBytes)
|
||||
|
|
@ -221,32 +221,32 @@ func TestLoadMTProtoAdmissionAndRPCBudgets(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestLoadRPCResultFairBudgetDefaults(t *testing.T) {
|
||||
func TestLoadRPCExecutionFairBudgetDefaults(t *testing.T) {
|
||||
disableDefaultConfigFile(t)
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if cfg.MTProtoRPCResultCacheMaxEntries != 1<<18 ||
|
||||
cfg.MTProtoRPCResultCacheAuthMaxEntries != 1<<15 ||
|
||||
cfg.MTProtoRPCResultCacheSessionMaxEntries != 1<<14 ||
|
||||
cfg.MTProtoRPCResultPendingPerAuth != 1<<11 {
|
||||
t.Fatalf("rpc_result receipt defaults = global:%d auth:%d session:%d pending/auth:%d",
|
||||
cfg.MTProtoRPCResultCacheMaxEntries,
|
||||
cfg.MTProtoRPCResultCacheAuthMaxEntries,
|
||||
cfg.MTProtoRPCResultCacheSessionMaxEntries,
|
||||
cfg.MTProtoRPCResultPendingPerAuth)
|
||||
if cfg.MTProtoRPCExecutionMaxEntries != 1<<18 ||
|
||||
cfg.MTProtoRPCExecutionAuthMaxEntries != 1<<15 ||
|
||||
cfg.MTProtoRPCExecutionSessionMaxEntries != 1<<14 ||
|
||||
cfg.MTProtoRPCExecutionPendingPerAuth != 1<<11 {
|
||||
t.Fatalf("rpc execution receipt defaults = global:%d auth:%d session:%d pending/auth:%d",
|
||||
cfg.MTProtoRPCExecutionMaxEntries,
|
||||
cfg.MTProtoRPCExecutionAuthMaxEntries,
|
||||
cfg.MTProtoRPCExecutionSessionMaxEntries,
|
||||
cfg.MTProtoRPCExecutionPendingPerAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidRPCResultFairBudgets(t *testing.T) {
|
||||
func TestLoadRejectsInvalidRPCExecutionFairBudgets(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
value string
|
||||
}{
|
||||
{name: "entry hierarchy", key: "TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", value: "1024"},
|
||||
{name: "pending hierarchy", key: "TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH", value: "9000"},
|
||||
{name: "entry hierarchy", key: "TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES", value: "1024"},
|
||||
{name: "pending hierarchy", key: "TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH", value: "9000"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
|
@ -266,7 +266,7 @@ func TestLoadRejectsMalformedMTProtoCapacity(t *testing.T) {
|
|||
value string
|
||||
}{
|
||||
{name: "worker tasks malformed", key: "TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS", value: "lots"},
|
||||
{name: "receipt entries overflow", key: "TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", value: "999999999999999999999999"},
|
||||
{name: "receipt entries overflow", key: "TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES", value: "999999999999999999999999"},
|
||||
{name: "tracked bytes overflow", key: "TELESRV_MTPROTO_OUTBOUND_TRACKED_GLOBAL_MAX_BYTES", value: "999999999999999999999999"},
|
||||
{name: "outbound queue malformed", key: "TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE", value: "many"},
|
||||
} {
|
||||
|
|
|
|||
|
|
@ -920,9 +920,9 @@ func evaluateReport(report *RunReport, cfg RunConfig) {
|
|||
checks := []string{
|
||||
"telesrv_mtproto_raw_connections", "telesrv_mtproto_logical_sessions",
|
||||
"telesrv_mtproto_logical_outbox_bytes", "telesrv_mtproto_pending_push_bytes",
|
||||
"telesrv_mtproto_outbound_tracked_bytes", "telesrv_mtproto_rpc_result_owners",
|
||||
"telesrv_mtproto_rpc_result_receipts", "telesrv_mtproto_rpc_result_receipt_bytes",
|
||||
"telesrv_mtproto_rpc_result_subscribers",
|
||||
"telesrv_mtproto_outbound_tracked_bytes", "telesrv_mtproto_rpc_execution_owners",
|
||||
"telesrv_mtproto_rpc_execution_reserved_entries", "telesrv_mtproto_rpc_execution_receipts",
|
||||
"telesrv_mtproto_rpc_execution_receipt_budget_bytes", "telesrv_mtproto_rpc_execution_subscribers",
|
||||
}
|
||||
for _, name := range checks {
|
||||
baseline := metricValue(report.BaselineServerMetrics, name)
|
||||
|
|
|
|||
|
|
@ -31,10 +31,11 @@ var selectedServerMetrics = map[string]struct{}{
|
|||
"telesrv_mtproto_inbound_frame_bytes": {},
|
||||
"telesrv_mtproto_outbound_tracked_bytes": {},
|
||||
"telesrv_mtproto_outbound_write_bytes": {},
|
||||
"telesrv_mtproto_rpc_result_owners": {},
|
||||
"telesrv_mtproto_rpc_result_receipts": {},
|
||||
"telesrv_mtproto_rpc_result_receipt_bytes": {},
|
||||
"telesrv_mtproto_rpc_result_subscribers": {},
|
||||
"telesrv_mtproto_rpc_execution_owners": {},
|
||||
"telesrv_mtproto_rpc_execution_reserved_entries": {},
|
||||
"telesrv_mtproto_rpc_execution_receipts": {},
|
||||
"telesrv_mtproto_rpc_execution_receipt_budget_bytes": {},
|
||||
"telesrv_mtproto_rpc_execution_subscribers": {},
|
||||
"telesrv_mtproto_rpc_result_inner_bytes_total": {},
|
||||
"telesrv_mtproto_rpc_result_wire_bytes_total": {},
|
||||
"telesrv_mtproto_rpc_result_delivered_bytes_total": {},
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ type Conn struct {
|
|||
rpcReady bool
|
||||
rpcClosed bool
|
||||
// rpcReplayRestores is a per-physical-connection ordering barrier. An exact
|
||||
// cached/rewrapped init request has already executed its business handler,
|
||||
// replayed/rewrapped init request has already executed its business handler,
|
||||
// but its wrapper/client/readiness state becomes authoritative only after the
|
||||
// replacement rpc_result is physically written. Queued naked RPCs remain
|
||||
// admitted and budgeted, but are not scheduler-runnable until every such
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ func (c *Conn) layerProfileRawEvidenceState() (LayerProfileSnapshot, int, int64)
|
|||
|
||||
// freezeLayerProfileAt is the production explicit-evidence transition. The
|
||||
// positive client msg_id is the protocol ordering authority across TCP
|
||||
// reconnects and cached request replays.
|
||||
// reconnects and retained request replays.
|
||||
func (c *Conn) freezeLayerProfileAt(profile tlprofile.Profile, msgID int64) (bool, error) {
|
||||
if c == nil {
|
||||
return false, fmt.Errorf("nil connection layer profile")
|
||||
|
|
|
|||
|
|
@ -898,24 +898,16 @@ func (s *Server) publishRPCResult(
|
|||
if s == nil || s.rpcResults == nil || c == nil || encoded == nil || reqMsgID == 0 {
|
||||
return errors.New("rpc result receipt ledger is unavailable")
|
||||
}
|
||||
if s.rpcResults.sessions == nil && int64(len(encoded.body)) > s.rpcResults.completedBytes.max {
|
||||
// Focused legacy-cache tests may intentionally use a byte budget smaller
|
||||
// than one legal transport result. Production receipts never own bodies.
|
||||
panic(fmt.Sprintf(
|
||||
"mtprotoedge: legacy encoded rpc result exceeds inline-ledger budget: body=%d max=%d",
|
||||
len(encoded.body), s.rpcResults.completedBytes.max,
|
||||
))
|
||||
}
|
||||
priority, visible := prepareEncoded(encoded)
|
||||
if owner != nil && !owner.HandOff() {
|
||||
return ErrRPCResultFlightInvalid
|
||||
}
|
||||
started := time.Now()
|
||||
encoded.markReplayable()
|
||||
// Put may expose a completed result only after the old logical connection
|
||||
// Complete may expose terminal execution only after the old connection
|
||||
// is irreversibly unable to accept another same-generation request.
|
||||
c.fenceUndeliveredRPCResult()
|
||||
s.storeRPCResult(c, reqMsgID, encoded)
|
||||
s.completeRPCResult(c, reqMsgID, encoded, false)
|
||||
latency := time.Since(started)
|
||||
if metrics, ok := s.metrics.(RPCResultMetrics); ok {
|
||||
metrics.RPCResultDelivered(method, latency, len(encoded.body), admissionErr)
|
||||
|
|
@ -989,7 +981,7 @@ func (s *Server) publishRPCResult(
|
|||
if deliveryErr != nil {
|
||||
encoded.markReplayable()
|
||||
c.fenceUndeliveredRPCResult()
|
||||
s.storeRPCResult(c, reqMsgID, encoded)
|
||||
s.completeRPCResult(c, reqMsgID, encoded, true)
|
||||
if checked := s.log.Check(resultLogLevel, "RPC result delivery fenced for replay"); checked != nil {
|
||||
checked.Write(
|
||||
zap.String("method", method), zap.Int64("req_msg_id", reqMsgID),
|
||||
|
|
@ -1001,7 +993,7 @@ func (s *Server) publishRPCResult(
|
|||
return
|
||||
}
|
||||
encoded.markDelivered()
|
||||
s.storeRPCResult(c, reqMsgID, encoded)
|
||||
s.completeRPCResult(c, reqMsgID, encoded, true)
|
||||
if checked := s.log.Check(resultLogLevel, "RPC result delivered"); checked != nil {
|
||||
checked.Write(
|
||||
zap.String("method", method), zap.Int64("req_msg_id", reqMsgID),
|
||||
|
|
@ -1054,24 +1046,24 @@ func (s *Server) sendResult(ctx context.Context, c *Conn, reqMsgID int64, result
|
|||
// same-Conn duplicate would be ACKed while no result can ever arrive.
|
||||
c.fenceUndeliveredRPCResult()
|
||||
encoded.markReplayable()
|
||||
s.storeRPCResult(c, reqMsgID, encoded)
|
||||
s.completeRPCResult(c, reqMsgID, encoded, true)
|
||||
return err
|
||||
}
|
||||
encoded.markDelivered()
|
||||
// On a live Conn, completed means the rpc_result has reached the reliable byte
|
||||
// stream. Same-physical duplicates can therefore be ACK-only without data loss.
|
||||
s.storeRPCResult(c, reqMsgID, encoded)
|
||||
s.completeRPCResult(c, reqMsgID, encoded, true)
|
||||
return nil
|
||||
}
|
||||
|
||||
// sendCachedRPCResult preserves the delivery half of the rpc_result invariant
|
||||
// sendReplayedRPCResult preserves the delivery half of the rpc_result invariant
|
||||
// for completed-flight replays: either the logical outbox result reaches this physical
|
||||
// byte stream, or this logical Conn is fenced so a replacement may retry it.
|
||||
func (s *Server) sendCachedRPCResult(ctx context.Context, c *Conn, encoded *encodedOutboundMessage) error {
|
||||
return s.sendCachedRPCResultWithHook(ctx, c, encoded, nil)
|
||||
func (s *Server) sendReplayedRPCResult(ctx context.Context, c *Conn, encoded *encodedOutboundMessage) error {
|
||||
return s.sendReplayedRPCResultWithHook(ctx, c, encoded, nil)
|
||||
}
|
||||
|
||||
func (s *Server) sendCachedRPCResultWithHook(
|
||||
func (s *Server) sendReplayedRPCResultWithHook(
|
||||
ctx context.Context,
|
||||
c *Conn,
|
||||
encoded *encodedOutboundMessage,
|
||||
|
|
@ -1079,7 +1071,7 @@ func (s *Server) sendCachedRPCResultWithHook(
|
|||
) error {
|
||||
if encoded == nil {
|
||||
c.fenceUndeliveredRPCResult()
|
||||
return errors.New("nil cached rpc_result")
|
||||
return errors.New("nil replayed rpc_result")
|
||||
}
|
||||
attempt, reserved, err := c.cloneRPCResultForRequestReserved(encoded, encoded.reqMsgID, false)
|
||||
if err != nil {
|
||||
|
|
@ -1096,7 +1088,7 @@ func (s *Server) sendCachedRPCResultWithHook(
|
|||
finishRestore = c.beginRPCReplayRestore()
|
||||
defer finishRestore()
|
||||
}
|
||||
// Cached replay owns its delivery-gated state synchronously. Calling the
|
||||
// Outbox replay owns its delivery-gated state synchronously. Calling the
|
||||
// lower send primitive avoids reserving the process-wide asynchronous hook
|
||||
// executor; the logical hook is claimed only after this physical write wins.
|
||||
if err := c.sendOutboundWithTerminalReserved(
|
||||
|
|
@ -1117,10 +1109,10 @@ func (s *Server) sendCachedRPCResultWithHook(
|
|||
// the sticky deferral). Fence before the deferred barrier is released; a
|
||||
// later physical generation may wait for Done and replay the same bytes.
|
||||
c.fenceUndeliveredRPCResult()
|
||||
return fmt.Errorf("wait for cached rpc_result logical restore: %w", claimErr)
|
||||
return fmt.Errorf("wait for replayed rpc_result logical restore: %w", claimErr)
|
||||
}
|
||||
return s.runBoundedRPCReplayRestore(
|
||||
restoreCtx, c, "cached rpc_result", logicalRestore, afterSuccessfulDelivery,
|
||||
restoreCtx, c, "replayed rpc_result", logicalRestore, afterSuccessfulDelivery,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -1320,11 +1312,11 @@ func (s *Server) encodeRPCResultWithoutSlot(ctx context.Context, c *Conn, reqMsg
|
|||
}, nil
|
||||
}
|
||||
|
||||
func (s *Server) cachedRPCResult(c *Conn, reqMsgID int64) (*encodedOutboundMessage, bool) {
|
||||
func (s *Server) replayableRPCResult(c *Conn, reqMsgID int64) (*encodedOutboundMessage, bool) {
|
||||
if s == nil || s.rpcResults == nil || c == nil {
|
||||
return nil, false
|
||||
}
|
||||
return s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID)
|
||||
return s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgID)
|
||||
}
|
||||
|
||||
func (s *Server) replayRPCResultByRequest(ctx context.Context, c *Conn, reqMsgID int64) error {
|
||||
|
|
@ -1335,24 +1327,26 @@ func (s *Server) replayRPCResultByRequest(ctx context.Context, c *Conn, reqMsgID
|
|||
c.fenceUndeliveredRPCResult()
|
||||
return err
|
||||
} else if resent {
|
||||
s.log.Debug("Resent connection cached rpc_result for duplicate msg_id", zap.Int64("msg_id", reqMsgID))
|
||||
s.log.Debug("Resent connection-retained rpc_result for duplicate msg_id", zap.Int64("msg_id", reqMsgID))
|
||||
return nil
|
||||
}
|
||||
if cached, ok := s.cachedRPCResult(c, reqMsgID); ok {
|
||||
if err := s.sendCachedRPCResult(ctx, c, cached); err != nil {
|
||||
if replayed, ok := s.replayableRPCResult(c, reqMsgID); ok {
|
||||
if err := s.sendReplayedRPCResult(ctx, c, replayed); err != nil {
|
||||
return err
|
||||
}
|
||||
s.log.Debug("Resent session cached rpc_result for duplicate msg_id", zap.Int64("msg_id", reqMsgID))
|
||||
s.log.Debug("Resent logical-outbox rpc_result for duplicate msg_id", zap.Int64("msg_id", reqMsgID))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) storeRPCResult(c *Conn, reqMsgID int64, encoded *encodedOutboundMessage) {
|
||||
func (s *Server) completeRPCResult(c *Conn, reqMsgID int64, encoded *encodedOutboundMessage, replayable bool) {
|
||||
if s == nil || s.rpcResults == nil || c == nil {
|
||||
return
|
||||
}
|
||||
if s.conns != nil {
|
||||
s.conns.adoptLogicalSession(c)
|
||||
s.rpcResults.Put(c.authKeyID, c.sessionID, reqMsgID, encoded)
|
||||
}
|
||||
s.rpcResults.Complete(c.authKeyID, c.sessionID, reqMsgID, encoded, replayable)
|
||||
}
|
||||
|
||||
// sendPong 回复 mt.PingRequest / mt.PingDelayDisconnectRequest。
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ type layerRPCProfileEvidence struct {
|
|||
|
||||
// layerRPCAdmissionCursor is the wire-ordered, side-effect-free view used while
|
||||
// decoding one MTProto container. evidenceMsgID is the last explicit
|
||||
// invokeWithLayer proof, not merely the profile used by an arbitrary cached
|
||||
// invokeWithLayer proof, not merely the profile used by an arbitrary retained
|
||||
// request. It therefore advances only after generated admission reports
|
||||
// ProfileEvidence.
|
||||
type layerRPCAdmissionCursor struct {
|
||||
|
|
@ -355,7 +355,7 @@ func (s *Server) prepareInboundLayerRPCBatch(ctx context.Context, c *Conn, plan
|
|||
item.method = method
|
||||
if profile, hasEvidence := admitted.ProfileEvidence(); hasEvidence {
|
||||
if existing && profile != existingProfile {
|
||||
return fmt.Errorf("%w: cached msg_id %d used Layer %d but replay selected Layer %d", ErrLayerProfileConflict, item.msgID, existingProfile, profile)
|
||||
return fmt.Errorf("%w: retained msg_id %d used Layer %d but replay selected Layer %d", ErrLayerProfileConflict, item.msgID, existingProfile, profile)
|
||||
}
|
||||
evidence[index] = layerRPCProfileEvidence{profile: profile, present: true, fresh: item.profileEvidenceFresh()}
|
||||
if evidence[index].fresh {
|
||||
|
|
@ -436,7 +436,7 @@ func (s *Server) prepareInboundLayerRPCBatch(ctx context.Context, c *Conn, plan
|
|||
case rpcResultAcquireCompleted:
|
||||
// Transfer the materialization ticket to the plan before any
|
||||
// profile-restore step can fail; plan.close is the universal abort
|
||||
// path and sendCached releases it after outbound-budget handoff.
|
||||
// path and sendReplayed releases it after outbound-budget handoff.
|
||||
item.payload = claim.encoded
|
||||
after, prepareErr := s.prepareAdmittedLayerRPCReplay(ctx, c, item.msgID, claim.admissionSeq, item.profileEvidenceFresh(), item.admitted)
|
||||
if prepareErr != nil {
|
||||
|
|
@ -1034,7 +1034,7 @@ func (s *Server) decodeInboundLayerRPCWithOptions(state LayerProfileSnapshot, bo
|
|||
// commitLayerProfileEvidence publishes one generated invokeWithLayer proof.
|
||||
// The exact-session registry is the cross-physical-connection linearization
|
||||
// point; the Conn cursor then prevents a concurrent older admission from
|
||||
// overwriting its local wire epoch. Older cached duplicates remain decodable
|
||||
// overwriting its local wire epoch. Older retained duplicates remain decodable
|
||||
// and request-bound, but cannot mutate session/profile state.
|
||||
func (s *Server) commitLayerProfileEvidence(ctx context.Context, c *Conn, profile tlprofile.Profile, msgID int64) (bool, error) {
|
||||
if s == nil || c == nil {
|
||||
|
|
|
|||
|
|
@ -966,7 +966,7 @@ func TestDurabilityOutageInitializesOnlyCurrentConnection(t *testing.T) {
|
|||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if _, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, msgID); !ok {
|
||||
if _, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, msgID); !ok {
|
||||
t.Fatal("successful outage-local init did not publish its exact RPC result")
|
||||
}
|
||||
if layer, found, err := router.ResolveInheritedAuthKeyLayer(ctx, c.authKeyID); err != nil || found || layer != 0 {
|
||||
|
|
@ -1071,7 +1071,7 @@ func TestInvariantReplayNeverCachesInternalCanonicalProfile(t *testing.T) {
|
|||
func TestSameMsgIDNakedReplayUsesWinnerAdmissionProfile(t *testing.T) {
|
||||
handler := newAdmissionOnlyLayerRPC()
|
||||
s := New(Options{DC: 2, LayerRPC: handler})
|
||||
s.rpcResults = newRPCResultCacheWithFlightLimit(time.Now, 8)
|
||||
s.rpcResults = newRPCExecutionLedgerForServerTest(s, time.Now, 8)
|
||||
authKeyID := [8]byte{0x22, 0x99}
|
||||
const sessionID = int64(2299)
|
||||
body := exactOutboundLayerRPCBody(t, tlprofile.Profile225, &tg.MessagesGetHistoryRequest{
|
||||
|
|
@ -1201,7 +1201,7 @@ func TestLayerEvidencePublicationBelongsOnlyToFreshFlightOwner(t *testing.T) {
|
|||
func TestLayerEvidenceNotPublishedWhenBatchFlightCapacityRollsBack(t *testing.T) {
|
||||
handler := newAdmissionOnlyLayerRPC()
|
||||
s := New(Options{DC: 2, LayerRPC: handler})
|
||||
s.rpcResults = newRPCResultCacheWithFlightLimit(time.Now, 1)
|
||||
s.rpcResults = newRPCExecutionLedgerForServerTest(s, time.Now, 1)
|
||||
c := &Conn{authKeyID: [8]byte{0x22, 0x97}, sessionID: 2297, metrics: NopMetrics{}}
|
||||
c.startInboundRPCScheduler(s.rpcScheduler, 1, 8, time.Second)
|
||||
defer c.Close()
|
||||
|
|
@ -1285,7 +1285,7 @@ func TestLogicalSessionLayerWatermarkSurvivesResultExpiryAndOldContainer(t *test
|
|||
now := newExpiryTestClock(time.Unix(1_900_000_000, 0))
|
||||
router := rpc.New(rpc.Config{DC: 2}, rpc.Deps{}, zaptest.NewLogger(t), now)
|
||||
s := New(Options{DC: 2, LayerRPC: router, Clock: now})
|
||||
s.rpcResults = newRPCResultCacheWithFlightLimit(now.Now, 8)
|
||||
s.rpcResults = newRPCExecutionLedgerForServerTest(s, now.Now, 8)
|
||||
authKeyID := [8]byte{0x22, 0x95}
|
||||
const sessionID = int64(2295)
|
||||
newConn := func() *Conn {
|
||||
|
|
@ -1331,7 +1331,7 @@ func TestLogicalSessionLayerWatermarkSurvivesResultExpiryAndOldContainer(t *test
|
|||
// inherited default remains Layer 227, while an old inner request may execute
|
||||
// request-bound but cannot recreate or roll back the exact-session watermark.
|
||||
now.Advance(10 * time.Minute)
|
||||
if _, ok := s.rpcResults.Get(authKeyID, sessionID, oldMsgID); ok {
|
||||
if _, ok := s.rpcResults.Replay(authKeyID, sessionID, oldMsgID); ok {
|
||||
t.Fatal("old completed result did not expire")
|
||||
}
|
||||
|
||||
|
|
@ -1978,7 +1978,7 @@ func TestUnprofiledInvariantBindKeepsProfileUnknownAndReturnsExactBool(t *testin
|
|||
func TestLayerRPCBatchCapacityKeepsExistingPendingReplay(t *testing.T) {
|
||||
router := rpc.New(rpc.Config{DC: 2, IP: "127.0.0.1", Port: 2398}, rpc.Deps{}, zaptest.NewLogger(t), clock.System)
|
||||
s := New(Options{DC: 2, LayerRPC: router})
|
||||
s.rpcResults = newRPCResultCacheWithFlightLimit(time.Now, 1)
|
||||
s.rpcResults = newRPCExecutionLedgerForServerTest(s, time.Now, 1)
|
||||
c := &Conn{
|
||||
authKeyID: [8]byte{7, 7, 1},
|
||||
sessionID: 771,
|
||||
|
|
@ -2033,7 +2033,7 @@ func TestLayerRPCBatchCapacityKeepsExistingPendingReplay(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestLayerRPCBatchCapacityAbortsRejectedRewrapOwner(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 1)
|
||||
authKeyID := [8]byte{7, 7, 9}
|
||||
const (
|
||||
sessionID = int64(779)
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ const (
|
|||
// It never dispatches business code a second time.
|
||||
inboundItemRewrappedRPC
|
||||
// inboundItemReplayRPC is a request first observed by this physical Conn whose
|
||||
// terminal result already exists in the cross-connection cache. It is distinct
|
||||
// terminal result already exists in the cross-connection execution ledger. It is distinct
|
||||
// from inboundItemDuplicate: a duplicate already present in this Conn's seen
|
||||
// table must only be ACKed. The original owner/result is already using the same
|
||||
// reliable TCP stream, so replaying it once per retransmit wave amplifies a
|
||||
|
|
@ -732,7 +732,7 @@ func preflightInboundItem(msgID int64, seqNo int32, typeID uint32, content bool,
|
|||
|
||||
// prepareInboundRPCBatch performs the whole container's count/byte admission
|
||||
// before copying or scheduling any API RPC. Capacity exhaustion is converted
|
||||
// into one consistent local 500 WORKER_BUSY_TOO_LONG_RETRY result per uncached
|
||||
// into one consistent local 500 WORKER_BUSY_TOO_LONG_RETRY result per new
|
||||
// RPC; no business handler from the batch is allowed to start in that case.
|
||||
func (s *Server) prepareInboundRPCBatch(ctx context.Context, c *Conn, plan *inboundPlan) error {
|
||||
if s.layerRPC != nil {
|
||||
|
|
@ -846,7 +846,7 @@ func (s *Server) prepareInboundRPCBatch(ctx context.Context, c *Conn, plan *inbo
|
|||
}
|
||||
switch claim.state {
|
||||
case rpcResultAcquireCompleted:
|
||||
s.log.Info("RPC duplicate replay from session cache",
|
||||
s.log.Info("RPC duplicate replay from logical-session outbox",
|
||||
zap.String("method", method),
|
||||
zap.Int64("msg_id", item.msgID),
|
||||
zap.String("auth_key_id", c.authKeyHex),
|
||||
|
|
@ -943,14 +943,14 @@ func (s *Server) executeInboundPlan(ctx context.Context, cs *connState, c *Conn,
|
|||
case inboundItemServiceDuplicate:
|
||||
// Classify from the originally committed connState record, never from the
|
||||
// retransmitted body. This prevents same-id payload replacement. Only an
|
||||
// already cached answer is eligible for resend (rpc_drop_answer today);
|
||||
// already retained answer is eligible for resend (rpc_drop_answer today);
|
||||
// other best-effort service traffic uses a later fresh request.
|
||||
if err := s.replayRPCResultByRequest(ctx, c, item.msgID); err != nil {
|
||||
return err
|
||||
}
|
||||
case inboundItemReplayRPC:
|
||||
if encoded, _ := item.payload.(*encodedOutboundMessage); encoded != nil {
|
||||
if err := s.sendCachedRPCResultWithHook(ctx, c, encoded, item.replayAfterSuccessfulDelivery); err != nil {
|
||||
if err := s.sendReplayedRPCResultWithHook(ctx, c, encoded, item.replayAfterSuccessfulDelivery); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err := s.replayRPCResultByRequest(ctx, c, item.msgID); err != nil {
|
||||
|
|
|
|||
|
|
@ -279,7 +279,7 @@ func TestLayerRPCAdmissionNestedGZIPSiblingsShareFrameExpansionLimit(t *testing.
|
|||
func TestLayerRPCAdmissionNestedGZIPReDecodeReusesMaterializationCharge(t *testing.T) {
|
||||
handler := newAdmissionOnlyLayerRPC()
|
||||
s := New(Options{DC: 2, LayerRPC: handler, Logger: zaptest.NewLogger(t)})
|
||||
s.rpcResults = newRPCResultCacheWithFlightLimit(time.Now, 8)
|
||||
s.rpcResults = newRPCExecutionLedgerForServerTest(s, time.Now, 8)
|
||||
scheduler := newInboundRPCScheduler(1, 4, 1<<30)
|
||||
s.rpcScheduler = scheduler
|
||||
authKeyID := [8]byte{8, 24}
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ var errLayerRPCResultIdentityMismatch = errors.New("layer RPC result does not ma
|
|||
// generated dispatcher. A LayerRPCHandler implementation must return the
|
||||
// result capability created from this exact admission; accepting a result from
|
||||
// another request would pair the wrong result TypeRef/profile with this
|
||||
// flight/cache identity even when both methods happen to share a Go type.
|
||||
// execution-ledger identity even when both methods happen to share a Go type.
|
||||
func bindAdmittedLayerRPCResult(request tlprofile.Admission, result tlprofile.Result) (*layerRPCResultEncoder, error) {
|
||||
if result == nil {
|
||||
return nil, nil
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ func TestProjectionFailureCachesInternalWithoutRepeatingBusiness(t *testing.T) {
|
|||
if rpcErr.ErrorCode != 500 || rpcErr.ErrorMessage != "INTERNAL" {
|
||||
t.Fatalf("projection terminal = %+v", rpcErr)
|
||||
}
|
||||
if _, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID); !ok {
|
||||
if _, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgID); !ok {
|
||||
t.Fatal("projection receipt disappeared before duplicate acquire")
|
||||
}
|
||||
// A same-msg replay is served from the completed exact identity; there is no
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ const (
|
|||
// legacy constructions which do not declare a stronger ownership model.
|
||||
outboundLayerBindingSession outboundLayerBindingKind = iota
|
||||
// A request-bound result retains the profile captured by admission. A later
|
||||
// invokeWithLayer correction must not invalidate that in-flight/cached result.
|
||||
// invokeWithLayer correction must not invalidate that in-flight/retained result.
|
||||
outboundLayerBindingRequest
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ func storeLogicalRPCResultForTest(
|
|||
encoded.replayMsgID = frame.msgID
|
||||
encoded.replaySeqNo = frame.seqNo
|
||||
state.mu.Unlock()
|
||||
s.rpcResults.Put(c.authKeyID, c.sessionID, reqMsgID, encoded)
|
||||
s.rpcResults.Complete(c.authKeyID, c.sessionID, reqMsgID, encoded, true)
|
||||
}
|
||||
|
||||
func acknowledgeLogicalRPCResultForTest(t *testing.T, s *Server, c *Conn, reqMsgID int64) {
|
||||
|
|
@ -160,12 +160,10 @@ func TestLogicalSessionACKReleasesPayloadAndReceipt(t *testing.T) {
|
|||
}
|
||||
c.outboundState.mu.Unlock()
|
||||
|
||||
ledger := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
|
||||
ledger := newRPCExecutionLedger(time.Now, rpcExecutionLedgerCapacity{
|
||||
maxPending: 16, maxPendingPerAuth: 16,
|
||||
globalMaxEntries: 16, globalMaxBytes: 16,
|
||||
authMaxEntries: 16, authMaxBytes: 16,
|
||||
sessionMaxEntries: 16, sessionMaxBytes: 16,
|
||||
sessions: manager,
|
||||
globalMaxEntries: 16, authMaxEntries: 16, sessionMaxEntries: 16,
|
||||
replayStore: manager,
|
||||
})
|
||||
claim, err := ledger.Acquire(authKeyID, 43, 88)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
|
|
@ -173,17 +171,20 @@ func TestLogicalSessionACKReleasesPayloadAndReceipt(t *testing.T) {
|
|||
}
|
||||
claim.owner.CompleteExecution(true)
|
||||
claim.owner.HandOff()
|
||||
ledger.Put(authKeyID, 43, 88, &encodedOutboundMessage{body: body, typeID: proto.ResultTypeID, reqMsgID: 88})
|
||||
ledger.Complete(authKeyID, 43, 88, &encodedOutboundMessage{body: body, typeID: proto.ResultTypeID, reqMsgID: 88}, true)
|
||||
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: 43, reqMsgID: 88}
|
||||
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: 43, reqMsgID: 88}
|
||||
shard := ledger.shard(key)
|
||||
shard.mu.Lock()
|
||||
entry := shard.byKey[key].Value.(*rpcResultCacheEntry)
|
||||
if entry.encoded != nil || entry.size != 1 {
|
||||
entry := shard.byKey[key].Value.(*rpcExecutionReceipt)
|
||||
if entry.unavailable {
|
||||
shard.mu.Unlock()
|
||||
t.Fatalf("completed ledger retained payload: encoded=%p size=%d", entry.encoded, entry.size)
|
||||
t.Fatal("logical outbox receipt was marked unavailable")
|
||||
}
|
||||
shard.mu.Unlock()
|
||||
if got := ledger.receiptBudgetBytes(); got != rpcExecutionReceiptBudgetBytes {
|
||||
t.Fatalf("receipt budget bytes = %d, want %d", got, rpcExecutionReceiptBudgetBytes)
|
||||
}
|
||||
|
||||
c.outboundState.mu.Lock()
|
||||
acked := c.outboundState.ack([]int64{201})
|
||||
|
|
@ -241,13 +242,13 @@ func TestLogicalSessionDestroyReleasesPayloadAndCompletedReceipt(t *testing.T) {
|
|||
storeLogicalRPCResultForTest(t, s, c, 901, &encodedOutboundMessage{
|
||||
body: make([]byte, 32), typeID: proto.ResultTypeID, reqMsgID: 901,
|
||||
})
|
||||
if _, ok := s.rpcResults.Get(authKeyID, 91, 901); !ok {
|
||||
if _, ok := s.rpcResults.Replay(authKeyID, 91, 901); !ok {
|
||||
t.Fatal("logical result fixture was not published")
|
||||
}
|
||||
if removed := s.conns.DestroySessionForAuthKey(authKeyID, 91); removed {
|
||||
t.Fatal("construction-only logical session unexpectedly reported active")
|
||||
}
|
||||
if _, ok := s.rpcResults.Get(authKeyID, 91, 901); ok {
|
||||
if _, ok := s.rpcResults.Replay(authKeyID, 91, 901); ok {
|
||||
t.Fatal("destroy retained completed receipt")
|
||||
}
|
||||
if got := s.outboundTrackedBudget.snapshot(); got != 0 {
|
||||
|
|
@ -272,7 +273,7 @@ func TestBusinessAuthRevocationReleasesOfflineTempLogicalSession(t *testing.T) {
|
|||
if closed := s.conns.CloseSessionsForBusinessAuthKey(businessAuthKeyID); closed != 0 {
|
||||
t.Fatalf("offline logical revocation closed %d physical conns", closed)
|
||||
}
|
||||
if _, ok := s.rpcResults.Get(rawAuthKeyID, 92, 902); ok {
|
||||
if _, ok := s.rpcResults.Replay(rawAuthKeyID, 92, 902); ok {
|
||||
t.Fatal("business auth revocation retained temp-key receipt")
|
||||
}
|
||||
if got := s.outboundTrackedBudget.snapshot(); got != 0 {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
|
@ -25,10 +26,72 @@ func (c *Conn) ForceClose() {
|
|||
c.waitOutboundShutdown()
|
||||
}
|
||||
|
||||
// newRPCResultCache keeps older focused cache tests concise without exposing a
|
||||
// second production constructor.
|
||||
func newRPCResultCache(now func() time.Time) *rpcResultCache {
|
||||
return newRPCResultCacheWithFlightLimit(now, rpcResultFlightDefaultMaxPending)
|
||||
type rpcReplayStoreForTest struct {
|
||||
mu sync.Mutex
|
||||
results map[rpcExecutionKey]*encodedOutboundMessage
|
||||
}
|
||||
|
||||
func newRPCReplayStoreForTest() *rpcReplayStoreForTest {
|
||||
return &rpcReplayStoreForTest{results: make(map[rpcExecutionKey]*encodedOutboundMessage)}
|
||||
}
|
||||
|
||||
func (s *rpcReplayStoreForTest) rpcResult(
|
||||
authKeyID [8]byte,
|
||||
sessionID, reqMsgID int64,
|
||||
) (*encodedOutboundMessage, bool) {
|
||||
if s == nil {
|
||||
return nil, false
|
||||
}
|
||||
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s.mu.Lock()
|
||||
result, ok := s.results[key]
|
||||
s.mu.Unlock()
|
||||
return result, ok
|
||||
}
|
||||
|
||||
func (s *rpcReplayStoreForTest) put(
|
||||
authKeyID [8]byte,
|
||||
sessionID, reqMsgID int64,
|
||||
encoded *encodedOutboundMessage,
|
||||
) {
|
||||
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s.mu.Lock()
|
||||
s.results[key] = encoded
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func newRPCExecutionLedgerForTest(now func() time.Time, maxPending int) *rpcExecutionLedger {
|
||||
return newRPCExecutionLedger(now, rpcExecutionLedgerCapacity{
|
||||
maxPending: maxPending, maxPendingPerAuth: maxPending,
|
||||
globalMaxEntries: rpcExecutionMaxEntries,
|
||||
authMaxEntries: rpcExecutionMaxEntries, sessionMaxEntries: rpcExecutionMaxEntries,
|
||||
replayStore: newRPCReplayStoreForTest(),
|
||||
})
|
||||
}
|
||||
|
||||
func newRPCExecutionLedgerForServerTest(s *Server, now func() time.Time, maxPending int) *rpcExecutionLedger {
|
||||
if s == nil || s.conns == nil {
|
||||
panic("newRPCExecutionLedgerForServerTest requires a server SessionManager")
|
||||
}
|
||||
return newRPCExecutionLedger(now, rpcExecutionLedgerCapacity{
|
||||
maxPending: maxPending, maxPendingPerAuth: maxPending,
|
||||
globalMaxEntries: rpcExecutionMaxEntries,
|
||||
authMaxEntries: rpcExecutionMaxEntries, sessionMaxEntries: rpcExecutionMaxEntries,
|
||||
replayStore: s.conns,
|
||||
})
|
||||
}
|
||||
|
||||
func (l *rpcExecutionLedger) completeReplayableForTest(
|
||||
authKeyID [8]byte,
|
||||
sessionID, reqMsgID int64,
|
||||
encoded *encodedOutboundMessage,
|
||||
) {
|
||||
store, ok := l.replayStore.(*rpcReplayStoreForTest)
|
||||
if !ok {
|
||||
panic("completeReplayableForTest requires rpcReplayStoreForTest")
|
||||
}
|
||||
store.put(authKeyID, sessionID, reqMsgID, encoded)
|
||||
l.Complete(authKeyID, sessionID, reqMsgID, encoded, true)
|
||||
}
|
||||
|
||||
// Conns is a white-box test accessor. Production wires the shared manager
|
||||
|
|
|
|||
|
|
@ -279,7 +279,7 @@ func TestCachedReplacementReplayBarrierWaitsForLogicalHookDone(t *testing.T) {
|
|||
firstConn := newOutboundTestConn(t, &collectingSessionTransport{}, newOutboundTrackedBudget(1<<20))
|
||||
firstResult := make(chan error, 1)
|
||||
go func() {
|
||||
firstResult <- s.sendCachedRPCResultWithHook(context.Background(), firstConn, encoded, func() error {
|
||||
firstResult <- s.sendReplayedRPCResultWithHook(context.Background(), firstConn, encoded, func() error {
|
||||
if !order.CompareAndSwap(0, 1) {
|
||||
return errors.New("first replacement restore ran out of order")
|
||||
}
|
||||
|
|
@ -300,7 +300,7 @@ func TestCachedReplacementReplayBarrierWaitsForLogicalHookDone(t *testing.T) {
|
|||
secondReplacement := make(chan struct{})
|
||||
secondResult := make(chan error, 1)
|
||||
go func() {
|
||||
secondResult <- s.sendCachedRPCResultWithHook(context.Background(), secondConn, encoded, func() error {
|
||||
secondResult <- s.sendReplayedRPCResultWithHook(context.Background(), secondConn, encoded, func() error {
|
||||
if !order.CompareAndSwap(2, 3) {
|
||||
return errors.New("second replacement restore passed logical hook completion")
|
||||
}
|
||||
|
|
|
|||
206
internal/mtprotoedge/rpc_execution_budget.go
Normal file
206
internal/mtprotoedge/rpc_execution_budget.go
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
package mtprotoedge
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash/maphash"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const rpcExecutionBudgetShards = 64
|
||||
|
||||
type rpcExecutionBudgetUsage struct {
|
||||
entries int64
|
||||
pending int64
|
||||
}
|
||||
|
||||
type rpcExecutionSessionBudgetKey struct {
|
||||
authKeyID [8]byte
|
||||
sessionID int64
|
||||
}
|
||||
|
||||
type rpcExecutionAuthBudgetShard struct {
|
||||
mu sync.Mutex
|
||||
usage map[[8]byte]rpcExecutionBudgetUsage
|
||||
}
|
||||
|
||||
type rpcExecutionSessionBudgetShard struct {
|
||||
mu sync.Mutex
|
||||
usage map[rpcExecutionSessionBudgetKey]rpcExecutionBudgetUsage
|
||||
}
|
||||
|
||||
// rpcExecutionFairBudget accounts one bounded ledger slot at global, auth and
|
||||
// session scopes. Pending owner and completed receipt are the same ownership:
|
||||
// Complete transfers the reservation; Abort/ACK/TTL return it.
|
||||
type rpcExecutionFairBudget struct {
|
||||
seed maphash.Seed
|
||||
globalEntries *rpcResultFlightLimit
|
||||
authLimit int64
|
||||
sessionLimit int64
|
||||
pendingPerAuth int64
|
||||
authShards [rpcExecutionBudgetShards]rpcExecutionAuthBudgetShard
|
||||
sessionShards [rpcExecutionBudgetShards]rpcExecutionSessionBudgetShard
|
||||
}
|
||||
|
||||
type rpcExecutionBudgetReservation struct {
|
||||
budget *rpcExecutionFairBudget
|
||||
key rpcExecutionKey
|
||||
pending bool
|
||||
released bool
|
||||
}
|
||||
|
||||
func newRPCExecutionFairBudget(
|
||||
seed maphash.Seed,
|
||||
globalEntries *rpcResultFlightLimit,
|
||||
authLimit int64,
|
||||
sessionLimit int64,
|
||||
pendingPerAuth int,
|
||||
) *rpcExecutionFairBudget {
|
||||
b := &rpcExecutionFairBudget{
|
||||
seed: seed, globalEntries: globalEntries, authLimit: authLimit,
|
||||
sessionLimit: sessionLimit, pendingPerAuth: int64(pendingPerAuth),
|
||||
}
|
||||
for i := range b.authShards {
|
||||
b.authShards[i].usage = make(map[[8]byte]rpcExecutionBudgetUsage)
|
||||
b.sessionShards[i].usage = make(map[rpcExecutionSessionBudgetKey]rpcExecutionBudgetUsage)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *rpcExecutionFairBudget) reserveOwner(key rpcExecutionKey) *rpcExecutionBudgetReservation {
|
||||
return b.reserve(key, true)
|
||||
}
|
||||
|
||||
func (b *rpcExecutionFairBudget) reserveCompleted(key rpcExecutionKey) *rpcExecutionBudgetReservation {
|
||||
return b.reserve(key, false)
|
||||
}
|
||||
|
||||
func (b *rpcExecutionFairBudget) reserve(key rpcExecutionKey, pending bool) *rpcExecutionBudgetReservation {
|
||||
if b == nil || b.globalEntries == nil {
|
||||
return nil
|
||||
}
|
||||
authShard := b.authShard(key.authKeyID)
|
||||
sessionKey := rpcExecutionSessionBudgetKey{authKeyID: key.authKeyID, sessionID: key.sessionID}
|
||||
sessionShard := b.sessionShard(sessionKey)
|
||||
authShard.mu.Lock()
|
||||
sessionShard.mu.Lock()
|
||||
authUsage := authShard.usage[key.authKeyID]
|
||||
sessionUsage := sessionShard.usage[sessionKey]
|
||||
allowed := withinRPCExecutionBudget(authUsage.entries, 1, b.authLimit) &&
|
||||
withinRPCExecutionBudget(sessionUsage.entries, 1, b.sessionLimit)
|
||||
if pending {
|
||||
allowed = allowed && withinRPCExecutionBudget(authUsage.pending, 1, b.pendingPerAuth)
|
||||
}
|
||||
if !allowed || !b.globalEntries.reserve() {
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
authUsage.entries++
|
||||
sessionUsage.entries++
|
||||
if pending {
|
||||
authUsage.pending++
|
||||
}
|
||||
authShard.usage[key.authKeyID] = authUsage
|
||||
sessionShard.usage[sessionKey] = sessionUsage
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
return &rpcExecutionBudgetReservation{budget: b, key: key, pending: pending}
|
||||
}
|
||||
|
||||
func withinRPCExecutionBudget(used, delta, limit int64) bool {
|
||||
return delta >= 0 && limit > 0 && used >= 0 && used <= limit-delta
|
||||
}
|
||||
|
||||
func (r *rpcExecutionBudgetReservation) releasePending() {
|
||||
if r == nil || r.budget == nil || r.released || !r.pending {
|
||||
return
|
||||
}
|
||||
b := r.budget
|
||||
shard := b.authShard(r.key.authKeyID)
|
||||
shard.mu.Lock()
|
||||
usage, ok := shard.usage[r.key.authKeyID]
|
||||
if !ok || usage.pending < 1 {
|
||||
shard.mu.Unlock()
|
||||
panic("mtprotoedge: rpc execution per-auth pending budget underflow")
|
||||
}
|
||||
usage.pending--
|
||||
shard.usage[r.key.authKeyID] = usage
|
||||
r.pending = false
|
||||
shard.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *rpcExecutionBudgetReservation) release() {
|
||||
if r == nil || r.budget == nil || r.released {
|
||||
return
|
||||
}
|
||||
b := r.budget
|
||||
authShard := b.authShard(r.key.authKeyID)
|
||||
sessionKey := rpcExecutionSessionBudgetKey{authKeyID: r.key.authKeyID, sessionID: r.key.sessionID}
|
||||
sessionShard := b.sessionShard(sessionKey)
|
||||
authShard.mu.Lock()
|
||||
sessionShard.mu.Lock()
|
||||
authUsage, authOK := authShard.usage[r.key.authKeyID]
|
||||
sessionUsage, sessionOK := sessionShard.usage[sessionKey]
|
||||
if !authOK || !sessionOK || authUsage.entries < 1 || sessionUsage.entries < 1 ||
|
||||
(r.pending && authUsage.pending < 1) {
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
panic("mtprotoedge: rpc execution fair budget underflow")
|
||||
}
|
||||
authUsage.entries--
|
||||
sessionUsage.entries--
|
||||
if r.pending {
|
||||
authUsage.pending--
|
||||
}
|
||||
if authUsage == (rpcExecutionBudgetUsage{}) {
|
||||
delete(authShard.usage, r.key.authKeyID)
|
||||
} else {
|
||||
authShard.usage[r.key.authKeyID] = authUsage
|
||||
}
|
||||
if sessionUsage == (rpcExecutionBudgetUsage{}) {
|
||||
delete(sessionShard.usage, sessionKey)
|
||||
} else {
|
||||
sessionShard.usage[sessionKey] = sessionUsage
|
||||
}
|
||||
r.released = true
|
||||
r.pending = false
|
||||
b.globalEntries.release()
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
}
|
||||
|
||||
func (b *rpcExecutionFairBudget) authSnapshot(authKeyID [8]byte) rpcExecutionBudgetUsage {
|
||||
if b == nil {
|
||||
return rpcExecutionBudgetUsage{}
|
||||
}
|
||||
shard := b.authShard(authKeyID)
|
||||
shard.mu.Lock()
|
||||
usage := shard.usage[authKeyID]
|
||||
shard.mu.Unlock()
|
||||
return usage
|
||||
}
|
||||
|
||||
func (b *rpcExecutionFairBudget) sessionSnapshot(authKeyID [8]byte, sessionID int64) rpcExecutionBudgetUsage {
|
||||
if b == nil {
|
||||
return rpcExecutionBudgetUsage{}
|
||||
}
|
||||
key := rpcExecutionSessionBudgetKey{authKeyID: authKeyID, sessionID: sessionID}
|
||||
shard := b.sessionShard(key)
|
||||
shard.mu.Lock()
|
||||
usage := shard.usage[key]
|
||||
shard.mu.Unlock()
|
||||
return usage
|
||||
}
|
||||
|
||||
func (b *rpcExecutionFairBudget) authShard(authKeyID [8]byte) *rpcExecutionAuthBudgetShard {
|
||||
index := maphash.Bytes(b.seed, authKeyID[:]) & (rpcExecutionBudgetShards - 1)
|
||||
return &b.authShards[index]
|
||||
}
|
||||
|
||||
func (b *rpcExecutionFairBudget) sessionShard(key rpcExecutionSessionBudgetKey) *rpcExecutionSessionBudgetShard {
|
||||
var raw [16]byte
|
||||
copy(raw[:8], key.authKeyID[:])
|
||||
binary.LittleEndian.PutUint64(raw[8:], uint64(key.sessionID))
|
||||
index := maphash.Bytes(b.seed, raw[:]) & (rpcExecutionBudgetShards - 1)
|
||||
return &b.sessionShards[index]
|
||||
}
|
||||
490
internal/mtprotoedge/rpc_execution_ledger.go
Normal file
490
internal/mtprotoedge/rpc_execution_ledger.go
Normal file
|
|
@ -0,0 +1,490 @@
|
|||
package mtprotoedge
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"hash/maphash"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// A valid client msg_id can be up to five minutes old or thirty seconds in
|
||||
// the future. The extra second covers scheduler and boundary jitter. This is
|
||||
// a no-ACK execution-receipt horizon, not a payload retention policy.
|
||||
rpcExecutionReceiptTTL = 331 * time.Second
|
||||
|
||||
rpcExecutionMaxEntries = 1 << 18
|
||||
rpcExecutionAuthMaxEntries = 1 << 15
|
||||
rpcExecutionSessionMaxEntries = 1 << 14
|
||||
rpcExecutionPendingPerAuth = 1 << 11
|
||||
|
||||
// Receipts contain only fixed-shape identity/outcome metadata. This
|
||||
// conservative charge covers the receipt, list node, map bucket share and
|
||||
// reservation bookkeeping. Payload bytes are accounted exclusively by the
|
||||
// logical-session outbox.
|
||||
rpcExecutionReceiptBudgetBytes = 384
|
||||
|
||||
// The complete replay identity is hashed with an instance-random seed. The
|
||||
// shard count is a power of two.
|
||||
rpcExecutionLedgerShards = 16
|
||||
)
|
||||
|
||||
type rpcExecutionKey struct {
|
||||
authKeyID [8]byte
|
||||
sessionID int64
|
||||
reqMsgID int64
|
||||
}
|
||||
|
||||
// rpcReplayStore is the sole source of retained rpc_result payloads. The
|
||||
// production implementation is SessionManager's logical-session outbox.
|
||||
type rpcReplayStore interface {
|
||||
rpcResult(authKeyID [8]byte, sessionID, reqMsgID int64) (*encodedOutboundMessage, bool)
|
||||
}
|
||||
|
||||
type rpcExecutionReceipt struct {
|
||||
key rpcExecutionKey
|
||||
expiresAt time.Time
|
||||
identity rpcResultRequestIdentity
|
||||
admissionSeq uint64
|
||||
executionKnown bool
|
||||
executionOK bool
|
||||
// acknowledged exists only for the ACK-before-Complete race. A completed
|
||||
// receipt is removed immediately when the ACK wins.
|
||||
acknowledged bool
|
||||
// unavailable is a bounded execution tombstone. It prevents a completed
|
||||
// business operation from running again when no exact outbox frame exists.
|
||||
unavailable bool
|
||||
// The pending owner transfers this same global/auth/session entry
|
||||
// reservation to the completed receipt.
|
||||
reservation *rpcExecutionBudgetReservation
|
||||
}
|
||||
|
||||
type rpcResultDependency struct {
|
||||
waiter *rpcResultWaiter
|
||||
completed bool
|
||||
success bool
|
||||
}
|
||||
|
||||
// rpcExecutionLedger owns request execution identity and duplicate
|
||||
// coordination. It never owns or copies rpc_result payload bytes; replay bytes
|
||||
// are resolved from replayStore only while the logical-session outbox retains
|
||||
// the exact unacknowledged frame.
|
||||
type rpcExecutionLedger struct {
|
||||
shards [rpcExecutionLedgerShards]rpcExecutionLedgerShard
|
||||
hashSeed maphash.Seed
|
||||
reservedEntries rpcResultFlightLimit
|
||||
receiptCount atomic.Int64
|
||||
fairBudget *rpcExecutionFairBudget
|
||||
flightLimit rpcResultFlightLimit
|
||||
subscriberBudget *rpcResultSubscriberBudget
|
||||
subscriberPerFlight int
|
||||
replayStore rpcReplayStore
|
||||
nextAdmissionSeq atomic.Uint64
|
||||
activeAdmissions rpcAdmissionTracker
|
||||
}
|
||||
|
||||
func (l *rpcExecutionLedger) stableAdmissionSafeFloor() uint64 {
|
||||
if l == nil {
|
||||
return 0
|
||||
}
|
||||
return l.activeAdmissions.stableSafeFloor(&l.nextAdmissionSeq)
|
||||
}
|
||||
|
||||
type rpcExecutionLedgerShard struct {
|
||||
mu sync.Mutex
|
||||
now func() time.Time
|
||||
ttl time.Duration
|
||||
receiptCount *atomic.Int64
|
||||
order *list.List
|
||||
byKey map[rpcExecutionKey]*list.Element
|
||||
// In-flight owners are independent from receipt TTL and cannot disappear
|
||||
// under completed-receipt pressure.
|
||||
pending map[rpcExecutionKey]*rpcResultFlight
|
||||
}
|
||||
|
||||
type rpcExecutionLedgerCapacity struct {
|
||||
maxPending int
|
||||
maxPendingPerAuth int
|
||||
globalMaxEntries int
|
||||
authMaxEntries int
|
||||
sessionMaxEntries int
|
||||
subscriberMaxGlobal int
|
||||
subscriberMaxAuth int
|
||||
subscriberMaxSession int
|
||||
subscriberMaxPerFlight int
|
||||
replayStore rpcReplayStore
|
||||
}
|
||||
|
||||
func newRPCExecutionLedger(now func() time.Time, capacity rpcExecutionLedgerCapacity) *rpcExecutionLedger {
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
if capacity.replayStore == nil {
|
||||
panic("mtprotoedge: rpc execution ledger requires a replay store")
|
||||
}
|
||||
if capacity.maxPending <= 0 {
|
||||
capacity.maxPending = rpcResultFlightDefaultMaxPending
|
||||
}
|
||||
if capacity.maxPendingPerAuth <= 0 {
|
||||
capacity.maxPendingPerAuth = capacity.maxPending
|
||||
}
|
||||
if capacity.globalMaxEntries <= 0 {
|
||||
capacity.globalMaxEntries = rpcExecutionMaxEntries
|
||||
}
|
||||
if capacity.authMaxEntries <= 0 {
|
||||
capacity.authMaxEntries = capacity.globalMaxEntries
|
||||
}
|
||||
if capacity.sessionMaxEntries <= 0 {
|
||||
capacity.sessionMaxEntries = capacity.authMaxEntries
|
||||
}
|
||||
if capacity.subscriberMaxGlobal <= 0 {
|
||||
capacity.subscriberMaxGlobal = rpcResultSubscriberMaxGlobal
|
||||
}
|
||||
if capacity.subscriberMaxAuth <= 0 {
|
||||
capacity.subscriberMaxAuth = rpcResultSubscriberMaxAuth
|
||||
}
|
||||
if capacity.subscriberMaxSession <= 0 {
|
||||
capacity.subscriberMaxSession = rpcResultSubscriberMaxSession
|
||||
}
|
||||
if capacity.subscriberMaxPerFlight <= 0 {
|
||||
capacity.subscriberMaxPerFlight = rpcResultSubscriberMaxPerFlight
|
||||
}
|
||||
|
||||
l := &rpcExecutionLedger{hashSeed: maphash.MakeSeed(), replayStore: capacity.replayStore}
|
||||
l.reservedEntries.max = int64(capacity.globalMaxEntries)
|
||||
l.flightLimit.max = int64(capacity.maxPending)
|
||||
l.fairBudget = newRPCExecutionFairBudget(
|
||||
l.hashSeed,
|
||||
&l.reservedEntries,
|
||||
int64(capacity.authMaxEntries),
|
||||
int64(capacity.sessionMaxEntries),
|
||||
capacity.maxPendingPerAuth,
|
||||
)
|
||||
l.subscriberBudget = newRPCResultSubscriberBudget(
|
||||
l.hashSeed,
|
||||
capacity.subscriberMaxGlobal,
|
||||
capacity.subscriberMaxAuth,
|
||||
capacity.subscriberMaxSession,
|
||||
)
|
||||
l.subscriberPerFlight = capacity.subscriberMaxPerFlight
|
||||
for i := range l.shards {
|
||||
s := &l.shards[i]
|
||||
s.now = now
|
||||
s.ttl = rpcExecutionReceiptTTL
|
||||
s.receiptCount = &l.receiptCount
|
||||
s.order = list.New()
|
||||
s.byKey = make(map[rpcExecutionKey]*list.Element)
|
||||
s.pending = make(map[rpcExecutionKey]*rpcResultFlight)
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func (l *rpcExecutionLedger) shard(key rpcExecutionKey) *rpcExecutionLedgerShard {
|
||||
return &l.shards[l.shardIndex(key)]
|
||||
}
|
||||
|
||||
func (l *rpcExecutionLedger) shardIndex(key rpcExecutionKey) uint64 {
|
||||
var raw [24]byte
|
||||
copy(raw[:8], key.authKeyID[:])
|
||||
binary.LittleEndian.PutUint64(raw[8:16], uint64(key.sessionID))
|
||||
binary.LittleEndian.PutUint64(raw[16:24], uint64(key.reqMsgID))
|
||||
return maphash.Bytes(l.hashSeed, raw[:]) & (rpcExecutionLedgerShards - 1)
|
||||
}
|
||||
|
||||
// Replay resolves an immutable result descriptor from the logical-session
|
||||
// outbox. A receipt hit without an outbox frame is never treated as permission
|
||||
// to execute the business handler again.
|
||||
func (l *rpcExecutionLedger) Replay(authKeyID [8]byte, sessionID, reqMsgID int64) (*encodedOutboundMessage, bool) {
|
||||
if l == nil || reqMsgID == 0 {
|
||||
return nil, false
|
||||
}
|
||||
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s := l.shard(key)
|
||||
now := s.now()
|
||||
s.mu.Lock()
|
||||
elem := s.byKey[key]
|
||||
if elem == nil {
|
||||
s.mu.Unlock()
|
||||
return nil, false
|
||||
}
|
||||
receipt := elem.Value.(*rpcExecutionReceipt)
|
||||
if !receipt.expiresAt.After(now) {
|
||||
s.removeElement(elem)
|
||||
s.mu.Unlock()
|
||||
return nil, false
|
||||
}
|
||||
if receipt.unavailable || receipt.acknowledged {
|
||||
s.mu.Unlock()
|
||||
return nil, false
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return l.replayStore.rpcResult(authKeyID, sessionID, reqMsgID)
|
||||
}
|
||||
|
||||
// Acknowledge removes a completed receipt immediately. reqMsgID must already
|
||||
// have been resolved from the outbound actor's trusted server-msg-id mapping.
|
||||
// A pending flight records the ACK so a racing completion cannot resurrect a
|
||||
// receipt after the outbox body has been released.
|
||||
func (l *rpcExecutionLedger) Acknowledge(authKeyID [8]byte, sessionID, reqMsgID int64) bool {
|
||||
if l == nil || reqMsgID == 0 {
|
||||
return false
|
||||
}
|
||||
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s := l.shard(key)
|
||||
s.mu.Lock()
|
||||
s.expireLocked(s.now())
|
||||
if elem := s.byKey[key]; elem != nil {
|
||||
s.removeElement(elem)
|
||||
s.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
if flight := s.pending[key]; flight != nil {
|
||||
flight.acknowledged = true
|
||||
s.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
|
||||
// ObserveDependency returns a waiter for an admitted in-flight dependency, a
|
||||
// completed execution outcome, or ok=false for an unknown request. It never
|
||||
// creates execution ownership.
|
||||
func (l *rpcExecutionLedger) ObserveDependency(authKeyID [8]byte, sessionID, reqMsgID int64) (rpcResultDependency, bool) {
|
||||
if l == nil || reqMsgID == 0 {
|
||||
return rpcResultDependency{}, false
|
||||
}
|
||||
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s := l.shard(key)
|
||||
now := s.now()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if elem := s.byKey[key]; elem != nil {
|
||||
receipt := elem.Value.(*rpcExecutionReceipt)
|
||||
if receipt.expiresAt.After(now) {
|
||||
if !receipt.executionKnown {
|
||||
return rpcResultDependency{}, false
|
||||
}
|
||||
return rpcResultDependency{completed: true, success: receipt.executionOK}, true
|
||||
}
|
||||
s.removeElement(elem)
|
||||
}
|
||||
if flight := s.pending[key]; flight != nil {
|
||||
if flight.executionDone {
|
||||
return rpcResultDependency{completed: true, success: flight.executionOK}, true
|
||||
}
|
||||
return rpcResultDependency{waiter: &rpcResultWaiter{ledger: l, key: key, flight: flight}}, true
|
||||
}
|
||||
return rpcResultDependency{}, false
|
||||
}
|
||||
|
||||
// Complete publishes terminal execution metadata and resolves current
|
||||
// waiters. replayable is only a claim from the egress path; the ledger verifies
|
||||
// that replayStore actually owns the exact frame before publishing a replayable
|
||||
// receipt. encoded is passed transiently to joined waiters and is never stored.
|
||||
func (l *rpcExecutionLedger) Complete(
|
||||
authKeyID [8]byte,
|
||||
sessionID, reqMsgID int64,
|
||||
encoded *encodedOutboundMessage,
|
||||
replayable bool,
|
||||
) {
|
||||
if l == nil || reqMsgID == 0 || encoded == nil {
|
||||
return
|
||||
}
|
||||
if replayable {
|
||||
_, replayable = l.replayStore.rpcResult(authKeyID, sessionID, reqMsgID)
|
||||
}
|
||||
if l.completeOnce(authKeyID, sessionID, reqMsgID, encoded, replayable) {
|
||||
return
|
||||
}
|
||||
// An expired receipt in another shard may be the only global blocker.
|
||||
l.expireReceipts()
|
||||
_ = l.completeOnce(authKeyID, sessionID, reqMsgID, encoded, replayable)
|
||||
}
|
||||
|
||||
// completeOnce returns false only when a cross-shard expiry reap may release
|
||||
// the global reservation required by a defensive completion without an owner.
|
||||
func (l *rpcExecutionLedger) completeOnce(
|
||||
authKeyID [8]byte,
|
||||
sessionID, reqMsgID int64,
|
||||
encoded *encodedOutboundMessage,
|
||||
replayable bool,
|
||||
) bool {
|
||||
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s := l.shard(key)
|
||||
s.mu.Lock()
|
||||
now := s.now()
|
||||
s.expireLocked(now)
|
||||
old := s.byKey[key]
|
||||
flight := s.pending[key]
|
||||
var oldReceipt *rpcExecutionReceipt
|
||||
if old != nil {
|
||||
oldReceipt = old.Value.(*rpcExecutionReceipt)
|
||||
if flight == nil {
|
||||
// Terminal publication is immutable. Late callbacks cannot extend TTL,
|
||||
// replace replay identity or resurrect an acknowledged result.
|
||||
s.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
identity, admissionSeq, executionKnown, executionOK, acknowledged := rpcResultFlightMetadataLocked(s, key)
|
||||
var reservation *rpcExecutionBudgetReservation
|
||||
switch {
|
||||
case flight != nil:
|
||||
reservation = flight.reservation
|
||||
if reservation == nil {
|
||||
s.mu.Unlock()
|
||||
panic("mtprotoedge: pending rpc execution has no fair-budget reservation")
|
||||
}
|
||||
case oldReceipt != nil && oldReceipt.reservation != nil:
|
||||
reservation = oldReceipt.reservation
|
||||
default:
|
||||
reservation = l.fairBudget.reserveCompleted(key)
|
||||
if reservation == nil {
|
||||
s.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if old != nil {
|
||||
s.unlinkElement(old)
|
||||
if oldReceipt.reservation != nil && oldReceipt.reservation != reservation {
|
||||
oldReceipt.reservation.release()
|
||||
oldReceipt.reservation = nil
|
||||
}
|
||||
}
|
||||
receipt := &rpcExecutionReceipt{
|
||||
key: key,
|
||||
expiresAt: now.Add(s.ttl),
|
||||
identity: identity,
|
||||
admissionSeq: admissionSeq,
|
||||
executionKnown: executionKnown,
|
||||
executionOK: executionOK,
|
||||
acknowledged: acknowledged,
|
||||
unavailable: !replayable,
|
||||
reservation: reservation,
|
||||
}
|
||||
elem := s.order.PushBack(receipt)
|
||||
s.byKey[key] = elem
|
||||
s.incrementReceiptCount()
|
||||
|
||||
subscribers, executionSubscribers, terminalExecutionOK := l.completeRPCResultFlightLocked(s, key, encoded)
|
||||
if acknowledged {
|
||||
// ACK won before completion. Current subscribers still receive encoded,
|
||||
// but no post-ACK receipt survives this critical section.
|
||||
s.removeElement(elem)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
for _, subscriber := range subscribers {
|
||||
subscriber(encoded, true)
|
||||
}
|
||||
for _, subscriber := range executionSubscribers {
|
||||
subscriber(terminalExecutionOK)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func rpcResultFlightMetadataLocked(s *rpcExecutionLedgerShard, key rpcExecutionKey) (
|
||||
rpcResultRequestIdentity,
|
||||
uint64,
|
||||
bool,
|
||||
bool,
|
||||
bool,
|
||||
) {
|
||||
if flight := s.pending[key]; flight != nil {
|
||||
return flight.identity, flight.admissionSeq, flight.executionDone, flight.executionOK, flight.acknowledged
|
||||
}
|
||||
return rpcResultRequestIdentity{}, 0, false, false, false
|
||||
}
|
||||
|
||||
func (l *rpcExecutionLedger) expireReceipts() {
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
for i := range l.shards {
|
||||
s := &l.shards[i]
|
||||
s.mu.Lock()
|
||||
s.expireLocked(s.now())
|
||||
s.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (l *rpcExecutionLedger) receiptBudgetBytes() int64 {
|
||||
if l == nil {
|
||||
return 0
|
||||
}
|
||||
return l.receiptCount.Load() * rpcExecutionReceiptBudgetBytes
|
||||
}
|
||||
|
||||
func (l *rpcExecutionLedger) Close() error { return nil }
|
||||
|
||||
func (l *rpcExecutionLedger) CloseContext(context.Context) error { return nil }
|
||||
|
||||
// forgetSession removes every terminal receipt for a destroyed logical
|
||||
// session. SessionManager calls it only after physical producers converge.
|
||||
func (l *rpcExecutionLedger) forgetSession(authKeyID [8]byte, sessionID int64) {
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
for i := range l.shards {
|
||||
s := &l.shards[i]
|
||||
s.mu.Lock()
|
||||
for elem := s.order.Front(); elem != nil; {
|
||||
next := elem.Next()
|
||||
receipt := elem.Value.(*rpcExecutionReceipt)
|
||||
if receipt.key.authKeyID == authKeyID && receipt.key.sessionID == sessionID {
|
||||
s.removeElement(elem)
|
||||
}
|
||||
elem = next
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *rpcExecutionLedgerShard) expireLocked(now time.Time) {
|
||||
for elem := s.order.Front(); elem != nil; {
|
||||
next := elem.Next()
|
||||
receipt := elem.Value.(*rpcExecutionReceipt)
|
||||
if receipt.expiresAt.After(now) {
|
||||
return
|
||||
}
|
||||
s.removeElement(elem)
|
||||
elem = next
|
||||
}
|
||||
}
|
||||
|
||||
func (s *rpcExecutionLedgerShard) removeElement(elem *list.Element) {
|
||||
receipt := s.unlinkElement(elem)
|
||||
if receipt != nil && receipt.reservation != nil {
|
||||
receipt.reservation.release()
|
||||
receipt.reservation = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *rpcExecutionLedgerShard) unlinkElement(elem *list.Element) *rpcExecutionReceipt {
|
||||
if elem == nil {
|
||||
return nil
|
||||
}
|
||||
receipt := elem.Value.(*rpcExecutionReceipt)
|
||||
delete(s.byKey, receipt.key)
|
||||
s.order.Remove(elem)
|
||||
s.decrementReceiptCount()
|
||||
return receipt
|
||||
}
|
||||
|
||||
func (s *rpcExecutionLedgerShard) incrementReceiptCount() {
|
||||
if s.receiptCount == nil {
|
||||
panic("mtprotoedge: rpc execution receipt counter is unavailable")
|
||||
}
|
||||
s.receiptCount.Add(1)
|
||||
}
|
||||
|
||||
func (s *rpcExecutionLedgerShard) decrementReceiptCount() {
|
||||
if s.receiptCount == nil || s.receiptCount.Add(-1) < 0 {
|
||||
panic("mtprotoedge: rpc execution receipt counter underflow")
|
||||
}
|
||||
}
|
||||
346
internal/mtprotoedge/rpc_execution_ledger_test.go
Normal file
346
internal/mtprotoedge/rpc_execution_ledger_test.go
Normal file
|
|
@ -0,0 +1,346 @@
|
|||
package mtprotoedge
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
func newRPCExecutionLedgerWithLimitsForTest(
|
||||
now func() time.Time,
|
||||
maxPending, maxPendingPerAuth, global, auth, session int,
|
||||
) *rpcExecutionLedger {
|
||||
return newRPCExecutionLedger(now, rpcExecutionLedgerCapacity{
|
||||
maxPending: maxPending, maxPendingPerAuth: maxPendingPerAuth,
|
||||
globalMaxEntries: global, authMaxEntries: auth, sessionMaxEntries: session,
|
||||
replayStore: newRPCReplayStoreForTest(),
|
||||
})
|
||||
}
|
||||
|
||||
func TestRPCExecutionLedgerSessionCapacityIsolatesAnotherAuth(t *testing.T) {
|
||||
ledger := newRPCExecutionLedgerWithLimitsForTest(time.Now, 8, 6, 8, 6, 2)
|
||||
authA := [8]byte{0xa1}
|
||||
authB := [8]byte{0xb1}
|
||||
for i := 0; i < 2; i++ {
|
||||
msgID := int64(1000 + i)
|
||||
claim, err := ledger.Acquire(authA, 77, msgID)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("same-session admission %d = %#v, %v", i, claim, err)
|
||||
}
|
||||
ledger.completeReplayableForTest(authA, 77, msgID, &encodedOutboundMessage{body: []byte{1}})
|
||||
}
|
||||
if _, err := ledger.Acquire(authA, 77, 2000); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("admission beyond session limit = %v, want capacity", err)
|
||||
}
|
||||
other, err := ledger.Acquire(authB, 88, 3000)
|
||||
if err != nil || other.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("other auth blocked by full session: %#v, %v", other, err)
|
||||
}
|
||||
other.owner.Abort()
|
||||
}
|
||||
|
||||
func TestRPCExecutionLedgerAuthCapacityIsolatesAnotherAuth(t *testing.T) {
|
||||
ledger := newRPCExecutionLedgerWithLimitsForTest(time.Now, 8, 4, 8, 2, 2)
|
||||
authA := [8]byte{0xa2}
|
||||
authB := [8]byte{0xb2}
|
||||
for i := 0; i < 2; i++ {
|
||||
claim, err := ledger.Acquire(authA, int64(10+i), int64(100+i))
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("auth A admission %d = %#v, %v", i, claim, err)
|
||||
}
|
||||
ledger.completeReplayableForTest(authA, int64(10+i), int64(100+i), &encodedOutboundMessage{body: []byte{1}})
|
||||
}
|
||||
if _, err := ledger.Acquire(authA, 12, 102); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("same auth beyond limit = %v, want capacity", err)
|
||||
}
|
||||
other, err := ledger.Acquire(authB, 20, 200)
|
||||
if err != nil || other.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("other auth blocked by auth A: %#v, %v", other, err)
|
||||
}
|
||||
other.owner.Abort()
|
||||
}
|
||||
|
||||
func TestRPCExecutionLedgerPendingLimitIsAdditional(t *testing.T) {
|
||||
ledger := newRPCExecutionLedgerWithLimitsForTest(time.Now, 6, 2, 12, 6, 4)
|
||||
authA := [8]byte{0xa3}
|
||||
authB := [8]byte{0xb3}
|
||||
owners := make([]*rpcResultOwnerLease, 0, 3)
|
||||
for i := 0; i < 2; i++ {
|
||||
claim, err := ledger.Acquire(authA, int64(i+1), int64(100+i))
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("pending auth A %d = %#v, %v", i, claim, err)
|
||||
}
|
||||
owners = append(owners, claim.owner)
|
||||
}
|
||||
if _, err := ledger.Acquire(authA, 3, 103); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("third pending owner for auth A = %v, want capacity", err)
|
||||
}
|
||||
other, err := ledger.Acquire(authB, 4, 104)
|
||||
if err != nil || other.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("auth B blocked by auth A pending limit: %#v, %v", other, err)
|
||||
}
|
||||
owners = append(owners, other.owner)
|
||||
for _, owner := range owners {
|
||||
if !owner.Abort() {
|
||||
t.Fatal("pending owner did not abort")
|
||||
}
|
||||
}
|
||||
if usage := ledger.fairBudget.authSnapshot(authA); usage != (rpcExecutionBudgetUsage{}) {
|
||||
t.Fatalf("auth A budget after abort = %#v", usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCExecutionLedgerReceiptLifecycleACKAndTTL(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
ledger := newRPCExecutionLedgerWithLimitsForTest(func() time.Time { return now }, 4, 4, 6, 5, 3)
|
||||
auth := [8]byte{0xc1}
|
||||
|
||||
claim, err := ledger.Acquire(auth, 1, 101)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("owner = %#v, %v", claim, err)
|
||||
}
|
||||
if usage := ledger.fairBudget.authSnapshot(auth); usage.entries != 1 || usage.pending != 1 {
|
||||
t.Fatalf("pending reservation = %#v", usage)
|
||||
}
|
||||
claim.owner.CompleteExecution(true)
|
||||
ledger.completeReplayableForTest(auth, 1, 101, &encodedOutboundMessage{body: make([]byte, 8<<20)})
|
||||
if ledger.flightLimit.snapshot() != 0 || ledger.receiptCount.Load() != 1 || ledger.reservedEntries.snapshot() != 1 {
|
||||
t.Fatalf("terminal counts owner=%d receipt=%d reserved=%d", ledger.flightLimit.snapshot(), ledger.receiptCount.Load(), ledger.reservedEntries.snapshot())
|
||||
}
|
||||
if got := ledger.receiptBudgetBytes(); got != rpcExecutionReceiptBudgetBytes {
|
||||
t.Fatalf("8 MiB result charged %d receipt bytes, want fixed %d", got, rpcExecutionReceiptBudgetBytes)
|
||||
}
|
||||
if !ledger.Acknowledge(auth, 1, 101) {
|
||||
t.Fatal("ACK did not remove receipt")
|
||||
}
|
||||
if ledger.receiptCount.Load() != 0 || ledger.reservedEntries.snapshot() != 0 || ledger.receiptBudgetBytes() != 0 {
|
||||
t.Fatal("ACK leaked receipt reservation")
|
||||
}
|
||||
|
||||
second, err := ledger.Acquire(auth, 2, 201)
|
||||
if err != nil || second.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("second owner = %#v, %v", second, err)
|
||||
}
|
||||
ledger.completeReplayableForTest(auth, 2, 201, &encodedOutboundMessage{body: []byte{1}})
|
||||
now = now.Add(rpcExecutionReceiptTTL + time.Second)
|
||||
if _, ok := ledger.Replay(auth, 2, 201); ok {
|
||||
t.Fatal("expired receipt remained replayable")
|
||||
}
|
||||
if ledger.receiptCount.Load() != 0 || ledger.reservedEntries.snapshot() != 0 {
|
||||
t.Fatal("TTL leaked receipt reservation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCExecutionLedgerACKBeforeCompleteDoesNotResurrectReceipt(t *testing.T) {
|
||||
ledger := newRPCExecutionLedgerForTest(time.Now, 2)
|
||||
auth := [8]byte{0xd1}
|
||||
claim, err := ledger.Acquire(auth, 1, 101)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("owner = %#v, %v", claim, err)
|
||||
}
|
||||
joined, err := ledger.Acquire(auth, 1, 101)
|
||||
if err != nil || joined.state != rpcResultAcquirePending {
|
||||
t.Fatalf("join = %#v, %v", joined, err)
|
||||
}
|
||||
if !ledger.Acknowledge(auth, 1, 101) {
|
||||
t.Fatal("ACK did not mark pending owner")
|
||||
}
|
||||
want := &encodedOutboundMessage{body: []byte{1}, reqMsgID: 101}
|
||||
ledger.completeReplayableForTest(auth, 1, 101, want)
|
||||
if got, ok, waitErr := joined.waiter.Wait(t.Context()); waitErr != nil || !ok || got != want {
|
||||
t.Fatalf("joined waiter = %p/%v/%v", got, ok, waitErr)
|
||||
}
|
||||
if ledger.receiptCount.Load() != 0 || ledger.reservedEntries.snapshot() != 0 {
|
||||
t.Fatal("ACK-before-complete resurrected receipt")
|
||||
}
|
||||
newClaim, err := ledger.Acquire(auth, 1, 101)
|
||||
if err != nil || newClaim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("post-ACK request did not get a fresh owner: %#v, %v", newClaim, err)
|
||||
}
|
||||
newClaim.owner.Abort()
|
||||
}
|
||||
|
||||
func TestRPCExecutionLedgerUnavailableTombstonePreventsReexecution(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
ledger := newRPCExecutionLedgerWithLimitsForTest(func() time.Time { return now }, 2, 2, 4, 4, 4)
|
||||
auth := [8]byte{0xe1}
|
||||
claim, err := ledger.Acquire(auth, 1, 101)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("owner = %#v, %v", claim, err)
|
||||
}
|
||||
claim.owner.CompleteExecution(true)
|
||||
ledger.Complete(auth, 1, 101, &encodedOutboundMessage{body: make([]byte, 8<<20)}, false)
|
||||
if _, ok := ledger.Replay(auth, 1, 101); ok {
|
||||
t.Fatal("unavailable tombstone masqueraded as replayable")
|
||||
}
|
||||
if _, err := ledger.Acquire(auth, 1, 101); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("duplicate after unavailable completion = %v, want capacity", err)
|
||||
}
|
||||
if got := ledger.receiptBudgetBytes(); got != rpcExecutionReceiptBudgetBytes {
|
||||
t.Fatalf("unavailable receipt budget = %d", got)
|
||||
}
|
||||
now = now.Add(rpcExecutionReceiptTTL + time.Second)
|
||||
retry, err := ledger.Acquire(auth, 1, 101)
|
||||
if err != nil || retry.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("admission after tombstone expiry = %#v, %v", retry, err)
|
||||
}
|
||||
retry.owner.Abort()
|
||||
}
|
||||
|
||||
func TestRPCExecutionLedgerConcurrentReservationsNeverOvercommit(t *testing.T) {
|
||||
const limit = 24
|
||||
ledger := newRPCExecutionLedgerWithLimitsForTest(time.Now, limit, 4, limit, 8, 3)
|
||||
const callers = 256
|
||||
start := make(chan struct{})
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
owners []*rpcResultOwnerLease
|
||||
)
|
||||
for i := 0; i < callers; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
auth := [8]byte{byte(i % 4)}
|
||||
claim, err := ledger.Acquire(auth, int64(i%8), int64(1000+i))
|
||||
if errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
return
|
||||
}
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Errorf("Acquire %d = %#v, %v", i, claim, err)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
owners = append(owners, claim.owner)
|
||||
mu.Unlock()
|
||||
}(i)
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
if got := ledger.reservedEntries.snapshot(); got > limit || got != int64(len(owners)) {
|
||||
t.Fatalf("reserved=%d owners=%d limit=%d", got, len(owners), limit)
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
auth := [8]byte{byte(i)}
|
||||
usage := ledger.fairBudget.authSnapshot(auth)
|
||||
if usage.entries > 8 || usage.pending > 4 {
|
||||
t.Fatalf("auth %d overcommitted: %#v", i, usage)
|
||||
}
|
||||
}
|
||||
for _, owner := range owners {
|
||||
owner.Abort()
|
||||
}
|
||||
if ledger.reservedEntries.snapshot() != 0 {
|
||||
t.Fatal("concurrent abort leaked reservations")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCExecutionLedgerFullKeyHashSpreadsOneSession(t *testing.T) {
|
||||
first := newRPCExecutionLedgerForTest(time.Now, 64)
|
||||
second := newRPCExecutionLedgerForTest(time.Now, 64)
|
||||
auth := [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
|
||||
seen := make(map[uint64]struct{})
|
||||
differentInstance := false
|
||||
for msgID := int64(1); msgID <= 256; msgID++ {
|
||||
key := rpcExecutionKey{authKeyID: auth, sessionID: 99, reqMsgID: msgID}
|
||||
firstIndex := first.shardIndex(key)
|
||||
seen[firstIndex] = struct{}{}
|
||||
if firstIndex != second.shardIndex(key) {
|
||||
differentInstance = true
|
||||
}
|
||||
}
|
||||
if len(seen) < rpcExecutionLedgerShards/2 {
|
||||
t.Fatalf("one session used only %d/%d shards", len(seen), rpcExecutionLedgerShards)
|
||||
}
|
||||
if !differentInstance {
|
||||
t.Fatal("two ledger instances used an identical shard stream")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCExecutionLedgerForgetSessionReleasesReceipts(t *testing.T) {
|
||||
ledger := newRPCExecutionLedgerForTest(time.Now, 8)
|
||||
auth := [8]byte{0xf1}
|
||||
for _, sessionID := range []int64{1, 1, 2} {
|
||||
msgID := int64(100 + ledger.receiptCount.Load())
|
||||
claim, err := ledger.Acquire(auth, sessionID, msgID)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("owner session=%d: %#v, %v", sessionID, claim, err)
|
||||
}
|
||||
ledger.completeReplayableForTest(auth, sessionID, msgID, &encodedOutboundMessage{body: []byte{1}})
|
||||
}
|
||||
ledger.forgetSession(auth, 1)
|
||||
if got := ledger.receiptCount.Load(); got != 1 {
|
||||
t.Fatalf("receipts after session forget = %d, want 1", got)
|
||||
}
|
||||
if _, ok := ledger.Replay(auth, 1, 100); ok {
|
||||
t.Fatal("forgotten session remained replayable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCExecutionLedgerServerOptionsPropagateLimits(t *testing.T) {
|
||||
s := New(Options{
|
||||
RPCGlobalMaxTasks: 6,
|
||||
RPCExecutionMaxEntries: 12,
|
||||
RPCExecutionAuthMaxEntries: 8,
|
||||
RPCExecutionSessionMaxEntries: 4,
|
||||
RPCExecutionPendingPerAuth: 3,
|
||||
})
|
||||
if s.rpcResults.reservedEntries.max != 12 {
|
||||
t.Fatalf("global option propagation = %d", s.rpcResults.reservedEntries.max)
|
||||
}
|
||||
budget := s.rpcResults.fairBudget
|
||||
if budget.authLimit != 8 || budget.sessionLimit != 4 || budget.pendingPerAuth != 3 {
|
||||
t.Fatalf("fair option propagation = auth:%d session:%d pending:%d", budget.authLimit, budget.sessionLimit, budget.pendingPerAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCExecutionLedgerServerOptionsFailFast(t *testing.T) {
|
||||
base := Options{
|
||||
RPCGlobalMaxTasks: 6,
|
||||
RPCExecutionMaxEntries: 12,
|
||||
RPCExecutionAuthMaxEntries: 8,
|
||||
RPCExecutionSessionMaxEntries: 4,
|
||||
RPCExecutionPendingPerAuth: 3,
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*Options)
|
||||
}{
|
||||
{name: "entry hierarchy", mutate: func(o *Options) { o.RPCExecutionAuthMaxEntries = 13 }},
|
||||
{name: "pending hierarchy", mutate: func(o *Options) { o.RPCExecutionPendingPerAuth = 7 }},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
opts := base
|
||||
test.mutate(&opts)
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("New accepted invalid rpc execution options")
|
||||
}
|
||||
}()
|
||||
_ = New(opts)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCExecutionLedgerRequiresReplayStore(t *testing.T) {
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("ledger accepted nil replay store")
|
||||
}
|
||||
}()
|
||||
_ = newRPCExecutionLedger(time.Now, rpcExecutionLedgerCapacity{})
|
||||
}
|
||||
|
||||
func TestRPCExecutionReceiptBudgetCoversOwnedFixedStructures(t *testing.T) {
|
||||
fixed := unsafe.Sizeof(rpcExecutionReceipt{}) +
|
||||
unsafe.Sizeof(list.Element{}) +
|
||||
unsafe.Sizeof(rpcExecutionBudgetReservation{})
|
||||
if fixed > rpcExecutionReceiptBudgetBytes {
|
||||
t.Fatalf("fixed receipt structures use %d bytes, budget charge is %d", fixed, rpcExecutionReceiptBudgetBytes)
|
||||
}
|
||||
}
|
||||
|
|
@ -23,7 +23,7 @@ var (
|
|||
)
|
||||
|
||||
// rpcResultIdentityMismatchError carries the winner's immutable admission
|
||||
// profile from inside the cache shard critical section. A replacement Conn can
|
||||
// profile from inside the ledger shard critical section. A replacement Conn can
|
||||
// re-decode the same naked body under that grammar even if the winner aborts
|
||||
// immediately after the mismatch is returned.
|
||||
type rpcResultIdentityMismatchError struct {
|
||||
|
|
@ -51,8 +51,8 @@ type rpcResultRequestIdentity struct {
|
|||
|
||||
func (i rpcResultRequestIdentity) matches(requested rpcResultRequestIdentity) bool {
|
||||
if !requested.valid {
|
||||
// Legacy service/test callers carry no API request identity and preserve
|
||||
// the historical cache lookup behavior. Exact callers must always match.
|
||||
// Service-message callers carry no API request identity. Exact API callers
|
||||
// must always match the winner's full prepared identity.
|
||||
return true
|
||||
}
|
||||
return i.valid && i.exact == requested.exact
|
||||
|
|
@ -72,10 +72,10 @@ const (
|
|||
//
|
||||
// Exactly one state-specific field is non-nil:
|
||||
// - completed: encoded contains the immutable completed rpc_result;
|
||||
// - acknowledged: the client ACKed the completed rpc_result, so only the
|
||||
// compact request receipt remains and the duplicate must be ACK-only;
|
||||
// - acknowledged: ACK won while the owner was still pending, so the duplicate
|
||||
// must be ACK-only until that owner completes;
|
||||
// - pending: waiter joins the already-running owner;
|
||||
// - owner: owner must eventually complete through rpcResultCache.Put or Abort.
|
||||
// - owner: owner must eventually complete through ledger Complete or Abort.
|
||||
type rpcResultAcquire struct {
|
||||
state rpcResultAcquireState
|
||||
admissionSeq uint64
|
||||
|
|
@ -87,7 +87,7 @@ type rpcResultAcquire struct {
|
|||
}
|
||||
|
||||
// rpcResultFlight is not part of the completed receipt TTL lifecycle. Its
|
||||
// done channel is closed exactly once while holding the owning cache shard lock;
|
||||
// done channel is closed exactly once while holding the owning ledger shard lock;
|
||||
// channel close publishes encoded/ok to all waiters without a waiter goroutine.
|
||||
type rpcResultFlight struct {
|
||||
done chan struct{}
|
||||
|
|
@ -98,9 +98,9 @@ type rpcResultFlight struct {
|
|||
executionOK bool
|
||||
executionSubscribers []func(bool)
|
||||
// acknowledged is set only from the sole outbound actor's server-msg-id to
|
||||
// request-msg-id mapping. It can win the race with the asynchronous Put;
|
||||
// publication then keeps only the compact receipt while still resolving
|
||||
// waiters with the immutable result that was already written on the wire.
|
||||
// request-msg-id mapping. It can win the race with asynchronous completion;
|
||||
// publication then drops the receipt while still resolving waiters with the
|
||||
// immutable result that was already written on the wire.
|
||||
acknowledged bool
|
||||
// subscriberSlots counts callbacks retained by this pending flight. Result
|
||||
// and execution callbacks are charged independently; a replay alias installs
|
||||
|
|
@ -108,18 +108,18 @@ type rpcResultFlight struct {
|
|||
subscriberSlots int
|
||||
identity rpcResultRequestIdentity
|
||||
admissionSeq uint64
|
||||
// reservation owns one entry and at least one byte at global, raw-auth and
|
||||
// session scopes. Put transfers it to a result/tombstone; Abort releases it.
|
||||
reservation *rpcResultBudgetReservation
|
||||
// reservation owns one entry at global, raw-auth and session scopes.
|
||||
// Complete transfers it to a receipt/tombstone; Abort releases it.
|
||||
reservation *rpcExecutionBudgetReservation
|
||||
}
|
||||
|
||||
type rpcResultWaiter struct {
|
||||
cache *rpcResultCache
|
||||
key rpcResultCacheKey
|
||||
ledger *rpcExecutionLedger
|
||||
key rpcExecutionKey
|
||||
flight *rpcResultFlight
|
||||
}
|
||||
|
||||
// Wait blocks until the owner publishes through Put, aborts, or ctx expires.
|
||||
// Wait blocks until the owner publishes through Complete, aborts, or ctx expires.
|
||||
// ok=false with err=nil means the owner aborted without a result.
|
||||
func (w *rpcResultWaiter) Wait(ctx context.Context) (encoded *encodedOutboundMessage, ok bool, err error) {
|
||||
if w == nil || w.flight == nil || ctx == nil {
|
||||
|
|
@ -148,7 +148,7 @@ func (w *rpcResultWaiter) Wait(ctx context.Context) (encoded *encodedOutboundMes
|
|||
}
|
||||
|
||||
// Subscribe registers an event callback without creating a goroutine or
|
||||
// occupying an RPC worker. The callback is invoked after the cache shard lock is
|
||||
// occupying an RPC worker. The callback is invoked after the ledger shard lock is
|
||||
// released; it must remain non-blocking.
|
||||
func (w *rpcResultWaiter) Subscribe(fn func(*encodedOutboundMessage, bool)) error {
|
||||
if fn == nil {
|
||||
|
|
@ -186,10 +186,10 @@ func (w *rpcResultWaiter) subscribe(
|
|||
resultFn func(*encodedOutboundMessage, bool),
|
||||
executionFn func(bool),
|
||||
) error {
|
||||
if w == nil || w.cache == nil || w.flight == nil || (resultFn == nil && executionFn == nil) {
|
||||
if w == nil || w.ledger == nil || w.flight == nil || (resultFn == nil && executionFn == nil) {
|
||||
return ErrRPCResultFlightInvalid
|
||||
}
|
||||
s := w.cache.shard(w.key)
|
||||
s := w.ledger.shard(w.key)
|
||||
var (
|
||||
encoded *encodedOutboundMessage
|
||||
resultOK bool
|
||||
|
|
@ -207,8 +207,8 @@ func (w *rpcResultWaiter) subscribe(
|
|||
slots++
|
||||
}
|
||||
if slots > 0 {
|
||||
if flight.subscriberSlots > w.cache.subscriberPerFlight-slots ||
|
||||
!w.cache.subscriberBudget.reserve(w.key, slots) {
|
||||
if flight.subscriberSlots > w.ledger.subscriberPerFlight-slots ||
|
||||
!w.ledger.subscriberBudget.reserve(w.key, slots) {
|
||||
s.mu.Unlock()
|
||||
return ErrRPCResultSubscriberCapacity
|
||||
}
|
||||
|
|
@ -256,8 +256,8 @@ func (w *rpcResultWaiter) subscribe(
|
|||
}
|
||||
|
||||
type rpcResultOwnerLease struct {
|
||||
cache *rpcResultCache
|
||||
key rpcResultCacheKey
|
||||
ledger *rpcExecutionLedger
|
||||
key rpcExecutionKey
|
||||
flight *rpcResultFlight
|
||||
delivery *rpcResultDelivery
|
||||
hookMu sync.Mutex
|
||||
|
|
@ -278,13 +278,13 @@ func (l *rpcResultOwnerLease) SetAbortHook(fn func()) {
|
|||
}
|
||||
|
||||
// InstallAbortHook installs fn only while this lease still owns the pending
|
||||
// flight. The shard lock linearizes installation with Abort/Put so a registry
|
||||
// flight. The shard lock linearizes installation with Abort/Complete so a registry
|
||||
// cannot publish a candidate after its owner has already disappeared.
|
||||
func (l *rpcResultOwnerLease) InstallAbortHook(fn func()) bool {
|
||||
if l == nil || l.cache == nil || l.flight == nil || fn == nil {
|
||||
if l == nil || l.ledger == nil || l.flight == nil || fn == nil {
|
||||
return false
|
||||
}
|
||||
s := l.cache.shard(l.key)
|
||||
s := l.ledger.shard(l.key)
|
||||
s.mu.Lock()
|
||||
flight, ok := s.pending[l.key]
|
||||
if !ok || flight != l.flight {
|
||||
|
|
@ -299,10 +299,10 @@ func (l *rpcResultOwnerLease) InstallAbortHook(fn func()) bool {
|
|||
}
|
||||
|
||||
func (l *rpcResultOwnerLease) Waiter() *rpcResultWaiter {
|
||||
if l == nil || l.cache == nil || l.flight == nil {
|
||||
if l == nil || l.ledger == nil || l.flight == nil {
|
||||
return nil
|
||||
}
|
||||
return &rpcResultWaiter{cache: l.cache, key: l.key, flight: l.flight}
|
||||
return &rpcResultWaiter{ledger: l.ledger, key: l.key, flight: l.flight}
|
||||
}
|
||||
|
||||
func (l *rpcResultOwnerLease) TryRetarget(reqMsgID int64) bool {
|
||||
|
|
@ -318,13 +318,13 @@ func (l *rpcResultOwnerLease) Delivery() *rpcResultDelivery {
|
|||
|
||||
// CompleteExecution publishes the handler outcome exactly once while this
|
||||
// lease still owns the flight. success=false includes RPC errors, internal
|
||||
// failures and dependency failures. Delivery/cache completion remains a
|
||||
// failures and dependency failures. Delivery/ledger completion remains a
|
||||
// separate later transition.
|
||||
func (l *rpcResultOwnerLease) CompleteExecution(success bool) bool {
|
||||
if l == nil || l.cache == nil || l.flight == nil {
|
||||
if l == nil || l.ledger == nil || l.flight == nil {
|
||||
return false
|
||||
}
|
||||
s := l.cache.shard(l.key)
|
||||
s := l.ledger.shard(l.key)
|
||||
s.mu.Lock()
|
||||
flight, ok := s.pending[l.key]
|
||||
if !ok || flight != l.flight || flight.executionDone {
|
||||
|
|
@ -335,7 +335,7 @@ func (l *rpcResultOwnerLease) CompleteExecution(success bool) bool {
|
|||
flight.executionOK = success
|
||||
subscribers := append([]func(bool){}, flight.executionSubscribers...)
|
||||
flight.executionSubscribers = nil
|
||||
l.cache.releaseFlightSubscriberSlotsLocked(l.key, flight, len(subscribers))
|
||||
l.ledger.releaseFlightSubscriberSlotsLocked(l.key, flight, len(subscribers))
|
||||
s.mu.Unlock()
|
||||
for _, subscriber := range subscribers {
|
||||
subscriber(success)
|
||||
|
|
@ -345,12 +345,12 @@ func (l *rpcResultOwnerLease) CompleteExecution(success bool) bool {
|
|||
|
||||
// HandOff transfers completion responsibility from the inbound RPC task to an
|
||||
// already-admitted egress operation. The egress terminal callback must resolve
|
||||
// the flight through Put on both successful delivery and fenced failure.
|
||||
// the flight through Complete on both successful delivery and fenced failure.
|
||||
func (l *rpcResultOwnerLease) HandOff() bool {
|
||||
if l == nil || l.cache == nil || l.flight == nil {
|
||||
if l == nil || l.ledger == nil || l.flight == nil {
|
||||
return false
|
||||
}
|
||||
s := l.cache.shard(l.key)
|
||||
s := l.ledger.shard(l.key)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
flight, ok := s.pending[l.key]
|
||||
|
|
@ -365,13 +365,13 @@ func (l *rpcResultOwnerLease) HandOff() bool {
|
|||
// result. Pointer identity prevents an old lease from deleting a later owner
|
||||
// that reacquired the same key. It returns true only for the winning abort.
|
||||
func (l *rpcResultOwnerLease) Abort() bool {
|
||||
if l == nil || l.cache == nil || l.flight == nil {
|
||||
if l == nil || l.ledger == nil || l.flight == nil {
|
||||
return false
|
||||
}
|
||||
if l.handedOff.Load() {
|
||||
return false
|
||||
}
|
||||
s := l.cache.shard(l.key)
|
||||
s := l.ledger.shard(l.key)
|
||||
s.mu.Lock()
|
||||
if l.handedOff.Load() {
|
||||
s.mu.Unlock()
|
||||
|
|
@ -388,13 +388,13 @@ func (l *rpcResultOwnerLease) Abort() bool {
|
|||
flight.reservation.release()
|
||||
flight.reservation = nil
|
||||
}
|
||||
l.cache.flightLimit.release()
|
||||
l.cache.activeAdmissions.retire(flight.admissionSeq)
|
||||
l.ledger.flightLimit.release()
|
||||
l.ledger.activeAdmissions.retire(flight.admissionSeq)
|
||||
subscribers := append([]func(*encodedOutboundMessage, bool){}, flight.subscribers...)
|
||||
flight.subscribers = nil
|
||||
executionSubscribers := append([]func(bool){}, flight.executionSubscribers...)
|
||||
flight.executionSubscribers = nil
|
||||
l.cache.releaseFlightSubscriberSlotsLocked(
|
||||
l.ledger.releaseFlightSubscriberSlotsLocked(
|
||||
l.key, flight, len(subscribers)+len(executionSubscribers),
|
||||
)
|
||||
if !flight.executionDone {
|
||||
|
|
@ -458,7 +458,7 @@ func (l *rpcResultFlightLimit) releaseN(delta int64) {
|
|||
panic("mtproto rpc result counter release must be positive")
|
||||
}
|
||||
if remaining := l.used.Add(-delta); remaining < 0 {
|
||||
// Put/Abort use map removal and lease identity to make double release
|
||||
// Complete/Abort use map removal and lease identity to make double release
|
||||
// impossible. Fail fast instead of masking a capacity-accounting bug that
|
||||
// could otherwise admit more owners than the configured hard limit.
|
||||
panic("mtproto rpc result in-flight counter underflow")
|
||||
|
|
@ -475,29 +475,29 @@ func (l *rpcResultFlightLimit) snapshot() int64 {
|
|||
// Acquire atomically returns a completed result, joins the existing in-flight
|
||||
// owner, or installs the unique owner lease. Pending entries have a separate
|
||||
// lifecycle from completed receipt TTL but reserve the same global/auth/session
|
||||
// ownership that Put later transfers to a completed result or tombstone.
|
||||
func (c *rpcResultCache) Acquire(authKeyID [8]byte, sessionID, reqMsgID int64) (rpcResultAcquire, error) {
|
||||
return c.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{})
|
||||
// ownership that Complete later transfers to a receipt or tombstone.
|
||||
func (l *rpcExecutionLedger) Acquire(authKeyID [8]byte, sessionID, reqMsgID int64) (rpcResultAcquire, error) {
|
||||
return l.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{})
|
||||
}
|
||||
|
||||
func (c *rpcResultCache) AcquireIdentified(
|
||||
func (l *rpcExecutionLedger) AcquireIdentified(
|
||||
authKeyID [8]byte,
|
||||
sessionID, reqMsgID int64,
|
||||
identity tlprofile.PreparedIdentity,
|
||||
) (rpcResultAcquire, error) {
|
||||
return c.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{exact: identity, valid: true})
|
||||
return l.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{exact: identity, valid: true})
|
||||
}
|
||||
|
||||
// AcquireLayerIdentified is the production exact-RPC claim. In addition to the
|
||||
// immutable full request identity it retains the admission profile required to
|
||||
// decode a later same-msg_id naked replay under its original grammar.
|
||||
func (c *rpcResultCache) AcquireLayerIdentified(
|
||||
func (l *rpcExecutionLedger) AcquireLayerIdentified(
|
||||
authKeyID [8]byte,
|
||||
sessionID, reqMsgID int64,
|
||||
profile tlprofile.Profile,
|
||||
identity tlprofile.PreparedIdentity,
|
||||
) (rpcResultAcquire, error) {
|
||||
return c.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{
|
||||
return l.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{
|
||||
exact: identity, profile: profile, valid: true,
|
||||
})
|
||||
}
|
||||
|
|
@ -506,17 +506,17 @@ func (c *rpcResultCache) AcquireLayerIdentified(
|
|||
// owner/result. It does not create or join a flight. Callers still perform
|
||||
// AcquireLayerIdentified after decode, which atomically rejects a same-msg_id
|
||||
// body change by comparing the full prepared identity.
|
||||
func (c *rpcResultCache) ExactAdmissionProfile(authKeyID [8]byte, sessionID, reqMsgID int64) (tlprofile.Profile, bool) {
|
||||
if c == nil || reqMsgID == 0 {
|
||||
func (l *rpcExecutionLedger) ExactAdmissionProfile(authKeyID [8]byte, sessionID, reqMsgID int64) (tlprofile.Profile, bool) {
|
||||
if l == nil || reqMsgID == 0 {
|
||||
return 0, false
|
||||
}
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s := c.shard(key)
|
||||
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s := l.shard(key)
|
||||
now := s.now()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if elem := s.byKey[key]; elem != nil {
|
||||
entry := elem.Value.(*rpcResultCacheEntry)
|
||||
entry := elem.Value.(*rpcExecutionReceipt)
|
||||
if entry.expiresAt.After(now) {
|
||||
if entry.identity.valid && entry.identity.profile != 0 {
|
||||
return entry.identity.profile, true
|
||||
|
|
@ -531,16 +531,16 @@ func (c *rpcResultCache) ExactAdmissionProfile(authKeyID [8]byte, sessionID, req
|
|||
return 0, false
|
||||
}
|
||||
|
||||
func (c *rpcResultCache) acquire(
|
||||
func (l *rpcExecutionLedger) acquire(
|
||||
authKeyID [8]byte,
|
||||
sessionID, reqMsgID int64,
|
||||
identity rpcResultRequestIdentity,
|
||||
) (rpcResultAcquire, error) {
|
||||
if c == nil || reqMsgID == 0 {
|
||||
if l == nil || reqMsgID == 0 {
|
||||
return rpcResultAcquire{}, ErrRPCResultFlightInvalid
|
||||
}
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s := c.shard(key)
|
||||
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s := l.shard(key)
|
||||
reclaimedExpired := false
|
||||
for {
|
||||
now := s.now()
|
||||
|
|
@ -548,7 +548,7 @@ func (c *rpcResultCache) acquire(
|
|||
s.expireLocked(now)
|
||||
|
||||
if elem, ok := s.byKey[key]; ok {
|
||||
entry := elem.Value.(*rpcResultCacheEntry)
|
||||
entry := elem.Value.(*rpcExecutionReceipt)
|
||||
if !entry.identity.matches(identity) {
|
||||
s.mu.Unlock()
|
||||
return rpcResultAcquire{}, identityMismatch(entry.identity)
|
||||
|
|
@ -561,34 +561,22 @@ func (c *rpcResultCache) acquire(
|
|||
s.mu.Unlock()
|
||||
return result, nil
|
||||
}
|
||||
if entry.capacity {
|
||||
if entry.unavailable {
|
||||
s.mu.Unlock()
|
||||
return rpcResultAcquire{}, ErrRPCResultFlightCapacity
|
||||
}
|
||||
if c.sessions != nil {
|
||||
result := rpcResultAcquire{
|
||||
state: rpcResultAcquireCompleted, admissionSeq: entry.admissionSeq,
|
||||
executionKnown: entry.executionKnown, executionOK: entry.executionOK,
|
||||
}
|
||||
s.mu.Unlock()
|
||||
encoded, replayable := c.sessions.rpcResult(authKeyID, sessionID, reqMsgID)
|
||||
encoded, replayable := l.replayStore.rpcResult(authKeyID, sessionID, reqMsgID)
|
||||
if !replayable {
|
||||
return rpcResultAcquire{}, ErrRPCResultReplayUnavailable
|
||||
}
|
||||
result.encoded = encoded
|
||||
return result, nil
|
||||
}
|
||||
if entry.encoded == nil {
|
||||
s.mu.Unlock()
|
||||
return rpcResultAcquire{}, ErrRPCResultReplayUnavailable
|
||||
}
|
||||
result := rpcResultAcquire{
|
||||
state: rpcResultAcquireCompleted, admissionSeq: entry.admissionSeq, encoded: entry.encoded,
|
||||
executionKnown: entry.executionKnown, executionOK: entry.executionOK,
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return result, nil
|
||||
}
|
||||
if flight, ok := s.pending[key]; ok {
|
||||
if !flight.identity.matches(identity) {
|
||||
s.mu.Unlock()
|
||||
|
|
@ -605,22 +593,18 @@ func (c *rpcResultCache) acquire(
|
|||
result := rpcResultAcquire{
|
||||
state: rpcResultAcquirePending,
|
||||
admissionSeq: flight.admissionSeq,
|
||||
waiter: &rpcResultWaiter{cache: c, key: key, flight: flight},
|
||||
waiter: &rpcResultWaiter{ledger: l, key: key, flight: flight},
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return result, nil
|
||||
}
|
||||
if s.maxEntries > 0 && len(s.byKey)+len(s.pending) >= s.maxEntries {
|
||||
if !l.flightLimit.reserve() {
|
||||
s.mu.Unlock()
|
||||
return rpcResultAcquire{}, ErrRPCResultFlightCapacity
|
||||
}
|
||||
if !c.flightLimit.reserve() {
|
||||
s.mu.Unlock()
|
||||
return rpcResultAcquire{}, ErrRPCResultFlightCapacity
|
||||
}
|
||||
reservation := c.fairBudget.reserveOwner(key)
|
||||
reservation := l.fairBudget.reserveOwner(key)
|
||||
if reservation == nil {
|
||||
c.flightLimit.release()
|
||||
l.flightLimit.release()
|
||||
s.mu.Unlock()
|
||||
if reclaimedExpired {
|
||||
return rpcResultAcquire{}, ErrRPCResultFlightCapacity
|
||||
|
|
@ -628,17 +612,17 @@ func (c *rpcResultCache) acquire(
|
|||
// Expired rows in another full-key shard may be the only consumers at
|
||||
// the global, auth or session scope. Reap once, then retry every identity
|
||||
// and capacity check because another goroutine may have won this key.
|
||||
c.expireCompletedResults()
|
||||
l.expireReceipts()
|
||||
reclaimedExpired = true
|
||||
continue
|
||||
}
|
||||
var admissionSeq uint64
|
||||
if identity.valid {
|
||||
var err error
|
||||
admissionSeq, err = c.activeAdmissions.allocateAndRegister(&c.nextAdmissionSeq)
|
||||
admissionSeq, err = l.activeAdmissions.allocateAndRegister(&l.nextAdmissionSeq)
|
||||
if err != nil {
|
||||
reservation.release()
|
||||
c.flightLimit.release()
|
||||
l.flightLimit.release()
|
||||
s.mu.Unlock()
|
||||
return rpcResultAcquire{}, err
|
||||
}
|
||||
|
|
@ -648,7 +632,7 @@ func (c *rpcResultCache) acquire(
|
|||
reservation: reservation,
|
||||
}
|
||||
if s.pending == nil {
|
||||
s.pending = make(map[rpcResultCacheKey]*rpcResultFlight)
|
||||
s.pending = make(map[rpcExecutionKey]*rpcResultFlight)
|
||||
}
|
||||
s.pending[key] = flight
|
||||
s.mu.Unlock()
|
||||
|
|
@ -656,7 +640,7 @@ func (c *rpcResultCache) acquire(
|
|||
state: rpcResultAcquireOwner,
|
||||
admissionSeq: admissionSeq,
|
||||
owner: &rpcResultOwnerLease{
|
||||
cache: c, key: key, flight: flight, delivery: newRPCResultDelivery(reqMsgID),
|
||||
ledger: l, key: key, flight: flight, delivery: newRPCResultDelivery(reqMsgID),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
|
@ -664,16 +648,16 @@ func (c *rpcResultCache) acquire(
|
|||
|
||||
// completeRPCResultFlightLocked publishes encoded to the current owner claim.
|
||||
// The caller must hold s.mu and must publish the completed receipt first.
|
||||
func (c *rpcResultCache) completeRPCResultFlightLocked(
|
||||
s *rpcResultCacheShard,
|
||||
key rpcResultCacheKey,
|
||||
func (l *rpcExecutionLedger) completeRPCResultFlightLocked(
|
||||
s *rpcExecutionLedgerShard,
|
||||
key rpcExecutionKey,
|
||||
encoded *encodedOutboundMessage,
|
||||
) (
|
||||
[]func(*encodedOutboundMessage, bool),
|
||||
[]func(bool),
|
||||
bool,
|
||||
) {
|
||||
if c == nil || s == nil || encoded == nil {
|
||||
if l == nil || s == nil || encoded == nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
flight, ok := s.pending[key]
|
||||
|
|
@ -688,8 +672,8 @@ func (c *rpcResultCache) completeRPCResultFlightLocked(
|
|||
}
|
||||
flight.encoded = encoded
|
||||
flight.ok = true
|
||||
c.flightLimit.release()
|
||||
c.activeAdmissions.retire(flight.admissionSeq)
|
||||
l.flightLimit.release()
|
||||
l.activeAdmissions.retire(flight.admissionSeq)
|
||||
subscribers := append([]func(*encodedOutboundMessage, bool){}, flight.subscribers...)
|
||||
flight.subscribers = nil
|
||||
executionSubscribers := append([]func(bool){}, flight.executionSubscribers...)
|
||||
|
|
@ -697,13 +681,13 @@ func (c *rpcResultCache) completeRPCResultFlightLocked(
|
|||
executionOK := flight.executionOK
|
||||
if !flight.executionDone {
|
||||
// A result without an explicit handler-completion proof cannot satisfy an
|
||||
// invokeAfter dependency. Production completes execution before Put; this
|
||||
// invokeAfter dependency. Production completes execution before Complete;
|
||||
// branch is the conservative terminal cleanup for defensive callers.
|
||||
flight.executionDone = true
|
||||
flight.executionOK = false
|
||||
executionOK = false
|
||||
}
|
||||
c.releaseFlightSubscriberSlotsLocked(
|
||||
l.releaseFlightSubscriberSlotsLocked(
|
||||
key, flight, len(subscribers)+len(executionSubscribers),
|
||||
)
|
||||
close(flight.done)
|
||||
|
|
@ -711,19 +695,19 @@ func (c *rpcResultCache) completeRPCResultFlightLocked(
|
|||
}
|
||||
|
||||
// releaseFlightSubscriberSlotsLocked releases callbacks detached from a flight.
|
||||
// The caller holds that flight's cache shard lock, preserving the only lock
|
||||
// The caller holds that flight's ledger shard lock, preserving the only lock
|
||||
// order used by subscription: shard -> subscriber budget.
|
||||
func (c *rpcResultCache) releaseFlightSubscriberSlotsLocked(
|
||||
key rpcResultCacheKey,
|
||||
func (l *rpcExecutionLedger) releaseFlightSubscriberSlotsLocked(
|
||||
key rpcExecutionKey,
|
||||
flight *rpcResultFlight,
|
||||
slots int,
|
||||
) {
|
||||
if slots == 0 {
|
||||
return
|
||||
}
|
||||
if c == nil || flight == nil || slots < 0 || flight.subscriberSlots < slots {
|
||||
if l == nil || flight == nil || slots < 0 || flight.subscriberSlots < slots {
|
||||
panic("mtproto rpc result subscriber slot underflow")
|
||||
}
|
||||
flight.subscriberSlots -= slots
|
||||
c.subscriberBudget.release(key, slots)
|
||||
l.subscriberBudget.release(key, slots)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,25 +32,23 @@ func rpcFlightExactIdentity(t *testing.T, profile tlprofile.Profile, request bin
|
|||
return admitted.Prepared().Identity()
|
||||
}
|
||||
|
||||
func newRPCResultSubscriberTestCache(global, auth, session, perFlight int) *rpcResultCache {
|
||||
return newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
|
||||
func newRPCExecutionSubscriberTestLedger(global, auth, session, perFlight int) *rpcExecutionLedger {
|
||||
return newRPCExecutionLedger(time.Now, rpcExecutionLedgerCapacity{
|
||||
maxPending: 64,
|
||||
maxPendingPerAuth: 64,
|
||||
globalMaxBytes: rpcResultCacheMaxBytes,
|
||||
globalMaxEntries: rpcResultCacheMaxEntries,
|
||||
authMaxBytes: rpcResultCacheAuthMaxBytes,
|
||||
authMaxEntries: rpcResultCacheAuthMaxEntries,
|
||||
sessionMaxBytes: rpcResultCacheSessionMaxBytes,
|
||||
sessionMaxEntries: rpcResultCacheSessionMaxEntries,
|
||||
globalMaxEntries: rpcExecutionMaxEntries,
|
||||
authMaxEntries: rpcExecutionAuthMaxEntries,
|
||||
sessionMaxEntries: rpcExecutionSessionMaxEntries,
|
||||
subscriberMaxGlobal: global,
|
||||
subscriberMaxAuth: auth,
|
||||
subscriberMaxSession: session,
|
||||
subscriberMaxPerFlight: perFlight,
|
||||
replayStore: newRPCReplayStoreForTest(),
|
||||
})
|
||||
}
|
||||
|
||||
func TestRPCResultFlightSubscriberPairCapacityFailureIsAtomic(t *testing.T) {
|
||||
cache := newRPCResultSubscriberTestCache(8, 8, 8, 1)
|
||||
cache := newRPCExecutionSubscriberTestLedger(8, 8, 8, 1)
|
||||
authKeyID := rpcFlightTestAuthID(70)
|
||||
claim, err := cache.Acquire(authKeyID, 70, 700)
|
||||
if err != nil || claim.owner == nil {
|
||||
|
|
@ -64,7 +62,7 @@ func TestRPCResultFlightSubscriberPairCapacityFailureIsAtomic(t *testing.T) {
|
|||
if !errors.Is(err, ErrRPCResultSubscriberCapacity) {
|
||||
t.Fatalf("pair subscription err=%v, want %v", err, ErrRPCResultSubscriberCapacity)
|
||||
}
|
||||
s := cache.shard(rpcResultCacheKey{authKeyID: authKeyID, sessionID: 70, reqMsgID: 700})
|
||||
s := cache.shard(rpcExecutionKey{authKeyID: authKeyID, sessionID: 70, reqMsgID: 700})
|
||||
s.mu.Lock()
|
||||
if got := claim.owner.flight.subscriberSlots; got != 0 {
|
||||
t.Fatalf("failed pair retained %d subscriber slots", got)
|
||||
|
|
@ -74,7 +72,7 @@ func TestRPCResultFlightSubscriberPairCapacityFailureIsAtomic(t *testing.T) {
|
|||
}
|
||||
s.mu.Unlock()
|
||||
claim.owner.CompleteExecution(true)
|
||||
cache.Put(authKeyID, 70, 700, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 700})
|
||||
cache.completeReplayableForTest(authKeyID, 70, 700, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 700})
|
||||
if resultCalls != 0 || executionCalls != 0 {
|
||||
t.Fatalf("failed pair callbacks ran: result=%d execution=%d", resultCalls, executionCalls)
|
||||
}
|
||||
|
|
@ -84,7 +82,7 @@ func TestRPCResultFlightSubscriberPairCapacityFailureIsAtomic(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestRPCResultFlightSubscriberBudgetsIsolateSessionAndAuth(t *testing.T) {
|
||||
cache := newRPCResultSubscriberTestCache(3, 2, 1, 4)
|
||||
cache := newRPCExecutionSubscriberTestLedger(3, 2, 1, 4)
|
||||
authA := rpcFlightTestAuthID(71)
|
||||
authB := rpcFlightTestAuthID(72)
|
||||
type ownerKey struct {
|
||||
|
|
@ -149,7 +147,7 @@ func TestRPCResultFlightSubscriberBudgetsIsolateSessionAndAuth(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestRPCResultFlightSubscriberSlotsReleasePerTerminalHalf(t *testing.T) {
|
||||
cache := newRPCResultSubscriberTestCache(4, 4, 4, 4)
|
||||
cache := newRPCExecutionSubscriberTestLedger(4, 4, 4, 4)
|
||||
authKeyID := rpcFlightTestAuthID(73)
|
||||
claim, err := cache.Acquire(authKeyID, 73, 730)
|
||||
if err != nil || claim.owner == nil {
|
||||
|
|
@ -175,7 +173,7 @@ func TestRPCResultFlightSubscriberSlotsReleasePerTerminalHalf(t *testing.T) {
|
|||
if got := cache.subscriberBudget.sessionSnapshot(authKeyID, 73); got != 1 {
|
||||
t.Fatalf("post-execution subscriber usage=%d, want 1", got)
|
||||
}
|
||||
cache.Put(authKeyID, 73, 730, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 730})
|
||||
cache.completeReplayableForTest(authKeyID, 73, 730, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 730})
|
||||
if ok := <-result; !ok {
|
||||
t.Fatal("result callback reported failure")
|
||||
}
|
||||
|
|
@ -185,7 +183,7 @@ func TestRPCResultFlightSubscriberSlotsReleasePerTerminalHalf(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestRPCResultFlightRepeatedReplayJoinsStayBoundedAndPutCleansExecution(t *testing.T) {
|
||||
cache := newRPCResultSubscriberTestCache(2, 2, 2, 2)
|
||||
cache := newRPCExecutionSubscriberTestLedger(2, 2, 2, 2)
|
||||
authKeyID := rpcFlightTestAuthID(74)
|
||||
claim, err := cache.Acquire(authKeyID, 74, 740)
|
||||
if err != nil || claim.owner == nil {
|
||||
|
|
@ -212,7 +210,7 @@ func TestRPCResultFlightRepeatedReplayJoinsStayBoundedAndPutCleansExecution(t *t
|
|||
t.Fatalf("join %d err=%v, want capacity", i, err)
|
||||
}
|
||||
}
|
||||
cache.Put(authKeyID, 74, 740, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 740})
|
||||
cache.completeReplayableForTest(authKeyID, 74, 740, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 740})
|
||||
if resultCalls != 1 || executionCalls != 1 {
|
||||
t.Fatalf("terminal callback counts result=%d execution=%d", resultCalls, executionCalls)
|
||||
}
|
||||
|
|
@ -222,7 +220,7 @@ func TestRPCResultFlightRepeatedReplayJoinsStayBoundedAndPutCleansExecution(t *t
|
|||
}
|
||||
|
||||
func TestRPCResultFlightExactIdentityGuardsPendingAndCompletedReuse(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 2)
|
||||
authKeyID := rpcFlightTestAuthID(90)
|
||||
firstIdentity := rpcFlightExactIdentity(t, tlprofile.Profile225, &tg.HelpGetConfigRequest{})
|
||||
otherIdentity := rpcFlightExactIdentity(t, tlprofile.Profile225, &tg.HelpGetNearestDCRequest{})
|
||||
|
|
@ -240,7 +238,7 @@ func TestRPCResultFlightExactIdentityGuardsPendingAndCompletedReuse(t *testing.T
|
|||
}
|
||||
|
||||
want := &encodedOutboundMessage{body: []byte{1, 2, 3, 4}, reqMsgID: 900}
|
||||
cache.Put(authKeyID, 90, 900, want)
|
||||
cache.completeReplayableForTest(authKeyID, 90, 900, want)
|
||||
if _, err := cache.AcquireIdentified(authKeyID, 90, 900, otherIdentity); !errors.Is(err, ErrRPCResultIdentityMismatch) {
|
||||
t.Fatalf("completed mismatched Acquire err = %v, want %v", err, ErrRPCResultIdentityMismatch)
|
||||
}
|
||||
|
|
@ -254,7 +252,7 @@ func TestRPCResultFlightExactIdentityGuardsPendingAndCompletedReuse(t *testing.T
|
|||
}
|
||||
|
||||
func TestRPCResultFlightAdmissionSequenceAllocatedOnceAndReplayed(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 4)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 4)
|
||||
authKeyID := rpcFlightTestAuthID(89)
|
||||
identity := rpcFlightExactIdentity(t, tlprofile.Profile225, &tg.HelpGetConfigRequest{})
|
||||
owner, err := cache.AcquireLayerIdentified(authKeyID, 89, 890, tlprofile.Profile225, identity)
|
||||
|
|
@ -267,7 +265,7 @@ func TestRPCResultFlightAdmissionSequenceAllocatedOnceAndReplayed(t *testing.T)
|
|||
}
|
||||
owner.owner.CompleteExecution(true)
|
||||
encoded := &encodedOutboundMessage{body: []byte{1}, reqMsgID: 890}
|
||||
cache.Put(authKeyID, 89, 890, encoded)
|
||||
cache.completeReplayableForTest(authKeyID, 89, 890, encoded)
|
||||
completed, err := cache.AcquireLayerIdentified(authKeyID, 89, 890, tlprofile.Profile225, identity)
|
||||
if err != nil || completed.state != rpcResultAcquireCompleted || completed.admissionSeq != owner.admissionSeq {
|
||||
t.Fatalf("completed = state:%d seq:%d err:%v, want seq:%d", completed.state, completed.admissionSeq, err, owner.admissionSeq)
|
||||
|
|
@ -285,7 +283,7 @@ func TestRPCResultFlightAdmissionSequenceAllocatedOnceAndReplayed(t *testing.T)
|
|||
}
|
||||
|
||||
func TestRPCAdmissionSafeFloorTracksOwnersUntilPutOrAbort(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 4)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 4)
|
||||
authKeyID := rpcFlightTestAuthID(86)
|
||||
identity := rpcFlightExactIdentity(t, tlprofile.Profile225, &tg.HelpGetConfigRequest{})
|
||||
first, err := cache.AcquireLayerIdentified(authKeyID, 86, 860, tlprofile.Profile225, identity)
|
||||
|
|
@ -306,14 +304,14 @@ func TestRPCAdmissionSafeFloorTracksOwnersUntilPutOrAbort(t *testing.T) {
|
|||
t.Fatalf("post-abort safe floor=%d, want %d", floor, second.admissionSeq)
|
||||
}
|
||||
second.owner.CompleteExecution(true)
|
||||
cache.Put(authKeyID, 86, 864, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 864})
|
||||
cache.completeReplayableForTest(authKeyID, 86, 864, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 864})
|
||||
if floor := cache.stableAdmissionSafeFloor(); floor != second.admissionSeq+1 {
|
||||
t.Fatalf("terminal safe floor=%d, want %d", floor, second.admissionSeq+1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCAdmissionSequenceExhaustionCannotWrap(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 2)
|
||||
cache.nextAdmissionSeq.Store(^uint64(0) - 1)
|
||||
authKeyID := rpcFlightTestAuthID(85)
|
||||
identity := rpcFlightExactIdentity(t, tlprofile.Profile225, &tg.HelpGetConfigRequest{})
|
||||
|
|
@ -331,7 +329,7 @@ func TestRPCAdmissionSequenceExhaustionCannotWrap(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestRPCIdentityMismatchCarriesWinnerProfileAcrossAbort(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 2)
|
||||
authKeyID := rpcFlightTestAuthID(88)
|
||||
request := &tg.MessagesGetHistoryRequest{Peer: &tg.InputPeerSelf{}, Limit: 1}
|
||||
winnerIdentity := rpcFlightExactIdentity(t, tlprofile.Profile225, request)
|
||||
|
|
@ -357,7 +355,7 @@ func TestRPCIdentityMismatchCarriesWinnerProfileAcrossAbort(t *testing.T) {
|
|||
|
||||
func TestRPCAdmissionProfileHintSurvivesCompletedEvictionWindow(t *testing.T) {
|
||||
now := time.Unix(1_900_000_000, 0)
|
||||
cache := newRPCResultCacheWithFlightLimit(func() time.Time { return now }, 2)
|
||||
cache := newRPCExecutionLedgerForTest(func() time.Time { return now }, 2)
|
||||
authKeyID := rpcFlightTestAuthID(87)
|
||||
identity := rpcFlightExactIdentity(t, tlprofile.Profile225, &tg.MessagesGetHistoryRequest{
|
||||
Peer: &tg.InputPeerSelf{}, Limit: 1,
|
||||
|
|
@ -367,7 +365,7 @@ func TestRPCAdmissionProfileHintSurvivesCompletedEvictionWindow(t *testing.T) {
|
|||
t.Fatalf("owner err=%v", err)
|
||||
}
|
||||
claim.owner.CompleteExecution(true)
|
||||
cache.Put(authKeyID, 87, 870, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 870})
|
||||
cache.completeReplayableForTest(authKeyID, 87, 870, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 870})
|
||||
profile, ok := cache.ExactAdmissionProfile(authKeyID, 87, 870)
|
||||
if !ok || profile != tlprofile.Profile225 {
|
||||
t.Fatalf("profile hint = (%d,%v)", profile, ok)
|
||||
|
|
@ -375,7 +373,7 @@ func TestRPCAdmissionProfileHintSurvivesCompletedEvictionWindow(t *testing.T) {
|
|||
// Admission already copied the hint into its local decoder cursor. Expiry
|
||||
// between that probe and the atomic claim must not make it fall back to the
|
||||
// connection's newer default; it simply becomes a fresh owner under 225.
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
now = now.Add(rpcExecutionReceiptTTL + time.Second)
|
||||
replacement, err := cache.AcquireLayerIdentified(authKeyID, 87, 870, profile, identity)
|
||||
if err != nil || replacement.state != rpcResultAcquireOwner || replacement.owner == nil {
|
||||
t.Fatalf("post-eviction owner = state:%d err:%v", replacement.state, err)
|
||||
|
|
@ -384,7 +382,7 @@ func TestRPCAdmissionProfileHintSurvivesCompletedEvictionWindow(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestRPCInvariantIdentityDoesNotExposeCanonicalProfileHint(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 2)
|
||||
authKeyID := rpcFlightTestAuthID(84)
|
||||
identity := rpcFlightExactIdentity(t, tlprofile.Profile227, &tg.AuthBindTempAuthKeyRequest{
|
||||
PermAuthKeyID: 1, Nonce: 2, ExpiresAt: 3, EncryptedMessage: []byte("bind"),
|
||||
|
|
@ -397,14 +395,14 @@ func TestRPCInvariantIdentityDoesNotExposeCanonicalProfileHint(t *testing.T) {
|
|||
t.Fatalf("pending invariant profile hint=(%d,%v), want absent", profile, ok)
|
||||
}
|
||||
claim.owner.CompleteExecution(true)
|
||||
cache.Put(authKeyID, 84, 840, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 840})
|
||||
cache.completeReplayableForTest(authKeyID, 84, 840, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 840})
|
||||
if profile, ok := cache.ExactAdmissionProfile(authKeyID, 84, 840); ok || profile != 0 {
|
||||
t.Fatalf("completed invariant profile hint=(%d,%v), want absent", profile, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultExecutionCompletionIsExactlyOnceAndDurable(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 2)
|
||||
authKeyID := rpcFlightTestAuthID(91)
|
||||
claim, err := cache.Acquire(authKeyID, 91, 910)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner || claim.owner == nil {
|
||||
|
|
@ -436,7 +434,7 @@ func TestRPCResultExecutionCompletionIsExactlyOnceAndDurable(t *testing.T) {
|
|||
}
|
||||
|
||||
want := &encodedOutboundMessage{body: []byte{9, 1, 0, 0}, reqMsgID: 910}
|
||||
cache.Put(authKeyID, 91, 910, want)
|
||||
cache.completeReplayableForTest(authKeyID, 91, 910, want)
|
||||
dependency, ok := cache.ObserveDependency(authKeyID, 91, 910)
|
||||
if !ok || !dependency.completed || !dependency.success || dependency.waiter != nil {
|
||||
t.Fatalf("completed dependency = %#v ok:%v", dependency, ok)
|
||||
|
|
@ -451,7 +449,7 @@ func TestRPCResultExecutionCompletionIsExactlyOnceAndDurable(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestRPCResultExecutionAbortPublishesFailure(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 1)
|
||||
authKeyID := rpcFlightTestAuthID(92)
|
||||
claim, err := cache.Acquire(authKeyID, 92, 920)
|
||||
if err != nil || claim.owner == nil {
|
||||
|
|
@ -474,7 +472,7 @@ func TestRPCResultExecutionAbortPublishesFailure(t *testing.T) {
|
|||
|
||||
func TestRPCResultFlightConcurrentAcquireHasUniqueOwner(t *testing.T) {
|
||||
const callers = 64
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, callers)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, callers)
|
||||
authKeyID := rpcFlightTestAuthID(1)
|
||||
start := make(chan struct{})
|
||||
results := make(chan rpcResultAcquire, callers)
|
||||
|
|
@ -535,7 +533,7 @@ func TestRPCResultFlightConcurrentAcquireHasUniqueOwner(t *testing.T) {
|
|||
|
||||
func TestRPCResultFlightPutPublishesAndWakesAllWaiters(t *testing.T) {
|
||||
const waiters = 24
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 32)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 32)
|
||||
authKeyID := rpcFlightTestAuthID(10)
|
||||
owner, err := cache.Acquire(authKeyID, 20, 200)
|
||||
if err != nil || owner.state != rpcResultAcquireOwner || owner.owner == nil {
|
||||
|
|
@ -563,13 +561,13 @@ func TestRPCResultFlightPutPublishesAndWakesAllWaiters(t *testing.T) {
|
|||
for _, waiter := range waiterClaims {
|
||||
go func(w *rpcResultWaiter) {
|
||||
encoded, ok, waitErr := w.Wait(ctx)
|
||||
cached, _ := cache.Get(authKeyID, 20, 200)
|
||||
cached, _ := cache.Replay(authKeyID, 20, 200)
|
||||
results <- waiterResult{encoded: encoded, cached: cached, ok: ok, err: waitErr}
|
||||
}(waiter)
|
||||
}
|
||||
|
||||
want := &encodedOutboundMessage{body: []byte{1, 2, 3, 4}, typeID: 42, reqMsgID: 200}
|
||||
cache.Put(authKeyID, 20, 200, want)
|
||||
cache.completeReplayableForTest(authKeyID, 20, 200, want)
|
||||
for i := 0; i < waiters; i++ {
|
||||
got := <-results
|
||||
if got.err != nil || !got.ok {
|
||||
|
|
@ -592,7 +590,7 @@ func TestRPCResultFlightPutPublishesAndWakesAllWaiters(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestRPCResultFlightAbortWakesAndAllowsReclaim(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 2)
|
||||
authKeyID := rpcFlightTestAuthID(20)
|
||||
first, err := cache.Acquire(authKeyID, 30, 300)
|
||||
if err != nil || first.state != rpcResultAcquireOwner {
|
||||
|
|
@ -627,22 +625,17 @@ func TestRPCResultFlightAbortWakesAndAllowsReclaim(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestRPCResultFlightCompletedCachePressureDoesNotEvictPending(t *testing.T) {
|
||||
func TestRPCResultFlightReceiptLifecycleDoesNotEvictPending(t *testing.T) {
|
||||
now := time.Unix(1_000, 0)
|
||||
cache := newRPCResultCacheWithFlightLimit(func() time.Time { return now }, 4)
|
||||
cache := newRPCExecutionLedgerForTest(func() time.Time { return now }, 4)
|
||||
authKeyID := rpcFlightTestAuthID(30)
|
||||
pending, err := cache.Acquire(authKeyID, 40, 400)
|
||||
if err != nil || pending.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("pending Acquire = state:%d err:%v", pending.state, err)
|
||||
}
|
||||
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: 40, reqMsgID: 400}
|
||||
shard := cache.shard(key)
|
||||
shard.mu.Lock()
|
||||
shard.maxEntries = 2
|
||||
shard.mu.Unlock()
|
||||
for i := int64(0); i < 16; i++ {
|
||||
cache.Put(authKeyID, 40, 500+i, &encodedOutboundMessage{body: []byte{byte(i)}})
|
||||
cache.completeReplayableForTest(authKeyID, 40, 500+i, &encodedOutboundMessage{body: []byte{byte(i)}})
|
||||
}
|
||||
if got := cache.flightLimit.snapshot(); got != 1 {
|
||||
t.Fatalf("completed trim changed pending count to %d", got)
|
||||
|
|
@ -652,9 +645,9 @@ func TestRPCResultFlightCompletedCachePressureDoesNotEvictPending(t *testing.T)
|
|||
t.Fatalf("Acquire after completed trim = state:%d err:%v", joined.state, err)
|
||||
}
|
||||
|
||||
// Expire the independent completed cache and prove the pending owner remains.
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
_, _ = cache.Get(authKeyID, 40, 515)
|
||||
// Expire independent completed receipts and prove the pending owner remains.
|
||||
now = now.Add(rpcExecutionReceiptTTL + time.Second)
|
||||
_, _ = cache.Replay(authKeyID, 40, 515)
|
||||
joinedAfterTTL, err := cache.Acquire(authKeyID, 40, 400)
|
||||
if err != nil || joinedAfterTTL.state != rpcResultAcquirePending {
|
||||
t.Fatalf("Acquire after completed TTL = state:%d err:%v", joinedAfterTTL.state, err)
|
||||
|
|
@ -665,7 +658,7 @@ func TestRPCResultFlightCompletedCachePressureDoesNotEvictPending(t *testing.T)
|
|||
}
|
||||
|
||||
func TestRPCResultFlightCapacityAndCountReturn(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 2)
|
||||
authKeyID := rpcFlightTestAuthID(40)
|
||||
first, err := cache.Acquire(authKeyID, 50, 501)
|
||||
if err != nil || first.state != rpcResultAcquireOwner {
|
||||
|
|
@ -687,7 +680,7 @@ func TestRPCResultFlightCapacityAndCountReturn(t *testing.T) {
|
|||
}
|
||||
|
||||
want := &encodedOutboundMessage{body: []byte{9}, reqMsgID: 501}
|
||||
cache.Put(authKeyID, 50, 501, want)
|
||||
cache.completeReplayableForTest(authKeyID, 50, 501, want)
|
||||
if got := cache.flightLimit.snapshot(); got != 1 {
|
||||
t.Fatalf("pending count after Put = %d, want 1", got)
|
||||
}
|
||||
|
|
@ -710,8 +703,8 @@ func TestRPCResultFlightCapacityAndCountReturn(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestRPCResultFlightLargePutPublishesCompletedBeforeResolvingWaiters(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
|
||||
func TestRPCResultFlightLargeCompletionDoesNotChargeLedgerBodyBytes(t *testing.T) {
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 1)
|
||||
authKeyID := rpcFlightTestAuthID(50)
|
||||
owner, err := cache.Acquire(authKeyID, 60, 600)
|
||||
if err != nil || owner.state != rpcResultAcquireOwner {
|
||||
|
|
@ -722,22 +715,22 @@ func TestRPCResultFlightLargePutPublishesCompletedBeforeResolvingWaiters(t *test
|
|||
t.Fatalf("joined Acquire = state:%d err:%v", joined.state, err)
|
||||
}
|
||||
|
||||
// This is larger than the removed 4 MiB per-shard partition but remains a
|
||||
// legal outbound result and fits the global/auth/session fair byte budgets.
|
||||
largeSize := rpcResultCacheMaxBytes/rpcResultCacheShards + 1
|
||||
// Large file results remain owned by the replay store. The ledger retains
|
||||
// only one fixed-shape receipt regardless of payload size.
|
||||
largeSize := 8 << 20
|
||||
want := &encodedOutboundMessage{body: make([]byte, largeSize), reqMsgID: 600}
|
||||
cache.Put(authKeyID, 60, 600, want)
|
||||
cache.completeReplayableForTest(authKeyID, 60, 600, want)
|
||||
if encoded, ok, waitErr := joined.waiter.Wait(context.Background()); waitErr != nil || !ok || encoded != want {
|
||||
t.Fatalf("large Wait = encoded:%p ok:%v err:%v", encoded, ok, waitErr)
|
||||
}
|
||||
if got, ok := cache.Get(authKeyID, 60, 600); !ok || got != want {
|
||||
t.Fatalf("large completed Get = encoded:%p ok:%v", got, ok)
|
||||
if got, ok := cache.Replay(authKeyID, 60, 600); !ok || got != want {
|
||||
t.Fatalf("large completed Replay = encoded:%p ok:%v", got, ok)
|
||||
}
|
||||
if got := cache.flightLimit.snapshot(); got != 0 {
|
||||
t.Fatalf("large Put leaked pending count %d", got)
|
||||
}
|
||||
if owner.owner.Abort() {
|
||||
t.Fatal("large Put left its old owner abortable")
|
||||
t.Fatal("large completion left its old owner abortable")
|
||||
}
|
||||
completed, err := cache.Acquire(authKeyID, 60, 600)
|
||||
if err != nil || completed.state != rpcResultAcquireCompleted || completed.encoded != want {
|
||||
|
|
@ -746,13 +739,13 @@ func TestRPCResultFlightLargePutPublishesCompletedBeforeResolvingWaiters(t *test
|
|||
if completed.owner != nil {
|
||||
t.Fatal("large completed result incorrectly returned a new owner")
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != int64(largeSize) {
|
||||
t.Fatalf("completed byte budget = %d, want %d", got, largeSize)
|
||||
if got := cache.receiptBudgetBytes(); got != rpcExecutionReceiptBudgetBytes {
|
||||
t.Fatalf("receipt budget bytes = %d, want %d", got, rpcExecutionReceiptBudgetBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultFlightWaitContextDoesNotReleaseOwner(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 1)
|
||||
authKeyID := rpcFlightTestAuthID(60)
|
||||
owner, err := cache.Acquire(authKeyID, 70, 700)
|
||||
if err != nil {
|
||||
|
|
@ -780,7 +773,7 @@ func TestRPCResultFlightConcurrentCapacityReturnsAllSlots(t *testing.T) {
|
|||
limit = 32
|
||||
callers = 512
|
||||
)
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, limit)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, limit)
|
||||
authKeyID := rpcFlightTestAuthID(70)
|
||||
start := make(chan struct{})
|
||||
errs := make(chan error, callers)
|
||||
|
|
@ -804,7 +797,7 @@ func TestRPCResultFlightConcurrentCapacityReturnsAllSlots(t *testing.T) {
|
|||
return
|
||||
}
|
||||
if i%2 == 0 {
|
||||
cache.Put(authKeyID, 80+int64(i%4), 1_000+int64(i), &encodedOutboundMessage{body: []byte{1}})
|
||||
cache.completeReplayableForTest(authKeyID, 80+int64(i%4), 1_000+int64(i), &encodedOutboundMessage{body: []byte{1}})
|
||||
} else if !claim.owner.Abort() {
|
||||
errs <- errors.New("owner Abort lost")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,292 +0,0 @@
|
|||
package mtprotoedge
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash/maphash"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const rpcResultBudgetShards = 64
|
||||
|
||||
type rpcResultBudgetLimit struct {
|
||||
entries int64
|
||||
bytes int64
|
||||
}
|
||||
|
||||
type rpcResultBudgetUsage struct {
|
||||
entries int64
|
||||
bytes int64
|
||||
pending int64
|
||||
}
|
||||
|
||||
type rpcResultSessionBudgetKey struct {
|
||||
authKeyID [8]byte
|
||||
sessionID int64
|
||||
}
|
||||
|
||||
type rpcResultAuthBudgetShard struct {
|
||||
mu sync.Mutex
|
||||
usage map[[8]byte]rpcResultBudgetUsage
|
||||
}
|
||||
|
||||
type rpcResultSessionBudgetShard struct {
|
||||
mu sync.Mutex
|
||||
usage map[rpcResultSessionBudgetKey]rpcResultBudgetUsage
|
||||
}
|
||||
|
||||
// rpcResultFairBudget accounts one ownership reservation at all three scopes.
|
||||
// A pending owner and its completed result are the same ownership: admission
|
||||
// reserves one entry plus one byte, Put resizes that byte reservation and moves
|
||||
// it to the completed row, while Abort/TTL return the whole reservation.
|
||||
//
|
||||
// Auth and session maps are striped independently. Every operation takes the
|
||||
// auth stripe before the session stripe; global counters remain atomic. This
|
||||
// keeps unrelated auth keys off one process-wide mutex while preserving hard
|
||||
// limits at every hierarchy level.
|
||||
type rpcResultFairBudget struct {
|
||||
seed maphash.Seed
|
||||
globalEntries *rpcResultFlightLimit
|
||||
globalBytes *rpcResultCacheByteBudget
|
||||
authLimit rpcResultBudgetLimit
|
||||
sessionLimit rpcResultBudgetLimit
|
||||
pendingPerAuth int64
|
||||
authShards [rpcResultBudgetShards]rpcResultAuthBudgetShard
|
||||
sessionShards [rpcResultBudgetShards]rpcResultSessionBudgetShard
|
||||
}
|
||||
|
||||
type rpcResultBudgetReservation struct {
|
||||
budget *rpcResultFairBudget
|
||||
key rpcResultCacheKey
|
||||
bytes int
|
||||
pending bool
|
||||
released bool
|
||||
}
|
||||
|
||||
func newRPCResultFairBudget(
|
||||
seed maphash.Seed,
|
||||
globalEntries *rpcResultFlightLimit,
|
||||
globalBytes *rpcResultCacheByteBudget,
|
||||
authLimit rpcResultBudgetLimit,
|
||||
sessionLimit rpcResultBudgetLimit,
|
||||
pendingPerAuth int,
|
||||
) *rpcResultFairBudget {
|
||||
b := &rpcResultFairBudget{
|
||||
seed: seed,
|
||||
globalEntries: globalEntries,
|
||||
globalBytes: globalBytes,
|
||||
authLimit: authLimit,
|
||||
sessionLimit: sessionLimit,
|
||||
pendingPerAuth: int64(pendingPerAuth),
|
||||
}
|
||||
for i := range b.authShards {
|
||||
b.authShards[i].usage = make(map[[8]byte]rpcResultBudgetUsage)
|
||||
b.sessionShards[i].usage = make(map[rpcResultSessionBudgetKey]rpcResultBudgetUsage)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *rpcResultFairBudget) reserveOwner(key rpcResultCacheKey) *rpcResultBudgetReservation {
|
||||
return b.reserve(key, 1, true)
|
||||
}
|
||||
|
||||
func (b *rpcResultFairBudget) reserveCompleted(key rpcResultCacheKey, bytes int) *rpcResultBudgetReservation {
|
||||
return b.reserve(key, bytes, false)
|
||||
}
|
||||
|
||||
func (b *rpcResultFairBudget) reserve(key rpcResultCacheKey, bytes int, pending bool) *rpcResultBudgetReservation {
|
||||
if b == nil || b.globalEntries == nil || b.globalBytes == nil || bytes < 1 {
|
||||
return nil
|
||||
}
|
||||
authShard := b.authShard(key.authKeyID)
|
||||
sessionKey := rpcResultSessionBudgetKey{authKeyID: key.authKeyID, sessionID: key.sessionID}
|
||||
sessionShard := b.sessionShard(sessionKey)
|
||||
authShard.mu.Lock()
|
||||
sessionShard.mu.Lock()
|
||||
|
||||
authUsage := authShard.usage[key.authKeyID]
|
||||
sessionUsage := sessionShard.usage[sessionKey]
|
||||
bytes64 := int64(bytes)
|
||||
canReserve := withinRPCResultBudget(authUsage.entries, 1, b.authLimit.entries) &&
|
||||
withinRPCResultBudget(authUsage.bytes, bytes64, b.authLimit.bytes) &&
|
||||
withinRPCResultBudget(sessionUsage.entries, 1, b.sessionLimit.entries) &&
|
||||
withinRPCResultBudget(sessionUsage.bytes, bytes64, b.sessionLimit.bytes)
|
||||
if pending {
|
||||
canReserve = canReserve && withinRPCResultBudget(authUsage.pending, 1, b.pendingPerAuth)
|
||||
}
|
||||
if !canReserve || !b.globalEntries.reserve() {
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
if !b.globalBytes.reserve(bytes) {
|
||||
b.globalEntries.release()
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
authUsage.entries++
|
||||
authUsage.bytes += bytes64
|
||||
sessionUsage.entries++
|
||||
sessionUsage.bytes += bytes64
|
||||
if pending {
|
||||
authUsage.pending++
|
||||
}
|
||||
authShard.usage[key.authKeyID] = authUsage
|
||||
sessionShard.usage[sessionKey] = sessionUsage
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
return &rpcResultBudgetReservation{budget: b, key: key, bytes: bytes, pending: pending}
|
||||
}
|
||||
|
||||
func withinRPCResultBudget(used, delta, limit int64) bool {
|
||||
return delta >= 0 && limit > 0 && used >= 0 && used <= limit-delta
|
||||
}
|
||||
|
||||
func (r *rpcResultBudgetReservation) resizeBytes(bytes int) bool {
|
||||
if r == nil || r.budget == nil || r.released || bytes < 1 {
|
||||
return false
|
||||
}
|
||||
if bytes == r.bytes {
|
||||
return true
|
||||
}
|
||||
b := r.budget
|
||||
authShard := b.authShard(r.key.authKeyID)
|
||||
sessionKey := rpcResultSessionBudgetKey{authKeyID: r.key.authKeyID, sessionID: r.key.sessionID}
|
||||
sessionShard := b.sessionShard(sessionKey)
|
||||
authShard.mu.Lock()
|
||||
sessionShard.mu.Lock()
|
||||
authUsage, authOK := authShard.usage[r.key.authKeyID]
|
||||
sessionUsage, sessionOK := sessionShard.usage[sessionKey]
|
||||
if !authOK || !sessionOK || authUsage.entries < 1 || sessionUsage.entries < 1 {
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
panic("mtprotoedge: rpc result budget reservation disappeared during resize")
|
||||
}
|
||||
delta := int64(bytes) - int64(r.bytes)
|
||||
if delta > 0 {
|
||||
if !withinRPCResultBudget(authUsage.bytes, delta, b.authLimit.bytes) ||
|
||||
!withinRPCResultBudget(sessionUsage.bytes, delta, b.sessionLimit.bytes) ||
|
||||
!b.globalBytes.reserve(int(delta)) {
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
if authUsage.bytes < -delta || sessionUsage.bytes < -delta {
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
panic("mtprotoedge: rpc result byte reservation underflow during resize")
|
||||
}
|
||||
}
|
||||
authUsage.bytes += delta
|
||||
sessionUsage.bytes += delta
|
||||
authShard.usage[r.key.authKeyID] = authUsage
|
||||
sessionShard.usage[sessionKey] = sessionUsage
|
||||
r.bytes = bytes
|
||||
if delta < 0 {
|
||||
b.globalBytes.release(int(-delta))
|
||||
}
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *rpcResultBudgetReservation) releasePending() {
|
||||
if r == nil || r.budget == nil || r.released || !r.pending {
|
||||
return
|
||||
}
|
||||
b := r.budget
|
||||
authShard := b.authShard(r.key.authKeyID)
|
||||
authShard.mu.Lock()
|
||||
authUsage, ok := authShard.usage[r.key.authKeyID]
|
||||
if !ok || authUsage.pending < 1 {
|
||||
authShard.mu.Unlock()
|
||||
panic("mtprotoedge: rpc result per-auth pending budget underflow")
|
||||
}
|
||||
authUsage.pending--
|
||||
authShard.usage[r.key.authKeyID] = authUsage
|
||||
r.pending = false
|
||||
authShard.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *rpcResultBudgetReservation) release() {
|
||||
if r == nil || r.budget == nil || r.released {
|
||||
return
|
||||
}
|
||||
b := r.budget
|
||||
authShard := b.authShard(r.key.authKeyID)
|
||||
sessionKey := rpcResultSessionBudgetKey{authKeyID: r.key.authKeyID, sessionID: r.key.sessionID}
|
||||
sessionShard := b.sessionShard(sessionKey)
|
||||
authShard.mu.Lock()
|
||||
sessionShard.mu.Lock()
|
||||
authUsage, authOK := authShard.usage[r.key.authKeyID]
|
||||
sessionUsage, sessionOK := sessionShard.usage[sessionKey]
|
||||
bytes64 := int64(r.bytes)
|
||||
if !authOK || !sessionOK || authUsage.entries < 1 || sessionUsage.entries < 1 ||
|
||||
authUsage.bytes < bytes64 || sessionUsage.bytes < bytes64 ||
|
||||
(r.pending && authUsage.pending < 1) {
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
panic("mtprotoedge: rpc result fair budget underflow")
|
||||
}
|
||||
authUsage.entries--
|
||||
authUsage.bytes -= bytes64
|
||||
sessionUsage.entries--
|
||||
sessionUsage.bytes -= bytes64
|
||||
if r.pending {
|
||||
authUsage.pending--
|
||||
}
|
||||
if authUsage == (rpcResultBudgetUsage{}) {
|
||||
delete(authShard.usage, r.key.authKeyID)
|
||||
} else {
|
||||
authShard.usage[r.key.authKeyID] = authUsage
|
||||
}
|
||||
if sessionUsage == (rpcResultBudgetUsage{}) {
|
||||
delete(sessionShard.usage, sessionKey)
|
||||
} else {
|
||||
sessionShard.usage[sessionKey] = sessionUsage
|
||||
}
|
||||
r.released = true
|
||||
r.pending = false
|
||||
r.bytes = 0
|
||||
b.globalBytes.release(int(bytes64))
|
||||
b.globalEntries.release()
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
}
|
||||
|
||||
func (b *rpcResultFairBudget) authSnapshot(authKeyID [8]byte) rpcResultBudgetUsage {
|
||||
if b == nil {
|
||||
return rpcResultBudgetUsage{}
|
||||
}
|
||||
shard := b.authShard(authKeyID)
|
||||
shard.mu.Lock()
|
||||
usage := shard.usage[authKeyID]
|
||||
shard.mu.Unlock()
|
||||
return usage
|
||||
}
|
||||
|
||||
func (b *rpcResultFairBudget) sessionSnapshot(authKeyID [8]byte, sessionID int64) rpcResultBudgetUsage {
|
||||
if b == nil {
|
||||
return rpcResultBudgetUsage{}
|
||||
}
|
||||
key := rpcResultSessionBudgetKey{authKeyID: authKeyID, sessionID: sessionID}
|
||||
shard := b.sessionShard(key)
|
||||
shard.mu.Lock()
|
||||
usage := shard.usage[key]
|
||||
shard.mu.Unlock()
|
||||
return usage
|
||||
}
|
||||
|
||||
func (b *rpcResultFairBudget) authShard(authKeyID [8]byte) *rpcResultAuthBudgetShard {
|
||||
index := maphash.Bytes(b.seed, authKeyID[:]) & (rpcResultBudgetShards - 1)
|
||||
return &b.authShards[index]
|
||||
}
|
||||
|
||||
func (b *rpcResultFairBudget) sessionShard(key rpcResultSessionBudgetKey) *rpcResultSessionBudgetShard {
|
||||
var raw [16]byte
|
||||
copy(raw[:8], key.authKeyID[:])
|
||||
binary.LittleEndian.PutUint64(raw[8:], uint64(key.sessionID))
|
||||
index := maphash.Bytes(b.seed, raw[:]) & (rpcResultBudgetShards - 1)
|
||||
return &b.sessionShards[index]
|
||||
}
|
||||
|
|
@ -1,670 +0,0 @@
|
|||
package mtprotoedge
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"hash/maphash"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// Telegram accepts client msg_id values up to five minutes old and up to
|
||||
// thirty seconds in the future. Retain the result across that complete
|
||||
// replay horizon, plus one second for boundary/scheduler jitter, so a valid
|
||||
// duplicate cannot rerun its handler merely because our cache expired first.
|
||||
rpcResultCacheTTL = 331 * time.Second
|
||||
// Completed results cover the complete replay horizon under explicit global,
|
||||
// auth and session hard ceilings. At the default 331-second TTL, the 1<<18
|
||||
// global entries permit about 792 unique RPC/s process-wide before bounded
|
||||
// backpressure; lower scopes provide noisy-neighbor isolation.
|
||||
rpcResultCacheMaxEntries = 1 << 18
|
||||
rpcResultCacheMaxBytes = 64 << 20
|
||||
rpcResultCacheAuthMaxEntries = 1 << 15
|
||||
rpcResultCacheAuthMaxBytes = 32 << 20
|
||||
rpcResultCacheSessionMaxEntries = 1 << 14
|
||||
rpcResultCacheSessionMaxBytes = 16 << 20
|
||||
rpcResultFlightMaxPendingPerAuth = 1 << 11
|
||||
// rpcResultCacheShards hashes the complete replay identity with a random
|
||||
// per-instance maphash seed. Including req_msg_id spreads one hot session's
|
||||
// independent requests instead of forcing them through one mutex. The shard
|
||||
// count is a power of two.
|
||||
rpcResultCacheShards = 16
|
||||
)
|
||||
|
||||
type rpcResultCacheKey struct {
|
||||
authKeyID [8]byte
|
||||
sessionID int64
|
||||
reqMsgID int64
|
||||
}
|
||||
|
||||
type rpcResultCacheEntry struct {
|
||||
key rpcResultCacheKey
|
||||
// encoded is used only by legacy focused constructors whose cache has no
|
||||
// SessionManager. Production rows are metadata receipts and always leave it
|
||||
// nil; exact bytes have one owner in the logical-session outbox.
|
||||
encoded *encodedOutboundMessage
|
||||
size int
|
||||
expiresAt time.Time
|
||||
identity rpcResultRequestIdentity
|
||||
admissionSeq uint64
|
||||
executionKnown bool
|
||||
executionOK bool
|
||||
// acknowledged exists only for the ACK-before-Put race. A completed row is
|
||||
// removed immediately on ACK; unlike the previous design, success receipts
|
||||
// are not retained for the rest of the 331-second horizon.
|
||||
acknowledged bool
|
||||
// capacity marks a bounded replay tombstone. The original owner and its
|
||||
// already-joined waiters received encoded, but the byte budget could not
|
||||
// retain that body. Keeping the immutable identity until TTL prevents a
|
||||
// duplicate from rerunning business; Acquire returns a capacity error.
|
||||
capacity bool
|
||||
// reservation is the same global+auth+session ownership acquired before the
|
||||
// handler ran. Put transfers it from the pending flight; TTL returns it.
|
||||
reservation *rpcResultBudgetReservation
|
||||
}
|
||||
|
||||
type rpcResultDependency struct {
|
||||
waiter *rpcResultWaiter
|
||||
completed bool
|
||||
success bool
|
||||
}
|
||||
|
||||
// rpcResultCache 缓存已有交付证明的 rpc_result(按 auth_key+session+req_msg_id),
|
||||
// 用于跨连接重放重复请求。Put 的调用方必须先证明结果已物理写出,或原 logical Conn
|
||||
// 已不可逆 fenced;绝不能发布“Conn 仍 current/open 但结果尚未上 wire”的完成态。
|
||||
// encodedOutboundMessage 构造后不可变(push fan-out 与 pending resend 均依赖该契约),
|
||||
// 因此 Get/Put 直接共享指针,不做防御性拷贝。
|
||||
type rpcResultCache struct {
|
||||
shards [rpcResultCacheShards]rpcResultCacheShard
|
||||
hashSeed maphash.Seed
|
||||
completedBytes rpcResultCacheByteBudget
|
||||
completedEntries rpcResultFlightLimit
|
||||
fairBudget *rpcResultFairBudget
|
||||
flightLimit rpcResultFlightLimit
|
||||
subscriberBudget *rpcResultSubscriberBudget
|
||||
subscriberPerFlight int
|
||||
// sessions is the production payload source. Completed ledger rows retain
|
||||
// only request identity/outcome/TTL metadata; exact bytes stay solely in the
|
||||
// logical-session unacked outbox. Nil is kept as a legacy unit-test seam.
|
||||
sessions *SessionManager
|
||||
// nextAdmissionSeq is the process-wide ordering authority for auth-key
|
||||
// shared Layer defaults. Exact owners allocate once; joins/replays retain the
|
||||
// owner's value from their flight/completed descriptor.
|
||||
nextAdmissionSeq atomic.Uint64
|
||||
activeAdmissions rpcAdmissionTracker
|
||||
}
|
||||
|
||||
func (c *rpcResultCache) stableAdmissionSafeFloor() uint64 {
|
||||
if c == nil {
|
||||
return 0
|
||||
}
|
||||
return c.activeAdmissions.stableSafeFloor(&c.nextAdmissionSeq)
|
||||
}
|
||||
|
||||
type rpcResultCacheShard struct {
|
||||
mu sync.Mutex
|
||||
now func() time.Time
|
||||
ttl time.Duration
|
||||
// maxEntries is a focused-test seam for one physical shard. Production leaves
|
||||
// it zero and uses the explicit global/auth/session fair-budget hierarchy.
|
||||
maxEntries int
|
||||
bytes int
|
||||
order *list.List
|
||||
byKey map[rpcResultCacheKey]*list.Element
|
||||
// pending is deliberately independent from the completed-result order/byKey
|
||||
// cache. In-flight owners and waiters must not disappear when completed
|
||||
// results expire or are trimmed under entry/byte pressure.
|
||||
pending map[rpcResultCacheKey]*rpcResultFlight
|
||||
}
|
||||
|
||||
func newRPCResultCacheWithFlightLimit(now func() time.Time, maxPending int) *rpcResultCache {
|
||||
if maxPending <= 0 {
|
||||
maxPending = rpcResultFlightDefaultMaxPending
|
||||
}
|
||||
pendingPerAuth := rpcResultFlightMaxPendingPerAuth
|
||||
if pendingPerAuth > maxPending {
|
||||
pendingPerAuth = maxPending
|
||||
}
|
||||
return newRPCResultCacheWithFairCapacity(now, rpcResultCacheCapacity{
|
||||
maxPending: maxPending,
|
||||
maxPendingPerAuth: pendingPerAuth,
|
||||
globalMaxBytes: rpcResultCacheMaxBytes,
|
||||
globalMaxEntries: rpcResultCacheMaxEntries,
|
||||
authMaxBytes: rpcResultCacheAuthMaxBytes,
|
||||
authMaxEntries: rpcResultCacheAuthMaxEntries,
|
||||
sessionMaxBytes: rpcResultCacheSessionMaxBytes,
|
||||
sessionMaxEntries: rpcResultCacheSessionMaxEntries,
|
||||
})
|
||||
}
|
||||
|
||||
func newRPCResultCacheWithLimits(now func() time.Time, maxPending, maxCompletedBytes int) *rpcResultCache {
|
||||
return newRPCResultCacheWithCapacity(now, maxPending, int64(maxCompletedBytes), rpcResultCacheMaxEntries)
|
||||
}
|
||||
|
||||
func newRPCResultCacheWithCapacity(
|
||||
now func() time.Time,
|
||||
maxPending int,
|
||||
maxCompletedBytes int64,
|
||||
maxCompletedEntries int,
|
||||
) *rpcResultCache {
|
||||
// Compatibility/test constructor: the caller supplied only global limits, so
|
||||
// keep every fairness scope equal to that global ceiling. Production always
|
||||
// calls newRPCResultCacheWithFairCapacity with explicit auth/session limits.
|
||||
return newRPCResultCacheWithFairCapacity(now, rpcResultCacheCapacity{
|
||||
maxPending: maxPending,
|
||||
maxPendingPerAuth: maxPending,
|
||||
globalMaxBytes: maxCompletedBytes,
|
||||
globalMaxEntries: maxCompletedEntries,
|
||||
authMaxBytes: maxCompletedBytes,
|
||||
authMaxEntries: maxCompletedEntries,
|
||||
sessionMaxBytes: maxCompletedBytes,
|
||||
sessionMaxEntries: maxCompletedEntries,
|
||||
})
|
||||
}
|
||||
|
||||
type rpcResultCacheCapacity struct {
|
||||
maxPending int
|
||||
maxPendingPerAuth int
|
||||
globalMaxBytes int64
|
||||
globalMaxEntries int
|
||||
authMaxBytes int64
|
||||
authMaxEntries int
|
||||
sessionMaxBytes int64
|
||||
sessionMaxEntries int
|
||||
subscriberMaxGlobal int
|
||||
subscriberMaxAuth int
|
||||
subscriberMaxSession int
|
||||
subscriberMaxPerFlight int
|
||||
sessions *SessionManager
|
||||
}
|
||||
|
||||
func newRPCResultCacheWithFairCapacity(now func() time.Time, capacity rpcResultCacheCapacity) *rpcResultCache {
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
if capacity.maxPending <= 0 {
|
||||
capacity.maxPending = rpcResultFlightDefaultMaxPending
|
||||
}
|
||||
if capacity.maxPendingPerAuth <= 0 {
|
||||
capacity.maxPendingPerAuth = capacity.maxPending
|
||||
}
|
||||
if capacity.globalMaxBytes <= 0 {
|
||||
capacity.globalMaxBytes = rpcResultCacheMaxBytes
|
||||
}
|
||||
if capacity.globalMaxEntries <= 0 {
|
||||
capacity.globalMaxEntries = rpcResultCacheMaxEntries
|
||||
}
|
||||
if capacity.authMaxBytes <= 0 {
|
||||
capacity.authMaxBytes = capacity.globalMaxBytes
|
||||
}
|
||||
if capacity.authMaxEntries <= 0 {
|
||||
capacity.authMaxEntries = capacity.globalMaxEntries
|
||||
}
|
||||
if capacity.sessionMaxBytes <= 0 {
|
||||
capacity.sessionMaxBytes = capacity.authMaxBytes
|
||||
}
|
||||
if capacity.sessionMaxEntries <= 0 {
|
||||
capacity.sessionMaxEntries = capacity.authMaxEntries
|
||||
}
|
||||
if capacity.subscriberMaxGlobal <= 0 {
|
||||
capacity.subscriberMaxGlobal = rpcResultSubscriberMaxGlobal
|
||||
}
|
||||
if capacity.subscriberMaxAuth <= 0 {
|
||||
capacity.subscriberMaxAuth = rpcResultSubscriberMaxAuth
|
||||
}
|
||||
if capacity.subscriberMaxSession <= 0 {
|
||||
capacity.subscriberMaxSession = rpcResultSubscriberMaxSession
|
||||
}
|
||||
if capacity.subscriberMaxPerFlight <= 0 {
|
||||
capacity.subscriberMaxPerFlight = rpcResultSubscriberMaxPerFlight
|
||||
}
|
||||
c := &rpcResultCache{hashSeed: maphash.MakeSeed(), sessions: capacity.sessions}
|
||||
c.completedBytes.max = capacity.globalMaxBytes
|
||||
c.completedEntries.max = int64(capacity.globalMaxEntries)
|
||||
c.flightLimit.max = int64(capacity.maxPending)
|
||||
c.fairBudget = newRPCResultFairBudget(
|
||||
c.hashSeed,
|
||||
&c.completedEntries,
|
||||
&c.completedBytes,
|
||||
rpcResultBudgetLimit{entries: int64(capacity.authMaxEntries), bytes: capacity.authMaxBytes},
|
||||
rpcResultBudgetLimit{entries: int64(capacity.sessionMaxEntries), bytes: capacity.sessionMaxBytes},
|
||||
capacity.maxPendingPerAuth,
|
||||
)
|
||||
c.subscriberBudget = newRPCResultSubscriberBudget(
|
||||
c.hashSeed,
|
||||
capacity.subscriberMaxGlobal,
|
||||
capacity.subscriberMaxAuth,
|
||||
capacity.subscriberMaxSession,
|
||||
)
|
||||
c.subscriberPerFlight = capacity.subscriberMaxPerFlight
|
||||
for i := range c.shards {
|
||||
s := &c.shards[i]
|
||||
s.now = now
|
||||
s.ttl = rpcResultCacheTTL
|
||||
s.maxEntries = 0
|
||||
s.order = list.New()
|
||||
s.byKey = make(map[rpcResultCacheKey]*list.Element)
|
||||
s.pending = make(map[rpcResultCacheKey]*rpcResultFlight)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *rpcResultCache) shard(key rpcResultCacheKey) *rpcResultCacheShard {
|
||||
return &c.shards[c.shardIndex(key)]
|
||||
|
||||
}
|
||||
|
||||
func (c *rpcResultCache) shardIndex(key rpcResultCacheKey) uint64 {
|
||||
var raw [24]byte
|
||||
copy(raw[:8], key.authKeyID[:])
|
||||
binary.LittleEndian.PutUint64(raw[8:16], uint64(key.sessionID))
|
||||
binary.LittleEndian.PutUint64(raw[16:24], uint64(key.reqMsgID))
|
||||
return maphash.Bytes(c.hashSeed, raw[:]) & (rpcResultCacheShards - 1)
|
||||
}
|
||||
|
||||
func (c *rpcResultCache) Get(authKeyID [8]byte, sessionID, reqMsgID int64) (*encodedOutboundMessage, bool) {
|
||||
if c == nil || reqMsgID == 0 {
|
||||
return nil, false
|
||||
}
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s := c.shard(key)
|
||||
now := s.now()
|
||||
|
||||
s.mu.Lock()
|
||||
|
||||
elem, ok := s.byKey[key]
|
||||
if !ok {
|
||||
s.mu.Unlock()
|
||||
return nil, false
|
||||
}
|
||||
entry := elem.Value.(*rpcResultCacheEntry)
|
||||
if !entry.expiresAt.After(now) {
|
||||
s.removeElement(elem)
|
||||
s.mu.Unlock()
|
||||
return nil, false
|
||||
}
|
||||
if entry.capacity || entry.acknowledged {
|
||||
s.mu.Unlock()
|
||||
return nil, false
|
||||
}
|
||||
if c.sessions != nil {
|
||||
s.mu.Unlock()
|
||||
return c.sessions.rpcResult(authKeyID, sessionID, reqMsgID)
|
||||
}
|
||||
encoded := entry.encoded
|
||||
s.mu.Unlock()
|
||||
return encoded, encoded != nil
|
||||
}
|
||||
|
||||
// Acknowledge removes a completed rpc_result receipt immediately.
|
||||
// The caller must have resolved reqMsgID from a client msgs_ack through the
|
||||
// sole outbound actor's tracked server-message mapping; an untrusted request
|
||||
// msg_id must never call this method directly.
|
||||
//
|
||||
// ACK can race the asynchronous terminal Put. A pending flight records the
|
||||
// event so Put resolves current waiters but does not retain a completed row.
|
||||
// This follows the logical-session model: once the client has acknowledged the
|
||||
// exact server message, neither its payload nor its duplicate receipt is useful.
|
||||
func (c *rpcResultCache) Acknowledge(authKeyID [8]byte, sessionID, reqMsgID int64) bool {
|
||||
if c == nil || reqMsgID == 0 {
|
||||
return false
|
||||
}
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s := c.shard(key)
|
||||
now := s.now()
|
||||
s.mu.Lock()
|
||||
s.expireLocked(now)
|
||||
|
||||
if elem := s.byKey[key]; elem != nil {
|
||||
s.removeElement(elem)
|
||||
s.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
if flight := s.pending[key]; flight != nil {
|
||||
flight.acknowledged = true
|
||||
s.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
|
||||
// ObserveDependency returns a waiter for an admitted in-flight dependency, a
|
||||
// nil waiter for an already completed dependency, or ok=false when the
|
||||
// referenced message never established API-RPC ownership. It never creates a
|
||||
// flight and therefore cannot turn a forged invokeAfterMsg into authority to
|
||||
// run another request.
|
||||
func (c *rpcResultCache) ObserveDependency(authKeyID [8]byte, sessionID, reqMsgID int64) (rpcResultDependency, bool) {
|
||||
if c == nil || reqMsgID == 0 {
|
||||
return rpcResultDependency{}, false
|
||||
}
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s := c.shard(key)
|
||||
now := s.now()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if elem, exists := s.byKey[key]; exists {
|
||||
entry := elem.Value.(*rpcResultCacheEntry)
|
||||
if entry.expiresAt.After(now) {
|
||||
if !entry.executionKnown {
|
||||
return rpcResultDependency{}, false
|
||||
}
|
||||
return rpcResultDependency{completed: true, success: entry.executionOK}, true
|
||||
}
|
||||
s.removeElement(elem)
|
||||
}
|
||||
if flight := s.pending[key]; flight != nil {
|
||||
if flight.executionDone {
|
||||
return rpcResultDependency{completed: true, success: flight.executionOK}, true
|
||||
}
|
||||
return rpcResultDependency{waiter: &rpcResultWaiter{cache: c, key: key, flight: flight}}, true
|
||||
}
|
||||
return rpcResultDependency{}, false
|
||||
}
|
||||
|
||||
func (c *rpcResultCache) Put(authKeyID [8]byte, sessionID, reqMsgID int64, encoded *encodedOutboundMessage) {
|
||||
if c == nil || reqMsgID == 0 || encoded == nil {
|
||||
return
|
||||
}
|
||||
if c.putOnce(authKeyID, sessionID, reqMsgID, encoded) {
|
||||
return
|
||||
}
|
||||
// A direct Put has no pre-reserved owner slot. Expired entries in another
|
||||
// shard may be its only blocker; reap once without holding a shard and retry.
|
||||
// Production owner publication already carries both reservations and never
|
||||
// needs this cold path.
|
||||
c.expireCompletedResults()
|
||||
_ = c.putOnce(authKeyID, sessionID, reqMsgID, encoded)
|
||||
}
|
||||
|
||||
// putOnce returns false only when a cross-shard expiry reap may release the
|
||||
// process-wide entry/body capacity needed by a defensive direct Put.
|
||||
func (c *rpcResultCache) putOnce(authKeyID [8]byte, sessionID, reqMsgID int64, encoded *encodedOutboundMessage) bool {
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||
s := c.shard(key)
|
||||
logicalOutbox := c.sessions != nil
|
||||
logicalReplayable := false
|
||||
if logicalOutbox {
|
||||
_, logicalReplayable = c.sessions.rpcResult(authKeyID, sessionID, reqMsgID)
|
||||
}
|
||||
accountedSize := len(encoded.body)
|
||||
if logicalOutbox {
|
||||
accountedSize = 1
|
||||
}
|
||||
if accountedSize < 1 {
|
||||
// Every owner reserves one byte at admission. Keeping zero-length results
|
||||
// at the same minimum makes entry and byte capacity linearizable.
|
||||
accountedSize = 1
|
||||
}
|
||||
|
||||
// Publication never evicts another unexpired result. A production owner has
|
||||
// already reserved its entry slot and one byte. If its actual result cannot
|
||||
// expand that reservation, publish a one-byte identity tombstone: the owner
|
||||
// and current waiters still receive the immutable result, while later
|
||||
// duplicates fail admission instead of rerunning the handler.
|
||||
s.mu.Lock()
|
||||
now := s.now()
|
||||
s.expireLocked(now)
|
||||
old := s.byKey[key]
|
||||
flight := s.pending[key]
|
||||
if old == nil && flight == nil && s.maxEntries > 0 && len(s.byKey) >= s.maxEntries {
|
||||
// Defensive direct Put callers do not own a reserved admission slot.
|
||||
// Preserve every existing unexpired result and decline the new cache row.
|
||||
s.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
|
||||
identity, admissionSeq, executionKnown, executionOK, acknowledged := rpcResultFlightMetadataLocked(s, key)
|
||||
var oldEntry *rpcResultCacheEntry
|
||||
if old != nil {
|
||||
oldEntry = old.Value.(*rpcResultCacheEntry)
|
||||
if flight == nil {
|
||||
// The first terminal publication is immutable, whether it retained an
|
||||
// legacy inline body, receipt or capacity tombstone. A
|
||||
// stale callback must not replace exact bytes, extend TTL, resurrect an
|
||||
// ACKed body or valid receipt.
|
||||
s.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
var reservation *rpcResultBudgetReservation
|
||||
switch {
|
||||
case flight != nil:
|
||||
reservation = flight.reservation
|
||||
if reservation == nil {
|
||||
s.mu.Unlock()
|
||||
panic("mtprotoedge: pending rpc result has no fair-budget reservation")
|
||||
}
|
||||
case oldEntry != nil && oldEntry.reservation != nil:
|
||||
reservation = oldEntry.reservation
|
||||
default:
|
||||
reservation = c.fairBudget.reserveCompleted(key, accountedSize)
|
||||
if reservation == nil {
|
||||
s.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
retainedSize := accountedSize
|
||||
retained := encoded
|
||||
capacity := false
|
||||
if acknowledged {
|
||||
const receiptSize = 1
|
||||
if !reservation.resizeBytes(receiptSize) {
|
||||
s.mu.Unlock()
|
||||
panic("mtprotoedge: acknowledged rpc result owner lost its receipt reservation")
|
||||
}
|
||||
retainedSize = receiptSize
|
||||
retained = nil
|
||||
} else if logicalOutbox {
|
||||
const receiptSize = 1
|
||||
if !reservation.resizeBytes(receiptSize) {
|
||||
s.mu.Unlock()
|
||||
panic("mtprotoedge: logical rpc result owner lost its receipt reservation")
|
||||
}
|
||||
retainedSize = receiptSize
|
||||
retained = nil
|
||||
capacity = !logicalReplayable
|
||||
} else if !reservation.resizeBytes(accountedSize) {
|
||||
if flight == nil {
|
||||
// A direct replacement cannot discard the prior replay body. Leave it
|
||||
// untouched and let Put perform one cross-shard expiry reap before its
|
||||
// final bounded failure.
|
||||
if oldEntry == nil {
|
||||
reservation.release()
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
// Owner admission already reserved one byte at all three scopes. When the
|
||||
// actual body cannot expand, transfer that reservation to an identity
|
||||
// tombstone so a duplicate never reruns business.
|
||||
const tombstoneSize = 1
|
||||
if !reservation.resizeBytes(tombstoneSize) {
|
||||
s.mu.Unlock()
|
||||
panic("mtprotoedge: rpc result owner lost its one-byte tombstone reservation")
|
||||
}
|
||||
retainedSize = tombstoneSize
|
||||
retained = nil
|
||||
capacity = true
|
||||
}
|
||||
|
||||
if old != nil {
|
||||
s.unlinkElement(old)
|
||||
if oldEntry.reservation != nil && oldEntry.reservation != reservation {
|
||||
oldEntry.reservation.release()
|
||||
oldEntry.reservation = nil
|
||||
}
|
||||
}
|
||||
entry := &rpcResultCacheEntry{
|
||||
key: key,
|
||||
encoded: retained,
|
||||
size: retainedSize,
|
||||
expiresAt: now.Add(s.ttl),
|
||||
identity: identity,
|
||||
admissionSeq: admissionSeq,
|
||||
executionKnown: executionKnown,
|
||||
executionOK: executionOK,
|
||||
acknowledged: acknowledged,
|
||||
capacity: capacity,
|
||||
reservation: reservation,
|
||||
}
|
||||
elem := s.order.PushBack(entry)
|
||||
s.byKey[key] = elem
|
||||
s.bytes += retainedSize
|
||||
|
||||
// Resolve the independent in-flight entry only after either the completed
|
||||
// result or its replay tombstone is published under the same shard lock.
|
||||
subscribers, executionSubscribers, executionOK := c.completeRPCResultFlightLocked(s, key, encoded)
|
||||
if acknowledged {
|
||||
// ACK won before terminal publication. Current subscribers still receive
|
||||
// encoded below, but no post-ACK receipt survives this critical section.
|
||||
s.removeElement(elem)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
for _, subscriber := range subscribers {
|
||||
subscriber(encoded, true)
|
||||
}
|
||||
for _, subscriber := range executionSubscribers {
|
||||
subscriber(executionOK)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func rpcResultFlightMetadataLocked(s *rpcResultCacheShard, key rpcResultCacheKey) (
|
||||
rpcResultRequestIdentity,
|
||||
uint64,
|
||||
bool,
|
||||
bool,
|
||||
bool,
|
||||
) {
|
||||
if flight := s.pending[key]; flight != nil {
|
||||
return flight.identity, flight.admissionSeq, flight.executionDone, flight.executionOK, flight.acknowledged
|
||||
}
|
||||
return rpcResultRequestIdentity{}, 0, false, false, false
|
||||
}
|
||||
|
||||
// expireCompletedResults performs the cold-path cross-shard reap used only
|
||||
// after a one-byte admission reservation fails. The caller must hold no shard
|
||||
// lock. Each shard is reaped independently so ordinary result publication on
|
||||
// the other shards remains parallel.
|
||||
func (c *rpcResultCache) expireCompletedResults() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
for i := range c.shards {
|
||||
s := &c.shards[i]
|
||||
s.mu.Lock()
|
||||
s.expireLocked(s.now())
|
||||
s.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *rpcResultCache) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *rpcResultCache) CloseContext(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgetCompletedSession removes every terminal receipt for a logical session.
|
||||
// SessionManager invokes it only after the session's physical producers have
|
||||
// converged, so no pending owner should remain and no later terminal callback
|
||||
// can recreate a receipt for the destroyed outbox.
|
||||
func (c *rpcResultCache) forgetCompletedSession(authKeyID [8]byte, sessionID int64) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
for i := range c.shards {
|
||||
s := &c.shards[i]
|
||||
s.mu.Lock()
|
||||
for elem := s.order.Front(); elem != nil; {
|
||||
next := elem.Next()
|
||||
entry := elem.Value.(*rpcResultCacheEntry)
|
||||
if entry.key.authKeyID == authKeyID && entry.key.sessionID == sessionID {
|
||||
s.removeElement(elem)
|
||||
}
|
||||
elem = next
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *rpcResultCacheShard) expireLocked(now time.Time) {
|
||||
for elem := s.order.Front(); elem != nil; {
|
||||
next := elem.Next()
|
||||
entry := elem.Value.(*rpcResultCacheEntry)
|
||||
if entry.expiresAt.After(now) {
|
||||
return
|
||||
}
|
||||
s.removeElement(elem)
|
||||
elem = next
|
||||
}
|
||||
}
|
||||
|
||||
func (s *rpcResultCacheShard) removeElement(elem *list.Element) {
|
||||
entry := s.unlinkElement(elem)
|
||||
if entry != nil && entry.reservation != nil {
|
||||
entry.reservation.release()
|
||||
entry.reservation = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *rpcResultCacheShard) unlinkElement(elem *list.Element) *rpcResultCacheEntry {
|
||||
if elem == nil {
|
||||
return nil
|
||||
}
|
||||
entry := elem.Value.(*rpcResultCacheEntry)
|
||||
delete(s.byKey, entry.key)
|
||||
s.bytes -= entry.size
|
||||
if s.bytes < 0 {
|
||||
s.bytes = 0
|
||||
}
|
||||
s.order.Remove(elem)
|
||||
return entry
|
||||
}
|
||||
|
||||
type rpcResultCacheByteBudget struct {
|
||||
max int64
|
||||
used atomic.Int64
|
||||
}
|
||||
|
||||
func (b *rpcResultCacheByteBudget) reserve(n int) bool {
|
||||
if n <= 0 {
|
||||
return true
|
||||
}
|
||||
bytes := int64(n)
|
||||
if b == nil || bytes > b.max {
|
||||
return false
|
||||
}
|
||||
for {
|
||||
used := b.used.Load()
|
||||
if used > b.max-bytes {
|
||||
return false
|
||||
}
|
||||
if b.used.CompareAndSwap(used, used+bytes) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *rpcResultCacheByteBudget) release(n int) {
|
||||
if b == nil || n <= 0 {
|
||||
return
|
||||
}
|
||||
if remaining := b.used.Add(-int64(n)); remaining < 0 {
|
||||
panic("mtprotoedge: rpc result completed-byte budget underflow")
|
||||
}
|
||||
}
|
||||
|
||||
func (b *rpcResultCacheByteBudget) snapshot() int64 {
|
||||
if b == nil {
|
||||
return 0
|
||||
}
|
||||
return b.used.Load()
|
||||
}
|
||||
|
|
@ -1,822 +0,0 @@
|
|||
package mtprotoedge
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRPCResultCacheFullSessionDoesNotBlockAnotherAuth(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithFairCapacity(func() time.Time { return now }, rpcResultCacheCapacity{
|
||||
maxPending: 8, maxPendingPerAuth: 6,
|
||||
globalMaxEntries: 8, globalMaxBytes: 64,
|
||||
authMaxEntries: 6, authMaxBytes: 48,
|
||||
sessionMaxEntries: 2, sessionMaxBytes: 16,
|
||||
})
|
||||
authA := [8]byte{0xa1}
|
||||
authB := [8]byte{0xb1}
|
||||
const sessionA = int64(77)
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
msgID := int64(1000 + i)
|
||||
claim, err := cache.Acquire(authA, sessionA, msgID)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("same-session admission %d = %#v, %v", i, claim, err)
|
||||
}
|
||||
cache.Put(authA, sessionA, msgID, &encodedOutboundMessage{body: []byte{1}})
|
||||
}
|
||||
if _, err := cache.Acquire(authA, sessionA, 2000); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("admission beyond session entry limit = %v, want capacity", err)
|
||||
}
|
||||
otherAuth, err := cache.Acquire(authB, 88, 3000)
|
||||
if err != nil || otherAuth.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("other auth blocked by full session: %#v, %v", otherAuth, err)
|
||||
}
|
||||
if !otherAuth.owner.Abort() {
|
||||
t.Fatal("other-auth owner did not abort")
|
||||
}
|
||||
if _, ok := cache.Get(authA, sessionA, 1000); !ok {
|
||||
t.Fatal("session capacity pressure evicted an unexpired result")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheFullAuthDoesNotBlockAnotherAuth(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
|
||||
maxPending: 8, maxPendingPerAuth: 4,
|
||||
globalMaxEntries: 8, globalMaxBytes: 64,
|
||||
authMaxEntries: 2, authMaxBytes: 32,
|
||||
sessionMaxEntries: 2, sessionMaxBytes: 16,
|
||||
})
|
||||
authA := [8]byte{0xa2}
|
||||
authB := [8]byte{0xb2}
|
||||
for i := 0; i < 2; i++ {
|
||||
claim, err := cache.Acquire(authA, int64(10+i), int64(100+i))
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("auth A admission %d = %#v, %v", i, claim, err)
|
||||
}
|
||||
cache.Put(authA, int64(10+i), int64(100+i), &encodedOutboundMessage{body: []byte{1}})
|
||||
}
|
||||
if _, err := cache.Acquire(authA, 12, 102); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("same-auth new session at auth limit = %v, want capacity", err)
|
||||
}
|
||||
other, err := cache.Acquire(authB, 20, 200)
|
||||
if err != nil || other.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("other auth blocked by full auth A: %#v, %v", other, err)
|
||||
}
|
||||
other.owner.Abort()
|
||||
}
|
||||
|
||||
func TestRPCResultCacheAuthAndSessionByteLimitsAreIndependent(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
|
||||
maxPending: 8, maxPendingPerAuth: 6,
|
||||
globalMaxEntries: 10, globalMaxBytes: 10,
|
||||
authMaxEntries: 8, authMaxBytes: 4,
|
||||
sessionMaxEntries: 6, sessionMaxBytes: 2,
|
||||
})
|
||||
authA := [8]byte{0xa4}
|
||||
authB := [8]byte{0xb4}
|
||||
first, err := cache.Acquire(authA, 1, 101)
|
||||
if err != nil || first.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("first owner = %#v, %v", first, err)
|
||||
}
|
||||
cache.Put(authA, 1, 101, &encodedOutboundMessage{body: []byte{1, 2}})
|
||||
if _, err := cache.Acquire(authA, 1, 102); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("same session beyond byte limit = %v, want capacity", err)
|
||||
}
|
||||
second, err := cache.Acquire(authA, 2, 201)
|
||||
if err != nil || second.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("second session owner = %#v, %v", second, err)
|
||||
}
|
||||
cache.Put(authA, 2, 201, &encodedOutboundMessage{body: []byte{3, 4}})
|
||||
if _, err := cache.Acquire(authA, 3, 301); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("same auth beyond byte limit = %v, want capacity", err)
|
||||
}
|
||||
other, err := cache.Acquire(authB, 3, 302)
|
||||
if err != nil || other.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("other auth blocked by auth A byte limit: %#v, %v", other, err)
|
||||
}
|
||||
other.owner.Abort()
|
||||
}
|
||||
|
||||
func TestRPCResultCachePerAuthPendingLimitIsAdditional(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
|
||||
maxPending: 6, maxPendingPerAuth: 2,
|
||||
globalMaxEntries: 12, globalMaxBytes: 64,
|
||||
authMaxEntries: 6, authMaxBytes: 32,
|
||||
sessionMaxEntries: 4, sessionMaxBytes: 16,
|
||||
})
|
||||
authA := [8]byte{0xa3}
|
||||
authB := [8]byte{0xb3}
|
||||
owners := make([]*rpcResultOwnerLease, 0, 3)
|
||||
for i := 0; i < 2; i++ {
|
||||
claim, err := cache.Acquire(authA, int64(i+1), int64(100+i))
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("pending auth A %d = %#v, %v", i, claim, err)
|
||||
}
|
||||
owners = append(owners, claim.owner)
|
||||
}
|
||||
if _, err := cache.Acquire(authA, 3, 103); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("third pending owner for auth A = %v, want capacity", err)
|
||||
}
|
||||
other, err := cache.Acquire(authB, 4, 104)
|
||||
if err != nil || other.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("auth B blocked by auth A pending limit: %#v, %v", other, err)
|
||||
}
|
||||
owners = append(owners, other.owner)
|
||||
for _, owner := range owners {
|
||||
if !owner.Abort() {
|
||||
t.Fatal("pending owner did not abort")
|
||||
}
|
||||
}
|
||||
if usage := cache.fairBudget.authSnapshot(authA); usage != (rpcResultBudgetUsage{}) {
|
||||
t.Fatalf("auth A budget after abort = %#v", usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheFairReservationLifecycleReturnsEveryScope(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithFairCapacity(func() time.Time { return now }, rpcResultCacheCapacity{
|
||||
maxPending: 4, maxPendingPerAuth: 3,
|
||||
globalMaxEntries: 6, globalMaxBytes: 10,
|
||||
authMaxEntries: 5, authMaxBytes: 8,
|
||||
sessionMaxEntries: 3, sessionMaxBytes: 6,
|
||||
})
|
||||
auth := [8]byte{0xc1}
|
||||
|
||||
aborted, err := cache.Acquire(auth, 1, 101)
|
||||
if err != nil || aborted.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("aborted owner = %#v, %v", aborted, err)
|
||||
}
|
||||
if usage := cache.fairBudget.authSnapshot(auth); usage.entries != 1 || usage.bytes != 1 || usage.pending != 1 {
|
||||
t.Fatalf("pending auth reservation = %#v", usage)
|
||||
}
|
||||
if !aborted.owner.Abort() {
|
||||
t.Fatal("owner Abort lost")
|
||||
}
|
||||
if usage := cache.fairBudget.authSnapshot(auth); usage != (rpcResultBudgetUsage{}) {
|
||||
t.Fatalf("Abort leaked auth reservation %#v", usage)
|
||||
}
|
||||
|
||||
body, err := cache.Acquire(auth, 1, 102)
|
||||
if err != nil || body.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("body owner = %#v, %v", body, err)
|
||||
}
|
||||
cache.Put(auth, 1, 102, &encodedOutboundMessage{body: make([]byte, 4)})
|
||||
if usage := cache.fairBudget.sessionSnapshot(auth, 1); usage.entries != 1 || usage.bytes != 4 || usage.pending != 0 {
|
||||
t.Fatalf("body session reservation = %#v", usage)
|
||||
}
|
||||
|
||||
tombstone, err := cache.Acquire(auth, 2, 201)
|
||||
if err != nil || tombstone.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("tombstone owner = %#v, %v", tombstone, err)
|
||||
}
|
||||
// This cannot fit the 10-byte global or 8-byte auth ceiling. Put must not
|
||||
// panic or lose ownership; it transfers the one-byte token to a tombstone.
|
||||
cache.Put(auth, 2, 201, &encodedOutboundMessage{body: make([]byte, 20)})
|
||||
if usage := cache.fairBudget.sessionSnapshot(auth, 2); usage.entries != 1 || usage.bytes != 1 || usage.pending != 0 {
|
||||
t.Fatalf("tombstone session reservation = %#v", usage)
|
||||
}
|
||||
if got := cache.completedEntries.snapshot(); got != 2 {
|
||||
t.Fatalf("global entries after body+tombstone = %d, want 2", got)
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != 5 {
|
||||
t.Fatalf("global bytes after body+tombstone = %d, want 5", got)
|
||||
}
|
||||
|
||||
cache.Put(auth, 1, 102, &encodedOutboundMessage{body: make([]byte, 2)})
|
||||
if got := cache.completedBytes.snapshot(); got != 5 {
|
||||
t.Fatalf("duplicate publication changed immutable global bytes: %d", got)
|
||||
}
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
_, _ = cache.Get(auth, 1, 102)
|
||||
_, _ = cache.Get(auth, 2, 201)
|
||||
if got := cache.completedEntries.snapshot(); got != 0 {
|
||||
t.Fatalf("TTL leaked global entries %d", got)
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != 0 {
|
||||
t.Fatalf("TTL leaked global bytes %d", got)
|
||||
}
|
||||
if usage := cache.fairBudget.authSnapshot(auth); usage != (rpcResultBudgetUsage{}) {
|
||||
t.Fatalf("TTL leaked auth reservation %#v", usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheFullKeyMaphashSpreadsOneSession(t *testing.T) {
|
||||
first := newRPCResultCacheWithFlightLimit(time.Now, 64)
|
||||
second := newRPCResultCacheWithFlightLimit(time.Now, 64)
|
||||
auth := [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
|
||||
const sessionID = int64(99)
|
||||
seen := make(map[uint64]struct{})
|
||||
differentInstance := false
|
||||
for msgID := int64(1); msgID <= 256; msgID++ {
|
||||
key := rpcResultCacheKey{authKeyID: auth, sessionID: sessionID, reqMsgID: msgID}
|
||||
firstIndex := first.shardIndex(key)
|
||||
seen[firstIndex] = struct{}{}
|
||||
if firstIndex != second.shardIndex(key) {
|
||||
differentInstance = true
|
||||
}
|
||||
}
|
||||
if len(seen) < rpcResultCacheShards/2 {
|
||||
t.Fatalf("one session used only %d/%d full-key shards", len(seen), rpcResultCacheShards)
|
||||
}
|
||||
if !differentInstance {
|
||||
t.Fatal("two cache instances produced an identical shard stream; seed is not instance-random")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheConcurrentFairReservationsNeverOvercommit(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
|
||||
maxPending: 24, maxPendingPerAuth: 4,
|
||||
globalMaxEntries: 24, globalMaxBytes: 24,
|
||||
authMaxEntries: 8, authMaxBytes: 8,
|
||||
sessionMaxEntries: 3, sessionMaxBytes: 3,
|
||||
})
|
||||
const callers = 256
|
||||
start := make(chan struct{})
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
owners []*rpcResultOwnerLease
|
||||
)
|
||||
for i := 0; i < callers; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
auth := [8]byte{byte(i % 4)}
|
||||
claim, err := cache.Acquire(auth, int64(i%8), int64(1000+i))
|
||||
if errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
return
|
||||
}
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Errorf("Acquire %d = %#v, %v", i, claim, err)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
owners = append(owners, claim.owner)
|
||||
mu.Unlock()
|
||||
}(i)
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
if got := cache.completedEntries.snapshot(); got > 24 || got != int64(len(owners)) {
|
||||
t.Fatalf("global entry usage=%d owners=%d limit=24", got, len(owners))
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got > 24 || got != int64(len(owners)) {
|
||||
t.Fatalf("global byte usage=%d owners=%d limit=24", got, len(owners))
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
auth := [8]byte{byte(i)}
|
||||
usage := cache.fairBudget.authSnapshot(auth)
|
||||
if usage.entries > 8 || usage.bytes > 8 || usage.pending > 4 {
|
||||
t.Fatalf("auth %d overcommitted: %#v", i, usage)
|
||||
}
|
||||
for sessionID := int64(0); sessionID < 8; sessionID++ {
|
||||
session := cache.fairBudget.sessionSnapshot(auth, sessionID)
|
||||
if session.entries > 3 || session.bytes > 3 {
|
||||
t.Fatalf("auth %d session %d overcommitted: %#v", i, sessionID, session)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, owner := range owners {
|
||||
if !owner.Abort() {
|
||||
t.Fatal("concurrent owner did not abort")
|
||||
}
|
||||
}
|
||||
if cache.completedEntries.snapshot() != 0 || cache.completedBytes.snapshot() != 0 {
|
||||
t.Fatal("concurrent Abort leaked global fair budget")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheConcurrentOwnerPublicationAcrossShards(t *testing.T) {
|
||||
const publications = 256
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithFairCapacity(func() time.Time { return now }, rpcResultCacheCapacity{
|
||||
maxPending: publications, maxPendingPerAuth: 4,
|
||||
globalMaxEntries: publications, globalMaxBytes: publications * 4,
|
||||
authMaxEntries: 4, authMaxBytes: 16,
|
||||
sessionMaxEntries: 1, sessionMaxBytes: 4,
|
||||
})
|
||||
|
||||
type publication struct {
|
||||
auth [8]byte
|
||||
session int64
|
||||
msgID int64
|
||||
owner *rpcResultOwnerLease
|
||||
}
|
||||
publicationsByKey := make([]publication, 0, publications)
|
||||
for i := 0; i < publications; i++ {
|
||||
auth := [8]byte{byte(i), byte(i >> 8), 0xa5}
|
||||
sessionID := int64(10_000 + i)
|
||||
msgID := int64(20_000 + i)
|
||||
claim, err := cache.Acquire(auth, sessionID, msgID)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("Acquire %d = %#v, %v", i, claim, err)
|
||||
}
|
||||
publicationsByKey = append(publicationsByKey, publication{
|
||||
auth: auth, session: sessionID, msgID: msgID, owner: claim.owner,
|
||||
})
|
||||
}
|
||||
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for i := range publicationsByKey {
|
||||
item := publicationsByKey[i]
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
if !item.owner.CompleteExecution(true) {
|
||||
t.Errorf("CompleteExecution(%d) lost owner", item.msgID)
|
||||
return
|
||||
}
|
||||
cache.Put(item.auth, item.session, item.msgID, &encodedOutboundMessage{body: []byte{1, 2, 3, 4}})
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
|
||||
for _, item := range publicationsByKey {
|
||||
encoded, ok := cache.Get(item.auth, item.session, item.msgID)
|
||||
if !ok || encoded == nil || len(encoded.body) != 4 {
|
||||
t.Fatalf("completed publication %d missing: ok=%v encoded=%#v", item.msgID, ok, encoded)
|
||||
}
|
||||
}
|
||||
if got := cache.completedEntries.snapshot(); got != publications {
|
||||
t.Fatalf("completed entries=%d, want %d", got, publications)
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != publications*4 {
|
||||
t.Fatalf("completed bytes=%d, want %d", got, publications*4)
|
||||
}
|
||||
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
cache.expireCompletedResults()
|
||||
if cache.completedEntries.snapshot() != 0 || cache.completedBytes.snapshot() != 0 {
|
||||
t.Fatal("parallel publications leaked fair-budget reservations after TTL")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRPCResultCacheParallelShardPut(b *testing.B) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, rpcResultFlightDefaultMaxPending)
|
||||
var nextWorker atomic.Uint64
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
id := nextWorker.Add(1)
|
||||
auth := [8]byte{
|
||||
byte(id), byte(id >> 8), byte(id >> 16), byte(id >> 24),
|
||||
byte(id >> 32), byte(id >> 40), byte(id >> 48), byte(id >> 56),
|
||||
}
|
||||
sessionID := int64(id)
|
||||
msgID := int64(1_000_000 + id)
|
||||
encoded := &encodedOutboundMessage{body: []byte{1, 2, 3, 4}}
|
||||
for pb.Next() {
|
||||
cache.Put(auth, sessionID, msgID, encoded)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRPCResultCacheEntryReservationTransfersAndReturns(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithCapacity(func() time.Time { return now }, 4, 2, 2)
|
||||
authKeyID := [8]byte{0xa2}
|
||||
|
||||
first, err := cache.Acquire(authKeyID, 1, 101)
|
||||
if err != nil || first.state != rpcResultAcquireOwner || cache.completedEntries.snapshot() != 1 {
|
||||
t.Fatalf("first pending reservation = %#v entries=%d err=%v", first, cache.completedEntries.snapshot(), err)
|
||||
}
|
||||
cache.Put(authKeyID, 1, 101, &encodedOutboundMessage{body: []byte{1}})
|
||||
if got := cache.completedEntries.snapshot(); got != 1 {
|
||||
t.Fatalf("pending -> body changed entry count to %d", got)
|
||||
}
|
||||
|
||||
second, err := cache.Acquire(authKeyID, 2, 202)
|
||||
if err != nil || second.state != rpcResultAcquireOwner || cache.completedEntries.snapshot() != 2 {
|
||||
t.Fatalf("second pending reservation = %#v entries=%d err=%v", second, cache.completedEntries.snapshot(), err)
|
||||
}
|
||||
// The byte budget has only the second owner's one-byte token remaining.
|
||||
// Publication therefore leaves an identity tombstone, which still owns its
|
||||
// real process-wide entry slot.
|
||||
cache.Put(authKeyID, 2, 202, &encodedOutboundMessage{body: []byte{2, 2, 2}})
|
||||
if got := cache.completedEntries.snapshot(); got != 2 {
|
||||
t.Fatalf("pending -> tombstone changed entry count to %d", got)
|
||||
}
|
||||
if _, err := cache.Acquire(authKeyID, 3, 303); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("third admission at entry limit = %v, want capacity", err)
|
||||
}
|
||||
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
firstShard := cache.shardIndex(rpcResultCacheKey{authKeyID: authKeyID, sessionID: 1, reqMsgID: 101})
|
||||
secondShard := cache.shardIndex(rpcResultCacheKey{authKeyID: authKeyID, sessionID: 2, reqMsgID: 202})
|
||||
thirdMsgID := rpcResultTestMsgIDOutsideShards(t, cache, authKeyID, 3, 303, firstShard, secondShard)
|
||||
third, err := cache.Acquire(authKeyID, 3, thirdMsgID)
|
||||
if err != nil || third.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("admission after global expiry reap = %#v, %v", third, err)
|
||||
}
|
||||
if got := cache.completedEntries.snapshot(); got != 1 {
|
||||
t.Fatalf("expired entries were not returned before new owner: %d", got)
|
||||
}
|
||||
if !third.owner.Abort() || cache.completedEntries.snapshot() != 0 {
|
||||
t.Fatalf("Abort did not return entry reservation: entries=%d", cache.completedEntries.snapshot())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheConcurrentGlobalEntryReservationNeverOvercommits(t *testing.T) {
|
||||
const limit = 8
|
||||
cache := newRPCResultCacheWithCapacity(time.Now, 128, 1<<20, limit)
|
||||
authKeyID := [8]byte{0xa3}
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
owners []*rpcResultOwnerLease
|
||||
)
|
||||
for i := 0; i < 64; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
claim, err := cache.Acquire(authKeyID, int64(i+1), int64(1000+i))
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Errorf("Acquire %d: %v", i, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if claim.state != rpcResultAcquireOwner {
|
||||
t.Errorf("Acquire %d state = %d", i, claim.state)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
owners = append(owners, claim.owner)
|
||||
mu.Unlock()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
if len(owners) != limit || cache.completedEntries.snapshot() != limit {
|
||||
t.Fatalf("concurrent owners=%d entries=%d, want %d", len(owners), cache.completedEntries.snapshot(), limit)
|
||||
}
|
||||
for _, owner := range owners {
|
||||
if !owner.Abort() {
|
||||
t.Fatal("reserved owner failed to abort")
|
||||
}
|
||||
}
|
||||
if got := cache.completedEntries.snapshot(); got != 0 {
|
||||
t.Fatalf("entry reservations after abort = %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheRoundTripAndTTL(t *testing.T) {
|
||||
if rpcResultCacheTTL != 331*time.Second {
|
||||
t.Fatalf("replay TTL = %v, want full 300s past + 30s future window + 1s", rpcResultCacheTTL)
|
||||
}
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCache(func() time.Time { return now })
|
||||
|
||||
var keyID [8]byte
|
||||
keyID[0] = 0xab
|
||||
encoded := &encodedOutboundMessage{body: []byte{1, 2, 3, 4}, typeID: 42, reqMsgID: 7}
|
||||
|
||||
if _, ok := cache.Get(keyID, 5, 7); ok {
|
||||
t.Fatal("unexpected hit on empty cache")
|
||||
}
|
||||
cache.Put(keyID, 5, 7, encoded)
|
||||
if got := cache.completedEntries.snapshot(); got != 1 {
|
||||
t.Fatalf("direct Put entry reservation = %d, want 1", got)
|
||||
}
|
||||
if usage := cache.fairBudget.sessionSnapshot(keyID, 5); usage.entries != 1 || usage.bytes != 4 || usage.pending != 0 {
|
||||
t.Fatalf("direct Put session reservation = %#v", usage)
|
||||
}
|
||||
|
||||
got, ok := cache.Get(keyID, 5, 7)
|
||||
if !ok {
|
||||
t.Fatal("expected hit")
|
||||
}
|
||||
// encodedOutboundMessage 不可变契约下 Get/Put 共享指针,不做防御性拷贝。
|
||||
if got != encoded {
|
||||
t.Fatal("expected shared pointer, got clone")
|
||||
}
|
||||
|
||||
// 不同 session / msg_id 不串。
|
||||
if _, ok := cache.Get(keyID, 6, 7); ok {
|
||||
t.Fatal("hit with wrong session id")
|
||||
}
|
||||
if _, ok := cache.Get(keyID, 5, 8); ok {
|
||||
t.Fatal("hit with wrong msg id")
|
||||
}
|
||||
|
||||
// TTL 过期。
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
if _, ok := cache.Get(keyID, 5, 7); ok {
|
||||
t.Fatal("expected expiry after TTL")
|
||||
}
|
||||
if got := cache.completedEntries.snapshot(); got != 0 {
|
||||
t.Fatalf("direct Put expiry left %d entry reservations", got)
|
||||
}
|
||||
if usage := cache.fairBudget.authSnapshot(keyID); usage != (rpcResultBudgetUsage{}) {
|
||||
t.Fatalf("direct Put expiry leaked auth reservation %#v", usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheDuplicatePutPreservesCompletedExecutionMetadata(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
|
||||
keyID := [8]byte{1, 9, 8, 4}
|
||||
const sessionID, reqMsgID = int64(11), int64(12)
|
||||
claim, err := cache.Acquire(keyID, sessionID, reqMsgID)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("Acquire owner = %#v, %v", claim, err)
|
||||
}
|
||||
if !claim.owner.CompleteExecution(true) || !claim.owner.HandOff() {
|
||||
t.Fatal("complete owner metadata")
|
||||
}
|
||||
first := &encodedOutboundMessage{body: []byte{1, 2, 3, 4}, typeID: 42, reqMsgID: reqMsgID}
|
||||
cache.Put(keyID, sessionID, reqMsgID, first)
|
||||
second := &encodedOutboundMessage{body: []byte{5, 6, 7, 8}, typeID: 42, reqMsgID: reqMsgID}
|
||||
cache.Put(keyID, sessionID, reqMsgID, second)
|
||||
|
||||
replay, err := cache.Acquire(keyID, sessionID, reqMsgID)
|
||||
if err != nil || replay.state != rpcResultAcquireCompleted || replay.encoded != first ||
|
||||
!replay.executionKnown || !replay.executionOK {
|
||||
t.Fatalf("duplicate Put metadata = %#v, err=%v", replay, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheShardCapacityNeverEvictsUnexpiredResult(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCache(func() time.Time { return now })
|
||||
|
||||
var keyID [8]byte
|
||||
firstKey := rpcResultCacheKey{authKeyID: keyID, sessionID: 1, reqMsgID: 100}
|
||||
shard := cache.shard(firstKey)
|
||||
shard.mu.Lock()
|
||||
shard.maxEntries = 1
|
||||
shard.mu.Unlock()
|
||||
|
||||
claim, err := cache.Acquire(keyID, 1, 100)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("first admission = %#v, %v", claim, err)
|
||||
}
|
||||
first := &encodedOutboundMessage{body: []byte{1}}
|
||||
cache.Put(keyID, 1, 100, first)
|
||||
secondMsgID := rpcResultTestMsgIDForShard(t, cache, keyID, 1, 101, cache.shardIndex(firstKey))
|
||||
if _, err := cache.Acquire(keyID, 1, secondMsgID); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("full-shard admission = %v, want capacity", err)
|
||||
}
|
||||
if got, ok := cache.Get(keyID, 1, 100); !ok || got != first {
|
||||
t.Fatalf("unexpired first result was displaced: got=%p ok=%v", got, ok)
|
||||
}
|
||||
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
claim, err = cache.Acquire(keyID, 1, secondMsgID)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("admission after expiry = %#v, %v", claim, err)
|
||||
}
|
||||
claim.owner.Abort()
|
||||
}
|
||||
|
||||
func TestRPCResultCacheGlobalByteCapacityNeverEvictsUnexpiredResults(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithLimits(func() time.Time { return now }, 32, 10)
|
||||
var keyID [8]byte
|
||||
|
||||
// Five two-byte results consume the global budget. The sixth admission must
|
||||
// fail bounded; none of the retained results may be sacrificed for it.
|
||||
for sessionID := int64(1); sessionID <= 5; sessionID++ {
|
||||
claim, err := cache.Acquire(keyID, sessionID, 100+sessionID)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("admission %d = %#v, %v", sessionID, claim, err)
|
||||
}
|
||||
cache.Put(keyID, sessionID, 100+sessionID, &encodedOutboundMessage{body: []byte{1, 2}})
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != 10 {
|
||||
t.Fatalf("completed bytes at capacity = %d, want 10", got)
|
||||
}
|
||||
if _, err := cache.Acquire(keyID, 6, 106); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("byte-full admission = %v, want capacity", err)
|
||||
}
|
||||
for sessionID := int64(1); sessionID <= 5; sessionID++ {
|
||||
if _, ok := cache.Get(keyID, sessionID, 100+sessionID); !ok {
|
||||
t.Fatalf("unexpired result %d was evicted", sessionID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheByteBudgetReturnsOnReplaceExpiryAndCapacity(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithLimits(func() time.Time { return now }, 32, 32)
|
||||
var keyID [8]byte
|
||||
|
||||
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 4)})
|
||||
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 7)})
|
||||
if got := cache.completedBytes.snapshot(); got != 4 {
|
||||
t.Fatalf("completed bytes after duplicate publication = %d, want 4", got)
|
||||
}
|
||||
if usage := cache.fairBudget.sessionSnapshot(keyID, 1); usage.entries != 1 || usage.bytes != 4 {
|
||||
t.Fatalf("immutable fair reservation after duplicate = %#v", usage)
|
||||
}
|
||||
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 2)})
|
||||
if got := cache.completedBytes.snapshot(); got != 4 {
|
||||
t.Fatalf("completed bytes after second duplicate = %d, want 4", got)
|
||||
}
|
||||
if usage := cache.fairBudget.sessionSnapshot(keyID, 1); usage.entries != 1 || usage.bytes != 4 {
|
||||
t.Fatalf("immutable fair reservation after second duplicate = %#v", usage)
|
||||
}
|
||||
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
if _, ok := cache.Get(keyID, 1, 101); ok {
|
||||
t.Fatal("replacement should expire")
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != 0 {
|
||||
t.Fatalf("completed bytes after expiry = %d, want 0", got)
|
||||
}
|
||||
|
||||
key := rpcResultCacheKey{authKeyID: keyID, sessionID: 2, reqMsgID: 201}
|
||||
shard := cache.shard(key)
|
||||
shard.mu.Lock()
|
||||
shard.maxEntries = 1
|
||||
shard.mu.Unlock()
|
||||
claim, err := cache.Acquire(keyID, 2, 201)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("entry-capacity first admission = %#v, %v", claim, err)
|
||||
}
|
||||
cache.Put(keyID, 2, 201, &encodedOutboundMessage{body: make([]byte, 3)})
|
||||
secondMsgID := rpcResultTestMsgIDForShard(t, cache, keyID, 2, 202, cache.shardIndex(key))
|
||||
if _, err := cache.Acquire(keyID, 2, secondMsgID); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("entry-capacity second admission = %v", err)
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != 3 {
|
||||
t.Fatalf("completed bytes after capacity rejection = %d, want 3", got)
|
||||
}
|
||||
if _, ok := cache.Get(keyID, 2, 201); !ok {
|
||||
t.Fatal("capacity rejection displaced the first result")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCachePublicationOverflowLeavesReplayCapacityTombstone(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithLimits(func() time.Time { return now }, 32, 4)
|
||||
var keyID [8]byte
|
||||
claim, err := cache.Acquire(keyID, 1, 101)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("owner admission = %#v, %v", claim, err)
|
||||
}
|
||||
claim.owner.CompleteExecution(true)
|
||||
tooLarge := &encodedOutboundMessage{body: make([]byte, 5)}
|
||||
cache.Put(keyID, 1, 101, tooLarge)
|
||||
if got := cache.completedBytes.snapshot(); got != 1 {
|
||||
t.Fatalf("tombstone bytes = %d, want 1", got)
|
||||
}
|
||||
if _, ok := cache.Get(keyID, 1, 101); ok {
|
||||
t.Fatal("capacity tombstone must not masquerade as a replayable body")
|
||||
}
|
||||
if _, err := cache.Acquire(keyID, 1, 101); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("duplicate after publication overflow = %v, want capacity", err)
|
||||
}
|
||||
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
retry, err := cache.Acquire(keyID, 1, 101)
|
||||
if err != nil || retry.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("admission after tombstone expiry = %#v, %v", retry, err)
|
||||
}
|
||||
retry.owner.Abort()
|
||||
}
|
||||
|
||||
func TestRPCResultCacheByteCapacityReclaimsExpiredAcrossShards(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithLimits(func() time.Time { return now }, 32, 2)
|
||||
var keyID [8]byte
|
||||
|
||||
first, err := cache.Acquire(keyID, 1, 101)
|
||||
if err != nil || first.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("first admission = %#v, %v", first, err)
|
||||
}
|
||||
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: []byte{1, 2}})
|
||||
if got := cache.completedBytes.snapshot(); got != 2 {
|
||||
t.Fatalf("full budget = %d, want 2", got)
|
||||
}
|
||||
|
||||
// Select a key in another full-key shard. Its failed one-byte reservation
|
||||
// must trigger the cold-path global expiry reap before returning capacity.
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
firstKey := rpcResultCacheKey{authKeyID: keyID, sessionID: 1, reqMsgID: 101}
|
||||
secondMsgID := rpcResultTestMsgIDOutsideShard(t, cache, keyID, 2, 202, cache.shardIndex(firstKey))
|
||||
second, err := cache.Acquire(keyID, 2, secondMsgID)
|
||||
if err != nil || second.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("cross-shard admission after expiry = %#v, %v", second, err)
|
||||
}
|
||||
second.owner.Abort()
|
||||
if got := cache.completedBytes.snapshot(); got != 0 {
|
||||
t.Fatalf("bytes after expired reap and abort = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheServerOptionsPropagateFairLimits(t *testing.T) {
|
||||
s := New(Options{
|
||||
RPCGlobalMaxTasks: 6,
|
||||
RPCResultCacheMaxEntries: 12,
|
||||
RPCResultCacheAuthMaxEntries: 8,
|
||||
RPCResultCacheSessionMaxEntries: 4,
|
||||
RPCResultPendingPerAuth: 3,
|
||||
})
|
||||
if s.rpcResults.completedEntries.max != 12 || s.rpcResults.completedBytes.max != 12 {
|
||||
t.Fatalf("global option propagation = %d/%d", s.rpcResults.completedEntries.max, s.rpcResults.completedBytes.max)
|
||||
}
|
||||
budget := s.rpcResults.fairBudget
|
||||
if budget.authLimit.entries != 8 || budget.authLimit.bytes != 8 ||
|
||||
budget.sessionLimit.entries != 4 || budget.sessionLimit.bytes != 4 || budget.pendingPerAuth != 3 {
|
||||
t.Fatalf("fair option propagation = auth:%#v session:%#v pending:%d",
|
||||
budget.authLimit, budget.sessionLimit, budget.pendingPerAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheServerOptionsFailFast(t *testing.T) {
|
||||
base := Options{
|
||||
RPCGlobalMaxTasks: 6,
|
||||
RPCResultCacheMaxEntries: 12,
|
||||
RPCResultCacheAuthMaxEntries: 8,
|
||||
RPCResultCacheSessionMaxEntries: 4,
|
||||
RPCResultPendingPerAuth: 3,
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*Options)
|
||||
}{
|
||||
{name: "entry hierarchy", mutate: func(o *Options) { o.RPCResultCacheAuthMaxEntries = 13 }},
|
||||
{name: "pending hierarchy", mutate: func(o *Options) { o.RPCResultPendingPerAuth = 7 }},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
opts := base
|
||||
test.mutate(&opts)
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("New accepted invalid rpc_result cache options")
|
||||
}
|
||||
}()
|
||||
_ = New(opts)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func rpcResultTestMsgIDForShard(
|
||||
t *testing.T,
|
||||
cache *rpcResultCache,
|
||||
authKeyID [8]byte,
|
||||
sessionID, start int64,
|
||||
target uint64,
|
||||
) int64 {
|
||||
t.Helper()
|
||||
for msgID := start; msgID < start+1_000_000; msgID++ {
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: msgID}
|
||||
if cache.shardIndex(key) == target {
|
||||
return msgID
|
||||
}
|
||||
}
|
||||
t.Fatal("failed to find rpc_result key for target shard")
|
||||
return 0
|
||||
}
|
||||
|
||||
func rpcResultTestMsgIDOutsideShard(
|
||||
t *testing.T,
|
||||
cache *rpcResultCache,
|
||||
authKeyID [8]byte,
|
||||
sessionID, start int64,
|
||||
excluded uint64,
|
||||
) int64 {
|
||||
t.Helper()
|
||||
for msgID := start; msgID < start+1_000_000; msgID++ {
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: msgID}
|
||||
if cache.shardIndex(key) != excluded {
|
||||
return msgID
|
||||
}
|
||||
}
|
||||
t.Fatal("failed to find rpc_result key outside excluded shard")
|
||||
return 0
|
||||
}
|
||||
|
||||
func rpcResultTestMsgIDOutsideShards(
|
||||
t *testing.T,
|
||||
cache *rpcResultCache,
|
||||
authKeyID [8]byte,
|
||||
sessionID, start int64,
|
||||
excluded ...uint64,
|
||||
) int64 {
|
||||
t.Helper()
|
||||
for msgID := start; msgID < start+1_000_000; msgID++ {
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: msgID}
|
||||
index := cache.shardIndex(key)
|
||||
allowed := true
|
||||
for _, blocked := range excluded {
|
||||
if index == blocked {
|
||||
allowed = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allowed {
|
||||
return msgID
|
||||
}
|
||||
}
|
||||
t.Fatal("failed to find rpc_result key outside excluded shards")
|
||||
return 0
|
||||
}
|
||||
|
|
@ -92,7 +92,7 @@ func (t *blockingCloseRPCResultTransport) Close() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func TestRPCResultCachePublishesOnlyAfterPhysicalWrite(t *testing.T) {
|
||||
func TestRPCExecutionLedgerPublishesOnlyAfterPhysicalWrite(t *testing.T) {
|
||||
tr := newGatedRequiredControlTransport(nil)
|
||||
s := New(Options{WriteTimeout: time.Second})
|
||||
key := newTestAuthKey(t)
|
||||
|
|
@ -113,7 +113,7 @@ func TestRPCResultCachePublishesOnlyAfterPhysicalWrite(t *testing.T) {
|
|||
case <-time.After(time.Second):
|
||||
t.Fatal("rpc_result did not reach physical writer")
|
||||
}
|
||||
if _, ok := s.rpcResults.Get(key.ID, c.sessionID, reqMsgID); ok {
|
||||
if _, ok := s.rpcResults.Replay(key.ID, c.sessionID, reqMsgID); ok {
|
||||
t.Fatal("rpc_result became completed before physical write")
|
||||
}
|
||||
pending, err := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
|
||||
|
|
@ -280,7 +280,7 @@ func TestRouterUpdateCursorDoesNotCommitAfterPhysicalWriteFailure(t *testing.T)
|
|||
deadline := time.Now().Add(time.Second)
|
||||
replayable := false
|
||||
for time.Now().Before(deadline) {
|
||||
if cached, ok := s.rpcResults.Get(key.ID, c.sessionID, reqMsgID); ok && cached.deliveryState() == rpcResultDeliveryReplayable {
|
||||
if cached, ok := s.rpcResults.Replay(key.ID, c.sessionID, reqMsgID); ok && cached.deliveryState() == rpcResultDeliveryReplayable {
|
||||
replayable = true
|
||||
break
|
||||
}
|
||||
|
|
@ -334,7 +334,7 @@ func TestRouterUpdateCursorDoesNotCommitWhenResultEncodingFails(t *testing.T) {
|
|||
}
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if cached, ok := s.rpcResults.Get(key.ID, c.sessionID, reqMsgID); ok && cached.deliveryState() == rpcResultDeliveryDelivered {
|
||||
if cached, ok := s.rpcResults.Replay(key.ID, c.sessionID, reqMsgID); ok && cached.deliveryState() == rpcResultDeliveryDelivered {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
|
|
|
|||
|
|
@ -203,7 +203,7 @@ func (h *saturatedSlotWaveRPC) Dispatch(context.Context, [8]byte, int64, *bin.Bu
|
|||
|
||||
func (*saturatedSlotWaveRPC) NegotiatedLayer([8]byte, int64) (int, bool) { return 227, true }
|
||||
|
||||
func TestPublishRPCResultSaturatedBudgetRetainsExactResultsAcrossSlotWaves(t *testing.T) {
|
||||
func TestPublishRPCResultSaturatedBudgetLeavesExecutionTombstonesAcrossSlotWaves(t *testing.T) {
|
||||
slotCount := cap(outboundEncodeSlots)
|
||||
requestCount := slotCount*2 + 1
|
||||
gate := &saturatedSlotWaveGate{
|
||||
|
|
@ -214,7 +214,7 @@ func TestPublishRPCResultSaturatedBudgetRetainsExactResultsAcrossSlotWaves(t *te
|
|||
handler := &saturatedSlotWaveRPC{gate: gate}
|
||||
s := New(Options{legacyRPC: handler})
|
||||
now := time.Unix(1_700_000_000, 0)
|
||||
s.rpcResults = newRPCResultCacheWithFlightLimit(func() time.Time { return now }, requestCount+1)
|
||||
s.rpcResults = newRPCExecutionLedgerForServerTest(s, func() time.Time { return now }, requestCount+1)
|
||||
|
||||
const primaryMax = 1 << 20
|
||||
primary := newOutboundTrackedBudget(primaryMax)
|
||||
|
|
@ -302,7 +302,6 @@ func TestPublishRPCResultSaturatedBudgetRetainsExactResultsAcrossSlotWaves(t *te
|
|||
t.Fatalf("primary budget changed under saturation = %d, want %d", got, primaryMax)
|
||||
}
|
||||
|
||||
var completedBytes int64
|
||||
for i, c := range conns {
|
||||
if !errors.Is(errs[i], ErrOutboundTrackedBudget) {
|
||||
t.Fatalf("publish request %d error = %v, want terminal budget saturation", i, errs[i])
|
||||
|
|
@ -311,49 +310,31 @@ func TestPublishRPCResultSaturatedBudgetRetainsExactResultsAcrossSlotWaves(t *te
|
|||
t.Fatalf("request %d connection was not explicitly fenced", i)
|
||||
}
|
||||
if !owners[i].handedOff.Load() {
|
||||
t.Fatalf("request %d owner was not handed to completed cache", i)
|
||||
t.Fatalf("request %d owner was not handed to execution ledger", i)
|
||||
}
|
||||
cached, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgIDs[i])
|
||||
if !ok || cached == nil {
|
||||
t.Fatalf("request %d exact result missing from completed cache", i)
|
||||
if cached, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgIDs[i]); ok || cached != nil {
|
||||
t.Fatalf("request %d retained a payload despite outbox saturation", i)
|
||||
}
|
||||
completedBytes += int64(len(cached.body))
|
||||
var envelope proto.Result
|
||||
if err := envelope.Decode(&bin.Buffer{Buf: cached.body}); err != nil {
|
||||
t.Fatalf("decode request %d cached rpc_result: %v", i, err)
|
||||
}
|
||||
if envelope.RequestMessageID != reqMsgIDs[i] {
|
||||
t.Fatalf("request %d cached req_msg_id = %d, want %d", i, envelope.RequestMessageID, reqMsgIDs[i])
|
||||
}
|
||||
var result tg.DataJSON
|
||||
if err := result.Decode(&bin.Buffer{Buf: envelope.Result}); err != nil {
|
||||
t.Fatalf("decode request %d exact business result (possibly INTERNAL): %v", i, err)
|
||||
}
|
||||
if result.Data != saturatedSlotWaveResultData {
|
||||
t.Fatalf("request %d cached result = %q, want %q", i, result.Data, saturatedSlotWaveResultData)
|
||||
}
|
||||
retry, err := s.rpcResults.Acquire(c.authKeyID, c.sessionID, reqMsgIDs[i])
|
||||
if err != nil || retry.state != rpcResultAcquireCompleted || retry.encoded != cached {
|
||||
t.Fatalf("retry request %d = %+v err=%v, want exact completed result", i, retry, err)
|
||||
if _, err := s.rpcResults.Acquire(c.authKeyID, c.sessionID, reqMsgIDs[i]); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("retry request %d err=%v, want execution tombstone capacity", i, err)
|
||||
}
|
||||
}
|
||||
if got := handler.calls.Load(); got != int32(requestCount) {
|
||||
t.Fatalf("business executions after retries = %d, want unchanged %d", got, requestCount)
|
||||
}
|
||||
if got := s.rpcResults.completedBytes.snapshot(); got != completedBytes {
|
||||
t.Fatalf("completed-cache charge = %d, want exact retained bytes %d", got, completedBytes)
|
||||
if got := s.rpcResults.receiptBudgetBytes(); got != int64(requestCount*rpcExecutionReceiptBudgetBytes) {
|
||||
t.Fatalf("receipt budget = %d, want %d fixed metadata bytes", got, requestCount*rpcExecutionReceiptBudgetBytes)
|
||||
}
|
||||
|
||||
// Expiry is the completed cache's ownership release point. Force it
|
||||
// deterministically and prove every retained byte is returned exactly once.
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
// Expiry releases only execution receipts; no result body was retained.
|
||||
now = now.Add(rpcExecutionReceiptTTL + time.Second)
|
||||
for i, c := range conns {
|
||||
if _, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgIDs[i]); ok {
|
||||
t.Fatalf("request %d remained cached after forced expiry", i)
|
||||
if _, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgIDs[i]); ok {
|
||||
t.Fatalf("request %d remained replayable after forced expiry", i)
|
||||
}
|
||||
}
|
||||
if got := s.rpcResults.completedBytes.snapshot(); got != 0 {
|
||||
t.Fatalf("completed-cache bytes after expiry = %d, want 0", got)
|
||||
if got := s.rpcResults.receiptBudgetBytes(); got != 0 {
|
||||
t.Fatalf("receipt budget after expiry = %d, want 0", got)
|
||||
}
|
||||
primary.release(primaryMax)
|
||||
if got := primary.snapshot(); got != 0 {
|
||||
|
|
@ -398,7 +379,7 @@ func TestCachedReplayRestoreIsSynchronousAndIndependentOfGlobalHookExecutor(t *t
|
|||
})
|
||||
|
||||
var restored atomic.Bool
|
||||
if err := s.sendCachedRPCResultWithHook(context.Background(), c, encoded, func() error {
|
||||
if err := s.sendReplayedRPCResultWithHook(context.Background(), c, encoded, func() error {
|
||||
if got := len(transport.snapshot()); got != 1 {
|
||||
return errors.New("replay restore ran before physical write")
|
||||
}
|
||||
|
|
@ -672,7 +653,7 @@ func TestWrappedConvergenceMethodDrivesEgressAndReplayPriority(t *testing.T) {
|
|||
var cached *encodedOutboundMessage
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if got, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID); ok {
|
||||
if got, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgID); ok {
|
||||
cached = got
|
||||
break
|
||||
}
|
||||
|
|
@ -725,7 +706,7 @@ func TestRPCWorkerReleasesAfterEgressAdmissionWhileWriteBlocked(t *testing.T) {
|
|||
tr.once.Do(func() { close(tr.release) })
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for {
|
||||
if _, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID); ok {
|
||||
if _, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgID); ok {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
|
|
@ -754,7 +735,7 @@ func TestDeliveryHookRunsOnceAfterReplayNotFailedWrite(t *testing.T) {
|
|||
deadline := time.Now().Add(time.Second)
|
||||
var cached *encodedOutboundMessage
|
||||
for time.Now().Before(deadline) {
|
||||
if got, ok := s.rpcResults.Get(oldConn.authKeyID, oldConn.sessionID, reqMsgID); ok {
|
||||
if got, ok := s.rpcResults.Replay(oldConn.authKeyID, oldConn.sessionID, reqMsgID); ok {
|
||||
cached = got
|
||||
break
|
||||
}
|
||||
|
|
@ -772,7 +753,7 @@ func TestDeliveryHookRunsOnceAfterReplayNotFailedWrite(t *testing.T) {
|
|||
|
||||
replayTransport := &failAfterTransport{}
|
||||
replayConn := newOutboundTestConn(t, replayTransport, newOutboundTrackedBudget(1<<20))
|
||||
if err := s.sendCachedRPCResult(context.Background(), replayConn, cached); err != nil {
|
||||
if err := s.sendReplayedRPCResult(context.Background(), replayConn, cached); err != nil {
|
||||
t.Fatalf("replay result: %v", err)
|
||||
}
|
||||
deadline = time.Now().Add(time.Second)
|
||||
|
|
@ -789,7 +770,7 @@ func TestDeliveryHookRunsOnceAfterReplayNotFailedWrite(t *testing.T) {
|
|||
cached.delivery.coordinator.hookState() != rpcResultDeliveryHookDone {
|
||||
t.Fatal("successful replay did not complete shared delivery coordinator")
|
||||
}
|
||||
if err := s.sendCachedRPCResult(context.Background(), replayConn, cached); err != nil {
|
||||
if err := s.sendReplayedRPCResult(context.Background(), replayConn, cached); err != nil {
|
||||
t.Fatalf("second replay: %v", err)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ func TestCachedRPCResultReplayUsesPreReservedBodyWithoutDoubleCharge(t *testing.
|
|||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- s.sendCachedRPCResult(context.Background(), c, encoded)
|
||||
done <- s.sendReplayedRPCResult(context.Background(), c, encoded)
|
||||
}()
|
||||
select {
|
||||
case <-tr.started:
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ type rpcResultSubscriberBudget struct {
|
|||
global rpcResultFlightLimit
|
||||
authLimit int64
|
||||
sessionLimit int64
|
||||
authShards [rpcResultBudgetShards]rpcResultSubscriberBudgetShard[[8]byte]
|
||||
sessionShards [rpcResultBudgetShards]rpcResultSubscriberBudgetShard[rpcResultSessionBudgetKey]
|
||||
authShards [rpcExecutionBudgetShards]rpcResultSubscriberBudgetShard[[8]byte]
|
||||
sessionShards [rpcExecutionBudgetShards]rpcResultSubscriberBudgetShard[rpcExecutionSessionBudgetKey]
|
||||
}
|
||||
|
||||
func newRPCResultSubscriberBudget(
|
||||
|
|
@ -43,25 +43,25 @@ func newRPCResultSubscriberBudget(
|
|||
b.global.max = int64(globalLimit)
|
||||
for i := range b.authShards {
|
||||
b.authShards[i].usage = make(map[[8]byte]int64)
|
||||
b.sessionShards[i].usage = make(map[rpcResultSessionBudgetKey]int64)
|
||||
b.sessionShards[i].usage = make(map[rpcExecutionSessionBudgetKey]int64)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *rpcResultSubscriberBudget) reserve(key rpcResultCacheKey, slots int) bool {
|
||||
func (b *rpcResultSubscriberBudget) reserve(key rpcExecutionKey, slots int) bool {
|
||||
if b == nil || slots <= 0 {
|
||||
return false
|
||||
}
|
||||
authShard := b.authShard(key.authKeyID)
|
||||
sessionKey := rpcResultSessionBudgetKey{authKeyID: key.authKeyID, sessionID: key.sessionID}
|
||||
sessionKey := rpcExecutionSessionBudgetKey{authKeyID: key.authKeyID, sessionID: key.sessionID}
|
||||
sessionShard := b.sessionShard(sessionKey)
|
||||
delta := int64(slots)
|
||||
authShard.mu.Lock()
|
||||
sessionShard.mu.Lock()
|
||||
authUsed := authShard.usage[key.authKeyID]
|
||||
sessionUsed := sessionShard.usage[sessionKey]
|
||||
if !withinRPCResultBudget(authUsed, delta, b.authLimit) ||
|
||||
!withinRPCResultBudget(sessionUsed, delta, b.sessionLimit) ||
|
||||
if !withinRPCExecutionBudget(authUsed, delta, b.authLimit) ||
|
||||
!withinRPCExecutionBudget(sessionUsed, delta, b.sessionLimit) ||
|
||||
!b.global.reserveN(delta) {
|
||||
sessionShard.mu.Unlock()
|
||||
authShard.mu.Unlock()
|
||||
|
|
@ -74,12 +74,12 @@ func (b *rpcResultSubscriberBudget) reserve(key rpcResultCacheKey, slots int) bo
|
|||
return true
|
||||
}
|
||||
|
||||
func (b *rpcResultSubscriberBudget) release(key rpcResultCacheKey, slots int) {
|
||||
func (b *rpcResultSubscriberBudget) release(key rpcExecutionKey, slots int) {
|
||||
if b == nil || slots <= 0 {
|
||||
panic("mtproto rpc result subscriber release must be positive")
|
||||
}
|
||||
authShard := b.authShard(key.authKeyID)
|
||||
sessionKey := rpcResultSessionBudgetKey{authKeyID: key.authKeyID, sessionID: key.sessionID}
|
||||
sessionKey := rpcExecutionSessionBudgetKey{authKeyID: key.authKeyID, sessionID: key.sessionID}
|
||||
sessionShard := b.sessionShard(sessionKey)
|
||||
delta := int64(slots)
|
||||
authShard.mu.Lock()
|
||||
|
|
@ -123,7 +123,7 @@ func (b *rpcResultSubscriberBudget) sessionSnapshot(authKeyID [8]byte, sessionID
|
|||
if b == nil {
|
||||
return 0
|
||||
}
|
||||
key := rpcResultSessionBudgetKey{authKeyID: authKeyID, sessionID: sessionID}
|
||||
key := rpcExecutionSessionBudgetKey{authKeyID: authKeyID, sessionID: sessionID}
|
||||
shard := b.sessionShard(key)
|
||||
shard.mu.Lock()
|
||||
used := shard.usage[key]
|
||||
|
|
@ -134,16 +134,16 @@ func (b *rpcResultSubscriberBudget) sessionSnapshot(authKeyID [8]byte, sessionID
|
|||
func (b *rpcResultSubscriberBudget) authShard(
|
||||
authKeyID [8]byte,
|
||||
) *rpcResultSubscriberBudgetShard[[8]byte] {
|
||||
index := maphash.Bytes(b.seed, authKeyID[:]) & (rpcResultBudgetShards - 1)
|
||||
index := maphash.Bytes(b.seed, authKeyID[:]) & (rpcExecutionBudgetShards - 1)
|
||||
return &b.authShards[index]
|
||||
}
|
||||
|
||||
func (b *rpcResultSubscriberBudget) sessionShard(
|
||||
key rpcResultSessionBudgetKey,
|
||||
) *rpcResultSubscriberBudgetShard[rpcResultSessionBudgetKey] {
|
||||
key rpcExecutionSessionBudgetKey,
|
||||
) *rpcResultSubscriberBudgetShard[rpcExecutionSessionBudgetKey] {
|
||||
var raw [16]byte
|
||||
copy(raw[:8], key.authKeyID[:])
|
||||
binary.LittleEndian.PutUint64(raw[8:], uint64(key.sessionID))
|
||||
index := maphash.Bytes(b.seed, raw[:]) & (rpcResultBudgetShards - 1)
|
||||
index := maphash.Bytes(b.seed, raw[:]) & (rpcExecutionBudgetShards - 1)
|
||||
return &b.sessionShards[index]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -446,7 +446,7 @@ func (a *rpcRewrapAlias) storeResultOnce(s *Server, encoded *encodedOutboundMess
|
|||
if a == nil || s == nil || encoded == nil || !a.resultStoreClaimed.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
s.storeRPCResult(a.conn, a.newReqID, encoded)
|
||||
s.completeRPCResult(a.conn, a.newReqID, encoded, true)
|
||||
}
|
||||
|
||||
func claimRPCRewrapLogicalHook(
|
||||
|
|
@ -465,8 +465,8 @@ func claimRPCRewrapLogicalHook(
|
|||
// completeDeliveredRPCRewrapResult is safe after a watchdog has already fenced
|
||||
// this physical generation. The caller has independent proof that the
|
||||
// retargeted bytes reached the stream; deliveredFinalizeOnce, the shared hook
|
||||
// coordinator and cache publication make late/concurrent invocations converge
|
||||
// while preserving replacement -> logical -> cache -> barrier order.
|
||||
// coordinator and ledger publication make late/concurrent invocations converge
|
||||
// while preserving replacement -> logical -> ledger -> barrier order.
|
||||
func (s *Server) completeDeliveredRPCRewrapResult(
|
||||
ctx context.Context,
|
||||
a *rpcRewrapAlias,
|
||||
|
|
@ -618,7 +618,7 @@ func (c *rpcRewrapDeliveryControl) timeout() bool {
|
|||
|
||||
// commit records successful physical delivery (or an already-proven retarget)
|
||||
// without disarming the watchdog. The same absolute deadline covers the
|
||||
// replacement/logical restore and cache/barrier terminal path; complete is the
|
||||
// replacement/logical restore and ledger/barrier terminal path; complete is the
|
||||
// only successful transition that stops the timer.
|
||||
func (c *rpcRewrapDeliveryControl) commit() bool {
|
||||
if c == nil || !c.transition(rpcRewrapJobRunning, rpcRewrapJobCommitted) {
|
||||
|
|
@ -840,7 +840,7 @@ func (s *Server) failRPCRewrapResultJob(
|
|||
publish := a.newOwner.HandOff()
|
||||
encoded.markReplayable()
|
||||
encoded.releaseDeferredLogicalDeliveryHook()
|
||||
// Release the connection-local scheduler before any defensive cache panic;
|
||||
// Release the connection-local scheduler before any defensive ledger panic;
|
||||
// the physical generation is already fenced, so no following task can run.
|
||||
a.releaseReplayRestoreBarrier()
|
||||
if publish {
|
||||
|
|
@ -1093,7 +1093,7 @@ func (s *Server) publishRewrappedRPCResult(
|
|||
}
|
||||
ctx, cancel := context.WithDeadline(context.Background(), deadline)
|
||||
defer cancel()
|
||||
// attachRPCRewrapReplayPreparation froze cache-copied scheduling metadata
|
||||
// attachRPCRewrapReplayPreparation froze replay-copied scheduling metadata
|
||||
// before the watchdog started. The delivery path must not mutate plain fields
|
||||
// that a concurrent timeout publication can copy into the replay ledger.
|
||||
encoded.markQueued()
|
||||
|
|
|
|||
|
|
@ -165,7 +165,7 @@ func TestRPCRewrapFailedSourceAttemptPhysicallyDeliversAliasOnce(t *testing.T) {
|
|||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
cached, ok := s.rpcResults.Get(key.ID, sessionID, newReqID)
|
||||
cached, ok := s.rpcResults.Replay(key.ID, sessionID, newReqID)
|
||||
if ok && cached.deliveryState() == rpcResultDeliveryDelivered && hooks.Load() == 1 {
|
||||
if got := cached.writtenRequestID(); got != newReqID {
|
||||
t.Fatalf("alias physical request ID = %d, want %d", got, newReqID)
|
||||
|
|
@ -173,7 +173,7 @@ func TestRPCRewrapFailedSourceAttemptPhysicallyDeliversAliasOnce(t *testing.T) {
|
|||
if got := len(aliasTransport.snapshot()); got != 1 {
|
||||
t.Fatalf("alias physical writes = %d, want 1", got)
|
||||
}
|
||||
sourceCached, sourceOK := s.rpcResults.Get(key.ID, sessionID, oldReqID)
|
||||
sourceCached, sourceOK := s.rpcResults.Replay(key.ID, sessionID, oldReqID)
|
||||
if !sourceOK || sourceCached.deliveryState() != rpcResultDeliveryReplayable {
|
||||
t.Fatalf("source physical attempt = cached:%v state:%d, want replayable", sourceOK, sourceCached.deliveryState())
|
||||
}
|
||||
|
|
@ -181,13 +181,13 @@ func TestRPCRewrapFailedSourceAttemptPhysicallyDeliversAliasOnce(t *testing.T) {
|
|||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
cached, ok := s.rpcResults.Get(key.ID, sessionID, newReqID)
|
||||
cached, ok := s.rpcResults.Replay(key.ID, sessionID, newReqID)
|
||||
t.Fatalf("alias result = cached:%v state:%v hooks:%d writes:%d", ok, cached.deliveryState(), hooks.Load(), len(aliasTransport.snapshot()))
|
||||
}
|
||||
|
||||
func TestRPCRewrapRepeatedReplacementSubscriberCapacityStaysBounded(t *testing.T) {
|
||||
s := New(Options{WriteTimeout: time.Second})
|
||||
s.rpcResults = newRPCResultSubscriberTestCache(2, 2, 2, 2)
|
||||
s.rpcResults = newRPCExecutionSubscriberTestLedger(2, 2, 2, 2)
|
||||
transport := &collectingSessionTransport{}
|
||||
key := newTestAuthKey(t)
|
||||
const sessionID, oldReqID, newReqID = int64(188), int64(15101), int64(15201)
|
||||
|
|
@ -289,7 +289,7 @@ func TestRPCRewrapRetargetFailureRequiresReplacementAliasWrite(t *testing.T) {
|
|||
deadline := time.Now().Add(2 * time.Second)
|
||||
var aliasCached *encodedOutboundMessage
|
||||
for time.Now().Before(deadline) {
|
||||
if cached, ok := s.rpcResults.Get(key.ID, sessionID, newReqID); ok && cached.deliveryState() == rpcResultDeliveryReplayable {
|
||||
if cached, ok := s.rpcResults.Replay(key.ID, sessionID, newReqID); ok && cached.deliveryState() == rpcResultDeliveryReplayable {
|
||||
aliasCached = cached
|
||||
break
|
||||
}
|
||||
|
|
@ -306,7 +306,7 @@ func TestRPCRewrapRetargetFailureRequiresReplacementAliasWrite(t *testing.T) {
|
|||
replacement := s.newConn(replacementTransport, key, sessionID, 1)
|
||||
legacyCanonicalTestConn(t, replacement)
|
||||
t.Cleanup(replacement.ForceClose)
|
||||
if err := s.sendCachedRPCResult(context.Background(), replacement, aliasCached); err != nil {
|
||||
if err := s.sendReplayedRPCResult(context.Background(), replacement, aliasCached); err != nil {
|
||||
t.Fatalf("replacement alias replay: %v", err)
|
||||
}
|
||||
deadline = time.Now().Add(time.Second)
|
||||
|
|
@ -319,7 +319,7 @@ func TestRPCRewrapRetargetFailureRequiresReplacementAliasWrite(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestRPCResultWaiterSubscribeIsEventDriven(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 8)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 8)
|
||||
claim, err := cache.Acquire([8]byte{1}, 2, 3)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("Acquire owner = %+v, %v", claim, err)
|
||||
|
|
@ -337,14 +337,14 @@ func TestRPCResultWaiterSubscribeIsEventDriven(t *testing.T) {
|
|||
t.Fatal("Subscribe waited for or fabricated a result")
|
||||
}
|
||||
encoded := &encodedOutboundMessage{typeID: mt.RPCResultTypeID, body: make([]byte, 16), reqMsgID: 3}
|
||||
cache.Put([8]byte{1}, 2, 3, encoded)
|
||||
cache.completeReplayableForTest([8]byte{1}, 2, 3, encoded)
|
||||
if !called.Load() {
|
||||
t.Fatal("completion event did not invoke subscriber")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultOwnerAbortHookInstallationIsFlightBound(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 8)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 8)
|
||||
claim, err := cache.Acquire([8]byte{2}, 3, 4)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("Acquire owner = %+v, %v", claim, err)
|
||||
|
|
@ -362,7 +362,7 @@ func TestRPCResultOwnerAbortHookInstallationIsFlightBound(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestRPCRewrapRegistryIsPlatformAgnosticAndAckBound(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 8)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 8)
|
||||
claim, err := cache.Acquire([8]byte{4}, 5, 6)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("Acquire owner = %+v, %v", claim, err)
|
||||
|
|
@ -451,7 +451,7 @@ func TestInitRewrapAfterWritingReplaysWithoutBusinessExecution(t *testing.T) {
|
|||
deadline := time.Now().Add(2 * time.Second)
|
||||
var replayed *encodedOutboundMessage
|
||||
for time.Now().Before(deadline) {
|
||||
if got, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, newReqID); ok {
|
||||
if got, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, newReqID); ok {
|
||||
replayed = got
|
||||
break
|
||||
}
|
||||
|
|
@ -543,7 +543,7 @@ func TestInitRewrapAliasesExecutionAndRetargetsQueuedResult(t *testing.T) {
|
|||
)
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if aliased, ok = s.rpcResults.Get(c.authKeyID, c.sessionID, newReqID); ok {
|
||||
if aliased, ok = s.rpcResults.Replay(c.authKeyID, c.sessionID, newReqID); ok {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
|
|
@ -609,8 +609,8 @@ func TestRPCRewrapDeliveryJobPanicAndDeadlineReleaseBarrier(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestExpiredRPCRewrapResultJobPublishesCompletedAliasExactlyOnce(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
|
||||
func TestExpiredRPCRewrapResultJobPublishesUnavailableAliasExactlyOnce(t *testing.T) {
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 1)
|
||||
s := &Server{log: zaptest.NewLogger(t), rpcResults: cache}
|
||||
c := &Conn{
|
||||
metrics: NopMetrics{},
|
||||
|
|
@ -668,27 +668,22 @@ func TestExpiredRPCRewrapResultJobPublishesCompletedAliasExactlyOnce(t *testing.
|
|||
t.Fatalf("pending result flights = %d, want 0", used)
|
||||
}
|
||||
|
||||
completed, ok := cache.Get(c.authKeyID, c.sessionID, reqMsgID)
|
||||
if !ok || completed != encoded {
|
||||
t.Fatalf("completed aliased result = (%p, %v), want (%p, true)", completed, ok, encoded)
|
||||
completed, ok := cache.Replay(c.authKeyID, c.sessionID, reqMsgID)
|
||||
if ok || completed != nil {
|
||||
t.Fatalf("failed alias retained payload = (%p, %v)", completed, ok)
|
||||
}
|
||||
replay, err := cache.Acquire(c.authKeyID, c.sessionID, reqMsgID)
|
||||
if err != nil {
|
||||
t.Fatalf("reacquire completed aliased result: %v", err)
|
||||
}
|
||||
if replay.state != rpcResultAcquireCompleted || replay.encoded != encoded ||
|
||||
!replay.executionKnown || !replay.executionOK {
|
||||
t.Fatalf("completed aliased result metadata = %#v", replay)
|
||||
if _, err := cache.Acquire(c.authKeyID, c.sessionID, reqMsgID); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("reacquire unavailable aliased result = %v, want capacity", err)
|
||||
}
|
||||
|
||||
// A defensive duplicate failure report must not republish or underflow the
|
||||
// completed flight. The first handoff/cache completion is the sole winner.
|
||||
// completed flight. The first handoff/ledger completion is the sole winner.
|
||||
s.failRPCRewrapResultJob(alias, encoded, context.DeadlineExceeded)
|
||||
if used := cache.flightLimit.snapshot(); used != 0 {
|
||||
t.Fatalf("pending result flights after duplicate failure = %d, want 0", used)
|
||||
}
|
||||
if got, ok := cache.Get(c.authKeyID, c.sessionID, reqMsgID); !ok || got != encoded {
|
||||
t.Fatalf("completed result changed after duplicate failure = (%p, %v)", got, ok)
|
||||
if got, ok := cache.Replay(c.authKeyID, c.sessionID, reqMsgID); ok || got != nil {
|
||||
t.Fatalf("unavailable result changed after duplicate failure = (%p, %v)", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -788,7 +783,7 @@ func TestRetargetedRPCRestoreIsOrderedAndIndependentOfGlobalHookExecutor(t *test
|
|||
if pending != 0 {
|
||||
t.Fatalf("retarget restore barriers = %d, want 0", pending)
|
||||
}
|
||||
if _, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, newReqID); !ok {
|
||||
if _, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, newReqID); !ok {
|
||||
t.Fatal("retargeted result was not cached under new req_msg_id")
|
||||
}
|
||||
}
|
||||
|
|
@ -980,7 +975,7 @@ func TestRPCRewrapPhysicalSuccessAfterWatchdogStillRunsLogicalRestore(t *testing
|
|||
}
|
||||
|
||||
func TestConcurrentRPCRewrapDeliveredFinalizationPublishesOnceWithMetadata(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
|
||||
cache := newRPCExecutionLedgerForTest(time.Now, 1)
|
||||
s := &Server{log: zaptest.NewLogger(t), rpcResults: cache}
|
||||
c := &Conn{
|
||||
metrics: NopMetrics{},
|
||||
|
|
@ -1006,6 +1001,7 @@ func TestConcurrentRPCRewrapDeliveredFinalizationPublishesOnceWithMetadata(t *te
|
|||
encoded := encodedRPCResultForPriorityTest(reqMsgID, 0)
|
||||
encoded.delivery = claim.owner.Delivery()
|
||||
encoded.setDeliveryHook(func() { logical.Add(1) })
|
||||
cache.replayStore.(*rpcReplayStoreForTest).put(c.authKeyID, c.sessionID, reqMsgID, encoded)
|
||||
alias := &rpcRewrapAlias{
|
||||
conn: c, newReqID: reqMsgID, method: "help.getConfig", newOwner: claim.owner,
|
||||
afterSuccessfulDelivery: func() error {
|
||||
|
|
@ -1036,7 +1032,7 @@ func TestConcurrentRPCRewrapDeliveredFinalizationPublishesOnceWithMetadata(t *te
|
|||
t.Fatalf("logical finalizations = %d, want 1", got)
|
||||
}
|
||||
if got := subscribers.Load(); got != 1 {
|
||||
t.Fatalf("cache subscriber calls = %d, want 1", got)
|
||||
t.Fatalf("ledger subscriber calls = %d, want 1", got)
|
||||
}
|
||||
replay, err := cache.Acquire(c.authKeyID, c.sessionID, reqMsgID)
|
||||
if err != nil || replay.state != rpcResultAcquireCompleted ||
|
||||
|
|
@ -1109,7 +1105,7 @@ func TestRPCRewrapSubscriberPanicCannotLoseClaimedLogicalHook(t *testing.T) {
|
|||
if pending != 0 {
|
||||
t.Fatalf("restore barriers after subscriber panic = %d, want 0", pending)
|
||||
}
|
||||
if cached, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID); !ok ||
|
||||
if cached, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgID); !ok ||
|
||||
cached.delivery != encoded.delivery || !bytes.Equal(cached.body, encoded.body) || cached.replayMsgID == 0 {
|
||||
t.Fatalf("logical outbox result after subscriber panic = (%p, %v), source=%p", cached, ok, encoded)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,10 +31,11 @@ type RuntimeSnapshot struct {
|
|||
OutboundControlMaxBytes int64
|
||||
OutboundWriteBytes int64
|
||||
OutboundWriteMaxBytes int64
|
||||
RPCResultOwners int64
|
||||
RPCResultReceipts int64
|
||||
RPCResultReceiptBytes int64
|
||||
RPCResultSubscribers int64
|
||||
RPCExecutionOwners int64
|
||||
RPCExecutionReservedEntries int64
|
||||
RPCExecutionReceipts int64
|
||||
RPCExecutionReceiptBudgetBytes int64
|
||||
RPCExecutionSubscribers int64
|
||||
}
|
||||
|
||||
type sessionManagerRuntimeSnapshot struct {
|
||||
|
|
@ -175,11 +176,12 @@ func (s *Server) RuntimeSnapshot() RuntimeSnapshot {
|
|||
result.OutboundWriteMaxBytes = s.outboundScratchPool.budget.maxBytes
|
||||
}
|
||||
if s.rpcResults != nil {
|
||||
result.RPCResultOwners = s.rpcResults.flightLimit.snapshot()
|
||||
result.RPCResultReceipts = s.rpcResults.completedEntries.snapshot()
|
||||
result.RPCResultReceiptBytes = s.rpcResults.completedBytes.snapshot()
|
||||
result.RPCExecutionOwners = s.rpcResults.flightLimit.snapshot()
|
||||
result.RPCExecutionReservedEntries = s.rpcResults.reservedEntries.snapshot()
|
||||
result.RPCExecutionReceipts = s.rpcResults.receiptCount.Load()
|
||||
result.RPCExecutionReceiptBudgetBytes = s.rpcResults.receiptBudgetBytes()
|
||||
if s.rpcResults.subscriberBudget != nil {
|
||||
result.RPCResultSubscribers = s.rpcResults.subscriberBudget.global.snapshot()
|
||||
result.RPCExecutionSubscribers = s.rpcResults.subscriberBudget.global.snapshot()
|
||||
}
|
||||
}
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
package mtprotoedge
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRuntimeSnapshotIsNilSafeAndReportsConfiguredLimits(t *testing.T) {
|
||||
if got := (*Server)(nil).RuntimeSnapshot(); got != (RuntimeSnapshot{}) {
|
||||
|
|
@ -25,3 +28,30 @@ func TestRuntimeSnapshotIsNilSafeAndReportsConfiguredLimits(t *testing.T) {
|
|||
t.Fatalf("fresh server reported live ownership: %#v", snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeSnapshotSeparatesExecutionOwnersReceiptsAndBudget(t *testing.T) {
|
||||
server := New(Options{})
|
||||
server.rpcResults = newRPCExecutionLedgerForTest(time.Now, 4)
|
||||
auth := [8]byte{1, 2, 3, 4}
|
||||
claim, err := server.rpcResults.Acquire(auth, 5, 6)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("owner = %#v, %v", claim, err)
|
||||
}
|
||||
pending := server.RuntimeSnapshot()
|
||||
if pending.RPCExecutionOwners != 1 || pending.RPCExecutionReservedEntries != 1 ||
|
||||
pending.RPCExecutionReceipts != 0 || pending.RPCExecutionReceiptBudgetBytes != 0 {
|
||||
t.Fatalf("pending execution snapshot = %#v", pending)
|
||||
}
|
||||
server.rpcResults.completeReplayableForTest(auth, 5, 6, &encodedOutboundMessage{body: make([]byte, 4<<20)})
|
||||
completed := server.RuntimeSnapshot()
|
||||
if completed.RPCExecutionOwners != 0 || completed.RPCExecutionReservedEntries != 1 ||
|
||||
completed.RPCExecutionReceipts != 1 || completed.RPCExecutionReceiptBudgetBytes != rpcExecutionReceiptBudgetBytes {
|
||||
t.Fatalf("completed execution snapshot = %#v", completed)
|
||||
}
|
||||
server.rpcResults.Acknowledge(auth, 5, 6)
|
||||
released := server.RuntimeSnapshot()
|
||||
if released.RPCExecutionReservedEntries != 0 || released.RPCExecutionReceipts != 0 ||
|
||||
released.RPCExecutionReceiptBudgetBytes != 0 {
|
||||
t.Fatalf("released execution snapshot = %#v", released)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ type legacyRPCHandlerWithMethod interface {
|
|||
// LayerRPCHandler is the production API-RPC boundary. Admission is a separate
|
||||
// allocation-bounded phase so the edge can freeze the connection profile,
|
||||
// validate wrapper dependencies and establish exact request identity before
|
||||
// flight/cache/scheduler ownership is acquired.
|
||||
// execution-ledger/scheduler ownership is acquired.
|
||||
type LayerRPCHandler interface {
|
||||
AdmitLayer(profile tlprofile.Profile, b *bin.Buffer, limits tlprofile.Limits) (tlprofile.Admission, error)
|
||||
AdmitUnprofiled(b *bin.Buffer, limits tlprofile.Limits) (tlprofile.Admission, error)
|
||||
|
|
@ -103,7 +103,7 @@ type LayerRPCSessionProfileResolver interface {
|
|||
|
||||
// LayerRPCOrderedSessionProfileResolver restores both the selected Layer and
|
||||
// the newest invokeWithLayer client msg_id which proved it. The cursor prevents
|
||||
// an old cached request replay on a replacement physical connection from
|
||||
// an old retained request replay on a replacement physical connection from
|
||||
// rolling the logical session back to an older profile.
|
||||
type LayerRPCOrderedSessionProfileResolver interface {
|
||||
NegotiatedSessionLayerEvidence(authKeyID [8]byte, sessionID int64) (layer int, msgID int64, ok bool)
|
||||
|
|
@ -175,7 +175,7 @@ type LayerRPCDurableSessionProfileDeleter interface {
|
|||
// LayerRPCReplayPreparer reapplies connection-local wrapper state for an
|
||||
// already-executed exact request without consuming its one-shot business
|
||||
// dispatch lease. The returned callback is safe to run only after a successful
|
||||
// cached rpc_result reaches the replacement physical connection.
|
||||
// replayed rpc_result reaches the replacement physical connection.
|
||||
type LayerRPCReplayPreparer interface {
|
||||
PrepareAdmittedReplay(
|
||||
ctx context.Context,
|
||||
|
|
@ -201,7 +201,7 @@ type LayerRPCProfileEvidenceContext interface {
|
|||
// LayerRPCAdmissionProfilePublisher advances the auth-key-wide inherited
|
||||
// default for fresh explicit evidence. admissionSeq is allocated once by the
|
||||
// edge's exact flight owner and globally orders different MTProto sessions;
|
||||
// cached joins/replays never call this hook again.
|
||||
// joined/replayed requests never call this hook again.
|
||||
type LayerRPCAdmissionProfilePublisher interface {
|
||||
PublishAdmittedLayerProfileEvidence(
|
||||
ctx context.Context,
|
||||
|
|
@ -284,16 +284,16 @@ type Options struct {
|
|||
// 等于 copied body;exact charge 是 typed decode 前的保守 materialization
|
||||
// 上界,因此该配置不表示可并发接收 512 MiB wire body。默认 512 MiB。
|
||||
RPCGlobalMaxBytes int64
|
||||
// RPCResultCache*Entries bound in-flight owners and compact completed
|
||||
// receipts. Exact payload bytes are not charged here: the logical-session
|
||||
// RPCExecution*Entries bound in-flight owners and compact completed
|
||||
// receipts. Payload bytes are not charged here: the logical-session
|
||||
// outbox owns them under OutboundTrackedGlobalMaxBytes until ACK. ACK removes
|
||||
// the receipt immediately; 331 seconds is only the no-ACK safety horizon.
|
||||
RPCResultCacheMaxEntries int
|
||||
RPCResultCacheAuthMaxEntries int
|
||||
RPCResultCacheSessionMaxEntries int
|
||||
// RPCResultPendingPerAuth is an additional active-owner bound, independent
|
||||
RPCExecutionMaxEntries int
|
||||
RPCExecutionAuthMaxEntries int
|
||||
RPCExecutionSessionMaxEntries int
|
||||
// RPCExecutionPendingPerAuth is an additional active-owner bound, independent
|
||||
// from the retained entry limits and RPCGlobalMaxTasks. Default 2048.
|
||||
RPCResultPendingPerAuth int
|
||||
RPCExecutionPendingPerAuth int
|
||||
// InboundFrameGlobalMaxBytes 是所有物理连接当前正在处理的 transport wire buffer
|
||||
// 与最大解密 plaintext buffer 的总预算。长度前缀读取后、payload 分配前预留,默认
|
||||
// 512 MiB;非正值使用默认值。
|
||||
|
|
@ -332,7 +332,7 @@ type Options struct {
|
|||
// generated Layer admission by configuring the canonical-only route.
|
||||
legacyRPC legacyRPCHandler
|
||||
// LayerRPC is the generated exact-profile production path. When configured,
|
||||
// every API request must complete admission before flight/cache scheduling.
|
||||
// every API request must complete admission before execution-ledger scheduling.
|
||||
LayerRPC LayerRPCHandler
|
||||
// Metrics 接收连接层指标。默认 NopMetrics。
|
||||
Metrics Metrics
|
||||
|
|
@ -389,19 +389,19 @@ func (o *Options) setDefaults() {
|
|||
if o.RPCGlobalMaxBytes <= 0 {
|
||||
o.RPCGlobalMaxBytes = 512 << 20
|
||||
}
|
||||
if o.RPCResultCacheMaxEntries == 0 {
|
||||
o.RPCResultCacheMaxEntries = rpcResultCacheMaxEntries
|
||||
if o.RPCExecutionMaxEntries == 0 {
|
||||
o.RPCExecutionMaxEntries = rpcExecutionMaxEntries
|
||||
}
|
||||
if o.RPCResultCacheAuthMaxEntries == 0 {
|
||||
o.RPCResultCacheAuthMaxEntries = rpcResultCacheAuthMaxEntries
|
||||
if o.RPCExecutionAuthMaxEntries == 0 {
|
||||
o.RPCExecutionAuthMaxEntries = rpcExecutionAuthMaxEntries
|
||||
}
|
||||
if o.RPCResultCacheSessionMaxEntries == 0 {
|
||||
o.RPCResultCacheSessionMaxEntries = rpcResultCacheSessionMaxEntries
|
||||
if o.RPCExecutionSessionMaxEntries == 0 {
|
||||
o.RPCExecutionSessionMaxEntries = rpcExecutionSessionMaxEntries
|
||||
}
|
||||
if o.RPCResultPendingPerAuth == 0 {
|
||||
o.RPCResultPendingPerAuth = rpcResultFlightMaxPendingPerAuth
|
||||
if o.RPCResultPendingPerAuth > o.RPCGlobalMaxTasks {
|
||||
o.RPCResultPendingPerAuth = o.RPCGlobalMaxTasks
|
||||
if o.RPCExecutionPendingPerAuth == 0 {
|
||||
o.RPCExecutionPendingPerAuth = rpcExecutionPendingPerAuth
|
||||
if o.RPCExecutionPendingPerAuth > o.RPCGlobalMaxTasks {
|
||||
o.RPCExecutionPendingPerAuth = o.RPCGlobalMaxTasks
|
||||
}
|
||||
}
|
||||
if o.InboundFrameGlobalMaxBytes <= 0 {
|
||||
|
|
@ -436,19 +436,19 @@ func (o *Options) setDefaults() {
|
|||
}
|
||||
}
|
||||
|
||||
func validateRPCResultCacheOptions(o Options) error {
|
||||
if o.RPCResultCacheMaxEntries <= 0 || o.RPCResultCacheAuthMaxEntries <= 0 || o.RPCResultCacheSessionMaxEntries <= 0 {
|
||||
return fmt.Errorf("rpc_result cache entry limits must be positive")
|
||||
func validateRPCExecutionOptions(o Options) error {
|
||||
if o.RPCExecutionMaxEntries <= 0 || o.RPCExecutionAuthMaxEntries <= 0 || o.RPCExecutionSessionMaxEntries <= 0 {
|
||||
return fmt.Errorf("rpc execution ledger entry limits must be positive")
|
||||
}
|
||||
if o.RPCResultCacheMaxEntries < o.RPCResultCacheAuthMaxEntries ||
|
||||
o.RPCResultCacheAuthMaxEntries < o.RPCResultCacheSessionMaxEntries {
|
||||
return fmt.Errorf("rpc_result cache entry hierarchy must satisfy global >= auth >= session: %d/%d/%d",
|
||||
o.RPCResultCacheMaxEntries, o.RPCResultCacheAuthMaxEntries, o.RPCResultCacheSessionMaxEntries)
|
||||
if o.RPCExecutionMaxEntries < o.RPCExecutionAuthMaxEntries ||
|
||||
o.RPCExecutionAuthMaxEntries < o.RPCExecutionSessionMaxEntries {
|
||||
return fmt.Errorf("rpc execution ledger entry hierarchy must satisfy global >= auth >= session: %d/%d/%d",
|
||||
o.RPCExecutionMaxEntries, o.RPCExecutionAuthMaxEntries, o.RPCExecutionSessionMaxEntries)
|
||||
}
|
||||
if o.RPCResultPendingPerAuth <= 0 || o.RPCResultPendingPerAuth > o.RPCGlobalMaxTasks ||
|
||||
o.RPCResultPendingPerAuth > o.RPCResultCacheAuthMaxEntries {
|
||||
return fmt.Errorf("rpc_result per-auth pending limit %d must be positive and <= global pending %d and auth entries %d",
|
||||
o.RPCResultPendingPerAuth, o.RPCGlobalMaxTasks, o.RPCResultCacheAuthMaxEntries)
|
||||
if o.RPCExecutionPendingPerAuth <= 0 || o.RPCExecutionPendingPerAuth > o.RPCGlobalMaxTasks ||
|
||||
o.RPCExecutionPendingPerAuth > o.RPCExecutionAuthMaxEntries {
|
||||
return fmt.Errorf("rpc execution per-auth pending limit %d must be positive and <= global pending %d and auth entries %d",
|
||||
o.RPCExecutionPendingPerAuth, o.RPCGlobalMaxTasks, o.RPCExecutionAuthMaxEntries)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -494,7 +494,7 @@ type Server struct {
|
|||
types *tmap.Map
|
||||
admission *admissionController
|
||||
|
||||
rpcResults *rpcResultCache
|
||||
rpcResults *rpcExecutionLedger
|
||||
rpcRewrap *rpcRewrapRegistry
|
||||
|
||||
// onFrame 是测试钩子:收到一帧时回调其字节数;生产为 nil。
|
||||
|
|
@ -504,8 +504,8 @@ type Server struct {
|
|||
// New 创建 Server。
|
||||
func New(opts Options) *Server {
|
||||
opts.setDefaults()
|
||||
if err := validateRPCResultCacheOptions(opts); err != nil {
|
||||
panic(fmt.Sprintf("mtprotoedge: invalid result-cache options: %v", err))
|
||||
if err := validateRPCExecutionOptions(opts); err != nil {
|
||||
panic(fmt.Sprintf("mtprotoedge: invalid rpc execution options: %v", err))
|
||||
}
|
||||
conns := opts.ActiveSessions
|
||||
if conns == nil {
|
||||
|
|
@ -544,22 +544,19 @@ func New(opts Options) *Server {
|
|||
clock: opts.Clock,
|
||||
rand: opts.Rand,
|
||||
types: tmap.New(tg.TypesMap(), mt.TypesMap(), proto.TypesMap()),
|
||||
rpcResults: newRPCResultCacheWithFairCapacity(opts.Clock.Now, rpcResultCacheCapacity{
|
||||
rpcResults: newRPCExecutionLedger(opts.Clock.Now, rpcExecutionLedgerCapacity{
|
||||
maxPending: opts.RPCGlobalMaxTasks,
|
||||
maxPendingPerAuth: opts.RPCResultPendingPerAuth,
|
||||
globalMaxBytes: int64(opts.RPCResultCacheMaxEntries),
|
||||
globalMaxEntries: opts.RPCResultCacheMaxEntries,
|
||||
authMaxBytes: int64(opts.RPCResultCacheAuthMaxEntries),
|
||||
authMaxEntries: opts.RPCResultCacheAuthMaxEntries,
|
||||
sessionMaxBytes: int64(opts.RPCResultCacheSessionMaxEntries),
|
||||
sessionMaxEntries: opts.RPCResultCacheSessionMaxEntries,
|
||||
sessions: conns,
|
||||
maxPendingPerAuth: opts.RPCExecutionPendingPerAuth,
|
||||
globalMaxEntries: opts.RPCExecutionMaxEntries,
|
||||
authMaxEntries: opts.RPCExecutionAuthMaxEntries,
|
||||
sessionMaxEntries: opts.RPCExecutionSessionMaxEntries,
|
||||
replayStore: conns,
|
||||
}),
|
||||
rpcRewrap: newRPCRewrapRegistry(opts.RPCGlobalMaxTasks),
|
||||
admission: newAdmissionController(opts.MaxConnections, opts.MaxConnectionsPerIP, opts.MaxConcurrentHandshakes),
|
||||
}
|
||||
conns.setLogicalSessionReleaseHook(func(key sessionKey) {
|
||||
server.rpcResults.forgetCompletedSession(key.authKeyID, key.sessionID)
|
||||
server.rpcResults.forgetSession(key.authKeyID, key.sessionID)
|
||||
})
|
||||
return server
|
||||
}
|
||||
|
|
|
|||
|
|
@ -457,7 +457,7 @@ func (m *SessionManager) ApplyOrderedRawLayerForSession(
|
|||
// ExplicitLayerEvidenceForAuthKey exposes live exact-session truth to
|
||||
// auth.bindTempAuthKey. Router's bounded exact registry may expire while a Conn
|
||||
// remains active; bind must not replace that explicit profile with a permanent
|
||||
// key's inherited default merely because the cache TTL elapsed.
|
||||
// key's inherited default merely because the execution-receipt TTL elapsed.
|
||||
func (m *SessionManager) ExplicitLayerEvidenceForAuthKey(rawAuthKeyID [8]byte, sessionID int64) (layer int, msgID int64, ok bool) {
|
||||
if m == nil || rawAuthKeyID == ([8]byte{}) || sessionID == 0 {
|
||||
return 0, 0, false
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue