astra/owpengram-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

refactor(mtproto): sync replace result cache with execution ledger

Browse source
This commit is contained in:
iamxvbaba 2026-08-02 12:02:08 +08:00
parent 141f2f20c4
commit c1597696af
39 changed files with 1621 additions and 2320 deletions
Download patch file Download diff file Expand all files Collapse all files

8
.env.example
View file

@ -33,10 +33,10 @@ TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS=8192
TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES=536870912
# Metadata-only rpc_result receipt budgets: global >= auth >= session. ACK deletes immediately;
# 331s is only the no-ACK horizon. Payloads live solely in the logical-session outbound budget.
TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES=262144
TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES=32768
TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES=16384
TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH=2048
TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES=262144
TELESRV_MTPROTO_RPC_EXECUTION_AUTH_MAX_ENTRIES=32768
TELESRV_MTPROTO_RPC_EXECUTION_SESSION_MAX_ENTRIES=16384
TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH=2048
# Process-wide in-flight transport wire + decrypted plaintext reservation.
TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES=536870912
# Per-connection outbound mailboxes (normal/control) and process-wide resend pending bodies.

67
cmd/telesrv/main.go
View file

@ -309,10 +309,11 @@ func mtprotoRuntimeGaugeSamples(snapshot mtprotoedge.RuntimeSnapshot) []obsmetri
{Name: "telesrv_mtproto_outbound_tracked_byte_limit", Labels: []obsmetrics.Label{{Name: "kind", Value: "control"}}, Value: float64(snapshot.OutboundControlMaxBytes)},
{Name: "telesrv_mtproto_outbound_write_bytes", Value: float64(snapshot.OutboundWriteBytes)},
{Name: "telesrv_mtproto_outbound_write_byte_limit", Value: float64(snapshot.OutboundWriteMaxBytes)},
{Name: "telesrv_mtproto_rpc_result_owners", Value: float64(snapshot.RPCResultOwners)},
{Name: "telesrv_mtproto_rpc_result_receipts", Value: float64(snapshot.RPCResultReceipts)},
{Name: "telesrv_mtproto_rpc_result_receipt_bytes", Value: float64(snapshot.RPCResultReceiptBytes)},
{Name: "telesrv_mtproto_rpc_result_subscribers", Value: float64(snapshot.RPCResultSubscribers)},
{Name: "telesrv_mtproto_rpc_execution_owners", Value: float64(snapshot.RPCExecutionOwners)},
{Name: "telesrv_mtproto_rpc_execution_reserved_entries", Value: float64(snapshot.RPCExecutionReservedEntries)},
{Name: "telesrv_mtproto_rpc_execution_receipts", Value: float64(snapshot.RPCExecutionReceipts)},
{Name: "telesrv_mtproto_rpc_execution_receipt_budget_bytes", Value: float64(snapshot.RPCExecutionReceiptBudgetBytes)},
{Name: "telesrv_mtproto_rpc_execution_subscribers", Value: float64(snapshot.RPCExecutionSubscribers)},
}
}
@ -1480,35 +1481,35 @@ func run(logger *zap.Logger) error {
}
srv := mtprotoedge.New(mtprotoedge.Options{
Logger: logger.Named("mtprotoedge"),
DC: cfg.DC,
StrictDC: cfg.StrictDCCheck,
RSAKey: rsaKey,
LayerRPC: router,
AuthKeys: authKeyStore,
ActiveSessions: activeSessions,
Metrics: metricRegistry,
ObfuscatedTCP: true,
WebSocket: cfg.WebSocketEnable,
WebSocketAllowedOrigins: cfg.WebSocketAllowedOrigins,
MaxConnections: cfg.MTProtoMaxConnections,
MaxConnectionsPerIP: cfg.MTProtoMaxConnectionsPerIP,
MaxConcurrentHandshakes: cfg.MTProtoMaxConcurrentHandshakes,
RPCMaxInflight: cfg.MTProtoRPCMaxInflight,
RPCQueueSize: cfg.MTProtoRPCQueueSize,
RPCTimeout: cfg.MTProtoRPCTimeout,
RPCGlobalWorkers: cfg.MTProtoRPCGlobalWorkers,
RPCGlobalMaxTasks: cfg.MTProtoRPCGlobalMaxTasks,
RPCGlobalMaxBytes: cfg.MTProtoRPCGlobalMaxBytes,
RPCResultCacheMaxEntries: cfg.MTProtoRPCResultCacheMaxEntries,
RPCResultCacheAuthMaxEntries: cfg.MTProtoRPCResultCacheAuthMaxEntries,
RPCResultCacheSessionMaxEntries: cfg.MTProtoRPCResultCacheSessionMaxEntries,
RPCResultPendingPerAuth: cfg.MTProtoRPCResultPendingPerAuth,
InboundFrameGlobalMaxBytes: cfg.MTProtoInboundFrameGlobalMaxBytes,
OutboundQueueSize: cfg.MTProtoOutboundQueueSize,
OutboundControlQueueSize: cfg.MTProtoOutboundControlQueueSize,
OutboundTrackedGlobalMaxBytes: cfg.MTProtoOutboundTrackedGlobalMaxBytes,
OutboundWriteGlobalMaxBytes: cfg.MTProtoOutboundWriteGlobalMaxBytes,
Logger: logger.Named("mtprotoedge"),
DC: cfg.DC,
StrictDC: cfg.StrictDCCheck,
RSAKey: rsaKey,
LayerRPC: router,
AuthKeys: authKeyStore,
ActiveSessions: activeSessions,
Metrics: metricRegistry,
ObfuscatedTCP: true,
WebSocket: cfg.WebSocketEnable,
WebSocketAllowedOrigins: cfg.WebSocketAllowedOrigins,
MaxConnections: cfg.MTProtoMaxConnections,
MaxConnectionsPerIP: cfg.MTProtoMaxConnectionsPerIP,
MaxConcurrentHandshakes: cfg.MTProtoMaxConcurrentHandshakes,
RPCMaxInflight: cfg.MTProtoRPCMaxInflight,
RPCQueueSize: cfg.MTProtoRPCQueueSize,
RPCTimeout: cfg.MTProtoRPCTimeout,
RPCGlobalWorkers: cfg.MTProtoRPCGlobalWorkers,
RPCGlobalMaxTasks: cfg.MTProtoRPCGlobalMaxTasks,
RPCGlobalMaxBytes: cfg.MTProtoRPCGlobalMaxBytes,
RPCExecutionMaxEntries: cfg.MTProtoRPCExecutionMaxEntries,
RPCExecutionAuthMaxEntries: cfg.MTProtoRPCExecutionAuthMaxEntries,
RPCExecutionSessionMaxEntries: cfg.MTProtoRPCExecutionSessionMaxEntries,
RPCExecutionPendingPerAuth: cfg.MTProtoRPCExecutionPendingPerAuth,
InboundFrameGlobalMaxBytes: cfg.MTProtoInboundFrameGlobalMaxBytes,
OutboundQueueSize: cfg.MTProtoOutboundQueueSize,
OutboundControlQueueSize: cfg.MTProtoOutboundControlQueueSize,
OutboundTrackedGlobalMaxBytes: cfg.MTProtoOutboundTrackedGlobalMaxBytes,
OutboundWriteGlobalMaxBytes: cfg.MTProtoOutboundWriteGlobalMaxBytes,
OnServing: func(_ net.Addr) {
logger.Info("telesrv 服务就绪",
zap.String("listen", cfg.ListenAddr),

8
docs/configuration.en.md
View file

@ -35,10 +35,10 @@ This document describes every setting loaded by `internal/config`. Defaults and
| `TELESRV_MTPROTO_RPC_GLOBAL_WORKERS` | int / `256` | Shared fair-scheduler worker count. |
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS` | int / `8192` | Process-wide scheduled/in-flight RPC task cap. |
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES` | int64 charge bytes / `536870912` | Process-wide reserved/queued/in-flight RPC memory charge. Exact admission reserves a conservative typed-materialization charge from wire size and grows it atomically before a nested-gzip-expanded graph is decoded; grow failure rejects the complete candidate batch. This is not an equal amount of concurrently receivable wire bytes. |
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES` | int / `262144` | Global cap for pending owners and compact unacknowledged-result receipts. Receipts retain request identity, execution outcome, and Layer admission metadata only—never TL bodies. `msgs_ack` removes them immediately; 331 seconds is only the no-ACK safety horizon. |
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES` | int / `32768` | Per-raw-auth owner/receipt cap; limits satisfy `global >= auth >= session`. |
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES` | int / `16384` | Per `raw auth key + session_id` owner/receipt cap. |
| `TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH` | int / `2048` | Additional active-owner cap per raw auth key; no greater than global pending tasks or auth entries. |
| `TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES` | int / `262144` | Global cap for pending owners and compact unacknowledged execution receipts. Receipts retain request identity, execution outcome, and Layer admission metadata only—never TL bodies. `msgs_ack` removes them immediately; 331 seconds is only the no-ACK safety horizon. |
| `TELESRV_MTPROTO_RPC_EXECUTION_AUTH_MAX_ENTRIES` | int / `32768` | Per-raw-auth owner/receipt cap; limits satisfy `global >= auth >= session`. |
| `TELESRV_MTPROTO_RPC_EXECUTION_SESSION_MAX_ENTRIES` | int / `16384` | Per `raw auth key + session_id` owner/receipt cap. |
| `TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH` | int / `2048` | Additional active-owner cap per raw auth key; no greater than global pending tasks or auth entries. |
| `TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | Process-wide reservation for transport wire bytes, maximum decrypted plaintext, and every live outer/nested gzip expansion, acquired before the corresponding payload allocation. |
| `TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE` | int / `128` | Per-connection normal outbound mailbox capacity. |
| `TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE` | int / `32` | Per-connection control-message mailbox capacity. |

8
docs/configuration.zh-CN.md
View file

@ -35,10 +35,10 @@
| `TELESRV_MTPROTO_RPC_GLOBAL_WORKERS` | int / `256` | 共享公平调度器 worker 数。 |
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS` | int / `8192` | 进程级排队与执行中的 RPC task 上限。 |
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES` | int64 charge bytes / `536870912` | 进程级已预留/排队/执行中 RPC 内存 charge 预算；legacy 等于 copied body，exact 是 typed decode 前按 wire 与生成对象放大计算的保守 materialization charge。nested gzip 展开后会在 decoder 分配 typed graph 前原子增长该 charge，grow 失败原子拒绝整批候选 RPC；该值不代表可并发接收同等大小的 wire body。 |
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES` | int / `262144` | pending owner 与未 ACK 的轻量结果收据全局上限。收据只存请求身份、执行结果和 Layer admission 元数据，不存 TL body；收到 `msgs_ack` 立即删除，331 秒仅是无 ACK 时的安全上限。 |
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES` | int / `32768` | 单 raw auth key 的 owner/收据条目上限；必须满足 `global >= auth >= session`。 |
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES` | int / `16384` | 单 `raw auth key + session_id` 的 owner/收据条目上限。 |
| `TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH` | int / `2048` | 单 raw auth key 的 active pending owner 附加上限；必须不大于 `RPC_GLOBAL_MAX_TASKS` 和 auth entry 上限。 |
| `TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES` | int / `262144` | pending owner 与未 ACK execution receipt 的全局上限。receipt 只存请求身份、执行结果和 Layer admission 元数据，不存 TL body；收到 `msgs_ack` 立即删除，331 秒仅是无 ACK 时的安全上限。 |
| `TELESRV_MTPROTO_RPC_EXECUTION_AUTH_MAX_ENTRIES` | int / `32768` | 单 raw auth key 的 owner/receipt 条目上限；必须满足 `global >= auth >= session`。 |
| `TELESRV_MTPROTO_RPC_EXECUTION_SESSION_MAX_ENTRIES` | int / `16384` | 单 `raw auth key + session_id` 的 owner/receipt 条目上限。 |
| `TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH` | int / `2048` | 单 raw auth key 的 active pending owner 附加上限；必须不大于 `RPC_GLOBAL_MAX_TASKS` 和 auth entry 上限。 |
| `TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | transport wire、最大解密明文以及每个 live outer/nested gzip 输出的进程级在途预算，均在对应 payload 分配前预留。 |
| `TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE` | int / `128` | 单连接普通 outbound mailbox 容量。 |
| `TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE` | int / `32` | 单连接控制消息 mailbox 容量。 |

82
internal/config/config.go
View file

@ -63,10 +63,10 @@ type Config struct {
// Pending ownership and compact completed receipts share three-level
// global/raw-auth/session entry accounting. Result bodies are never cached
// here; the logical-session outbox owns unacknowledged wire bytes.
MTProtoRPCResultCacheMaxEntries int
MTProtoRPCResultCacheAuthMaxEntries int
MTProtoRPCResultCacheSessionMaxEntries int
MTProtoRPCResultPendingPerAuth int
MTProtoRPCExecutionMaxEntries int
MTProtoRPCExecutionAuthMaxEntries int
MTProtoRPCExecutionSessionMaxEntries int
MTProtoRPCExecutionPendingPerAuth int
// MTProtoInboundFrameGlobalMaxBytes 是 transport wire + 最大解密 plaintext 的
// 进程级在途预算；frame 长度读出后、payload 分配前预留。
MTProtoInboundFrameGlobalMaxBytes int64
@ -618,26 +618,26 @@ func Load() (Config, error) {
}),
// help.getConfig 必须下发至少一个可重连的主 DC 地址；远端部署不能
// 沿用 loopback 默认值，需显式设置客户端实际可达的 IP。
AdvertiseIP: advertiseIP,
RSAKeyPath: envOr("TELESRV_RSA_KEY", "data/server_rsa.pem"),
DC: envIntOr("TELESRV_DC", 2),
DefaultCountryCode: countryCode,
StrictDCCheck: envBoolOr("TELESRV_STRICT_DC_CHECK", false),
MTProtoMaxConnections: envIntOr("TELESRV_MTPROTO_MAX_CONNECTIONS", 200000),
MTProtoMaxConnectionsPerIP: envIntOr("TELESRV_MTPROTO_MAX_CONNECTIONS_PER_IP", 4096),
MTProtoMaxConcurrentHandshakes: envIntOr("TELESRV_MTPROTO_MAX_CONCURRENT_HANDSHAKES", 256),
MTProtoRPCMaxInflight: envIntOr("TELESRV_MTPROTO_RPC_MAX_INFLIGHT", 32),
MTProtoRPCQueueSize: envIntOr("TELESRV_MTPROTO_RPC_QUEUE_SIZE", 64),
MTProtoRPCTimeout: envDurationOr("TELESRV_MTPROTO_RPC_TIMEOUT", 30*time.Second),
MTProtoRPCGlobalWorkers: envIntOr("TELESRV_MTPROTO_RPC_GLOBAL_WORKERS", 256),
MTProtoRPCGlobalMaxTasks: envIntOr("TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS", 8192),
MTProtoRPCGlobalMaxBytes: envInt64Or("TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES", 512<<20),
MTProtoRPCResultCacheMaxEntries: envIntOr("TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", 1<<18),
MTProtoRPCResultCacheAuthMaxEntries: envIntOr("TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES", 1<<15),
MTProtoRPCResultCacheSessionMaxEntries: envIntOr(
"TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES", 1<<14,
AdvertiseIP: advertiseIP,
RSAKeyPath: envOr("TELESRV_RSA_KEY", "data/server_rsa.pem"),
DC: envIntOr("TELESRV_DC", 2),
DefaultCountryCode: countryCode,
StrictDCCheck: envBoolOr("TELESRV_STRICT_DC_CHECK", false),
MTProtoMaxConnections: envIntOr("TELESRV_MTPROTO_MAX_CONNECTIONS", 200000),
MTProtoMaxConnectionsPerIP: envIntOr("TELESRV_MTPROTO_MAX_CONNECTIONS_PER_IP", 4096),
MTProtoMaxConcurrentHandshakes: envIntOr("TELESRV_MTPROTO_MAX_CONCURRENT_HANDSHAKES", 256),
MTProtoRPCMaxInflight: envIntOr("TELESRV_MTPROTO_RPC_MAX_INFLIGHT", 32),
MTProtoRPCQueueSize: envIntOr("TELESRV_MTPROTO_RPC_QUEUE_SIZE", 64),
MTProtoRPCTimeout: envDurationOr("TELESRV_MTPROTO_RPC_TIMEOUT", 30*time.Second),
MTProtoRPCGlobalWorkers: envIntOr("TELESRV_MTPROTO_RPC_GLOBAL_WORKERS", 256),
MTProtoRPCGlobalMaxTasks: envIntOr("TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS", 8192),
MTProtoRPCGlobalMaxBytes: envInt64Or("TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES", 512<<20),
MTProtoRPCExecutionMaxEntries: envIntOr("TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES", 1<<18),
MTProtoRPCExecutionAuthMaxEntries: envIntOr("TELESRV_MTPROTO_RPC_EXECUTION_AUTH_MAX_ENTRIES", 1<<15),
MTProtoRPCExecutionSessionMaxEntries: envIntOr(
"TELESRV_MTPROTO_RPC_EXECUTION_SESSION_MAX_ENTRIES", 1<<14,
),
MTProtoRPCResultPendingPerAuth: envIntOr("TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH", 1<<11),
MTProtoRPCExecutionPendingPerAuth: envIntOr("TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH", 1<<11),
MTProtoInboundFrameGlobalMaxBytes: envInt64Or("TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES", 512<<20),
MTProtoOutboundQueueSize: envIntOr("TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE", 128),
MTProtoOutboundControlQueueSize: envIntOr("TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE", 32),
@ -863,7 +863,7 @@ func Load() (Config, error) {
if err := validateLoginEmailConfig(cfg); err != nil {
return Config{}, err
}
if err := validateRPCResultCacheConfig(cfg); err != nil {
if err := validateRPCExecutionConfig(cfg); err != nil {
return Config{}, err
}
if err := validateStarGiftConfig(cfg); err != nil {
@ -1222,21 +1222,21 @@ func validateCollectibleUsernameConfig(cfg Config) error {
return nil
}
func validateRPCResultCacheConfig(cfg Config) error {
if cfg.MTProtoRPCResultCacheMaxEntries <= 0 || cfg.MTProtoRPCResultCacheAuthMaxEntries <= 0 ||
cfg.MTProtoRPCResultCacheSessionMaxEntries <= 0 {
return fmt.Errorf("MTProto rpc_result entry limits must be positive")
func validateRPCExecutionConfig(cfg Config) error {
if cfg.MTProtoRPCExecutionMaxEntries <= 0 || cfg.MTProtoRPCExecutionAuthMaxEntries <= 0 ||
cfg.MTProtoRPCExecutionSessionMaxEntries <= 0 {
return fmt.Errorf("MTProto rpc execution entry limits must be positive")
}
if cfg.MTProtoRPCResultCacheMaxEntries < cfg.MTProtoRPCResultCacheAuthMaxEntries ||
cfg.MTProtoRPCResultCacheAuthMaxEntries < cfg.MTProtoRPCResultCacheSessionMaxEntries {
return fmt.Errorf("MTProto rpc_result entry hierarchy must satisfy global >= auth >= session: %d/%d/%d",
cfg.MTProtoRPCResultCacheMaxEntries, cfg.MTProtoRPCResultCacheAuthMaxEntries, cfg.MTProtoRPCResultCacheSessionMaxEntries)
if cfg.MTProtoRPCExecutionMaxEntries < cfg.MTProtoRPCExecutionAuthMaxEntries ||
cfg.MTProtoRPCExecutionAuthMaxEntries < cfg.MTProtoRPCExecutionSessionMaxEntries {
return fmt.Errorf("MTProto rpc execution entry hierarchy must satisfy global >= auth >= session: %d/%d/%d",
cfg.MTProtoRPCExecutionMaxEntries, cfg.MTProtoRPCExecutionAuthMaxEntries, cfg.MTProtoRPCExecutionSessionMaxEntries)
}
if cfg.MTProtoRPCGlobalMaxTasks <= 0 || cfg.MTProtoRPCResultPendingPerAuth <= 0 ||
cfg.MTProtoRPCResultPendingPerAuth > cfg.MTProtoRPCGlobalMaxTasks ||
cfg.MTProtoRPCResultPendingPerAuth > cfg.MTProtoRPCResultCacheAuthMaxEntries {
return fmt.Errorf("MTProto rpc_result pending-per-auth %d must be positive and <= global pending %d and auth entries %d",
cfg.MTProtoRPCResultPendingPerAuth, cfg.MTProtoRPCGlobalMaxTasks, cfg.MTProtoRPCResultCacheAuthMaxEntries)
if cfg.MTProtoRPCGlobalMaxTasks <= 0 || cfg.MTProtoRPCExecutionPendingPerAuth <= 0 ||
cfg.MTProtoRPCExecutionPendingPerAuth > cfg.MTProtoRPCGlobalMaxTasks ||
cfg.MTProtoRPCExecutionPendingPerAuth > cfg.MTProtoRPCExecutionAuthMaxEntries {
return fmt.Errorf("MTProto rpc execution pending-per-auth %d must be positive and <= global pending %d and auth entries %d",
cfg.MTProtoRPCExecutionPendingPerAuth, cfg.MTProtoRPCGlobalMaxTasks, cfg.MTProtoRPCExecutionAuthMaxEntries)
}
return nil
}
@ -1531,10 +1531,10 @@ func validateStrictMTProtoCapacityEnv(e envSource) error {
"TELESRV_MTPROTO_RPC_QUEUE_SIZE",
"TELESRV_MTPROTO_RPC_GLOBAL_WORKERS",
"TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS",
"TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES",
"TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES",
"TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES",
"TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH",
"TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES",
"TELESRV_MTPROTO_RPC_EXECUTION_AUTH_MAX_ENTRIES",
"TELESRV_MTPROTO_RPC_EXECUTION_SESSION_MAX_ENTRIES",
"TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH",
"TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE",
"TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE",
} {

54
internal/config/config_test.go
View file

@ -176,10 +176,10 @@ func TestLoadMTProtoAdmissionAndRPCBudgets(t *testing.T) {
t.Setenv("TELESRV_MTPROTO_RPC_GLOBAL_WORKERS", "33")
t.Setenv("TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS", "444")
t.Setenv("TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES", "555555")
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", "555")
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES", "444")
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES", "333")
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH", "222")
t.Setenv("TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES", "555")
t.Setenv("TELESRV_MTPROTO_RPC_EXECUTION_AUTH_MAX_ENTRIES", "444")
t.Setenv("TELESRV_MTPROTO_RPC_EXECUTION_SESSION_MAX_ENTRIES", "333")
t.Setenv("TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH", "222")
t.Setenv("TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES", "777777")
t.Setenv("TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE", "88")
t.Setenv("TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE", "22")
@ -200,15 +200,15 @@ func TestLoadMTProtoAdmissionAndRPCBudgets(t *testing.T) {
cfg.MTProtoRPCGlobalWorkers != 33 || cfg.MTProtoRPCGlobalMaxTasks != 444 || cfg.MTProtoRPCGlobalMaxBytes != 555555 {
t.Fatalf("rpc budget config = %d/%d/%v/%d/%d/%d", cfg.MTProtoRPCMaxInflight, cfg.MTProtoRPCQueueSize, cfg.MTProtoRPCTimeout, cfg.MTProtoRPCGlobalWorkers, cfg.MTProtoRPCGlobalMaxTasks, cfg.MTProtoRPCGlobalMaxBytes)
}
if cfg.MTProtoRPCResultCacheMaxEntries != 555 ||
cfg.MTProtoRPCResultCacheAuthMaxEntries != 444 ||
cfg.MTProtoRPCResultCacheSessionMaxEntries != 333 ||
cfg.MTProtoRPCResultPendingPerAuth != 222 {
t.Fatalf("rpc result receipt config = global:%d auth:%d session:%d pending/auth:%d",
cfg.MTProtoRPCResultCacheMaxEntries,
cfg.MTProtoRPCResultCacheAuthMaxEntries,
cfg.MTProtoRPCResultCacheSessionMaxEntries,
cfg.MTProtoRPCResultPendingPerAuth)
if cfg.MTProtoRPCExecutionMaxEntries != 555 ||
cfg.MTProtoRPCExecutionAuthMaxEntries != 444 ||
cfg.MTProtoRPCExecutionSessionMaxEntries != 333 ||
cfg.MTProtoRPCExecutionPendingPerAuth != 222 {
t.Fatalf("rpc execution ledger config = global:%d auth:%d session:%d pending/auth:%d",
cfg.MTProtoRPCExecutionMaxEntries,
cfg.MTProtoRPCExecutionAuthMaxEntries,
cfg.MTProtoRPCExecutionSessionMaxEntries,
cfg.MTProtoRPCExecutionPendingPerAuth)
}
if cfg.MTProtoInboundFrameGlobalMaxBytes != 777777 {
t.Fatalf("inbound frame budget config = %d", cfg.MTProtoInboundFrameGlobalMaxBytes)
@ -221,32 +221,32 @@ func TestLoadMTProtoAdmissionAndRPCBudgets(t *testing.T) {
}
}
func TestLoadRPCResultFairBudgetDefaults(t *testing.T) {
func TestLoadRPCExecutionFairBudgetDefaults(t *testing.T) {
disableDefaultConfigFile(t)
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.MTProtoRPCResultCacheMaxEntries != 1<<18 ||
cfg.MTProtoRPCResultCacheAuthMaxEntries != 1<<15 ||
cfg.MTProtoRPCResultCacheSessionMaxEntries != 1<<14 ||
cfg.MTProtoRPCResultPendingPerAuth != 1<<11 {
t.Fatalf("rpc_result receipt defaults = global:%d auth:%d session:%d pending/auth:%d",
cfg.MTProtoRPCResultCacheMaxEntries,
cfg.MTProtoRPCResultCacheAuthMaxEntries,
cfg.MTProtoRPCResultCacheSessionMaxEntries,
cfg.MTProtoRPCResultPendingPerAuth)
if cfg.MTProtoRPCExecutionMaxEntries != 1<<18 ||
cfg.MTProtoRPCExecutionAuthMaxEntries != 1<<15 ||
cfg.MTProtoRPCExecutionSessionMaxEntries != 1<<14 ||
cfg.MTProtoRPCExecutionPendingPerAuth != 1<<11 {
t.Fatalf("rpc execution receipt defaults = global:%d auth:%d session:%d pending/auth:%d",
cfg.MTProtoRPCExecutionMaxEntries,
cfg.MTProtoRPCExecutionAuthMaxEntries,
cfg.MTProtoRPCExecutionSessionMaxEntries,
cfg.MTProtoRPCExecutionPendingPerAuth)
}
}
func TestLoadRejectsInvalidRPCResultFairBudgets(t *testing.T) {
func TestLoadRejectsInvalidRPCExecutionFairBudgets(t *testing.T) {
tests := []struct {
name string
key string
value string
}{
{name: "entry hierarchy", key: "TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", value: "1024"},
{name: "pending hierarchy", key: "TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH", value: "9000"},
{name: "entry hierarchy", key: "TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES", value: "1024"},
{name: "pending hierarchy", key: "TELESRV_MTPROTO_RPC_EXECUTION_PENDING_PER_AUTH", value: "9000"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
@ -266,7 +266,7 @@ func TestLoadRejectsMalformedMTProtoCapacity(t *testing.T) {
value string
}{
{name: "worker tasks malformed", key: "TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS", value: "lots"},
{name: "receipt entries overflow", key: "TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", value: "999999999999999999999999"},
{name: "receipt entries overflow", key: "TELESRV_MTPROTO_RPC_EXECUTION_MAX_ENTRIES", value: "999999999999999999999999"},
{name: "tracked bytes overflow", key: "TELESRV_MTPROTO_OUTBOUND_TRACKED_GLOBAL_MAX_BYTES", value: "999999999999999999999999"},
{name: "outbound queue malformed", key: "TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE", value: "many"},
} {

6
internal/loadharness/run.go
View file

@ -920,9 +920,9 @@ func evaluateReport(report *RunReport, cfg RunConfig) {
checks := []string{
"telesrv_mtproto_raw_connections", "telesrv_mtproto_logical_sessions",
"telesrv_mtproto_logical_outbox_bytes", "telesrv_mtproto_pending_push_bytes",
"telesrv_mtproto_outbound_tracked_bytes", "telesrv_mtproto_rpc_result_owners",
"telesrv_mtproto_rpc_result_receipts", "telesrv_mtproto_rpc_result_receipt_bytes",
"telesrv_mtproto_rpc_result_subscribers",
"telesrv_mtproto_outbound_tracked_bytes", "telesrv_mtproto_rpc_execution_owners",
"telesrv_mtproto_rpc_execution_reserved_entries", "telesrv_mtproto_rpc_execution_receipts",
"telesrv_mtproto_rpc_execution_receipt_budget_bytes", "telesrv_mtproto_rpc_execution_subscribers",
}
for _, name := range checks {
baseline := metricValue(report.BaselineServerMetrics, name)

9
internal/loadharness/server_metrics.go
View file

@ -31,10 +31,11 @@ var selectedServerMetrics = map[string]struct{}{
"telesrv_mtproto_inbound_frame_bytes": {},
"telesrv_mtproto_outbound_tracked_bytes": {},
"telesrv_mtproto_outbound_write_bytes": {},
"telesrv_mtproto_rpc_result_owners": {},
"telesrv_mtproto_rpc_result_receipts": {},
"telesrv_mtproto_rpc_result_receipt_bytes": {},
"telesrv_mtproto_rpc_result_subscribers": {},
"telesrv_mtproto_rpc_execution_owners": {},
"telesrv_mtproto_rpc_execution_reserved_entries": {},
"telesrv_mtproto_rpc_execution_receipts": {},
"telesrv_mtproto_rpc_execution_receipt_budget_bytes": {},
"telesrv_mtproto_rpc_execution_subscribers": {},
"telesrv_mtproto_rpc_result_inner_bytes_total": {},
"telesrv_mtproto_rpc_result_wire_bytes_total": {},
"telesrv_mtproto_rpc_result_delivered_bytes_total": {},

2
internal/mtprotoedge/conn.go
View file

@ -133,7 +133,7 @@ type Conn struct {
rpcReady bool
rpcClosed bool
// rpcReplayRestores is a per-physical-connection ordering barrier. An exact
// cached/rewrapped init request has already executed its business handler,
// replayed/rewrapped init request has already executed its business handler,
// but its wrapper/client/readiness state becomes authoritative only after the
// replacement rpc_result is physically written. Queued naked RPCs remain
// admitted and budgeted, but are not scheduler-runnable until every such

2
internal/mtprotoedge/conn_layer_profile.go
View file

@ -137,7 +137,7 @@ func (c *Conn) layerProfileRawEvidenceState() (LayerProfileSnapshot, int, int64)
// freezeLayerProfileAt is the production explicit-evidence transition. The
// positive client msg_id is the protocol ordering authority across TCP
// reconnects and cached request replays.
// reconnects and retained request replays.
func (c *Conn) freezeLayerProfileAt(profile tlprofile.Profile, msgID int64) (bool, error) {
if c == nil {
return false, fmt.Errorf("nil connection layer profile")

56
internal/mtprotoedge/encrypted.go
View file

@ -898,24 +898,16 @@ func (s *Server) publishRPCResult(
if s == nil || s.rpcResults == nil || c == nil || encoded == nil || reqMsgID == 0 {
return errors.New("rpc result receipt ledger is unavailable")
}
if s.rpcResults.sessions == nil && int64(len(encoded.body)) > s.rpcResults.completedBytes.max {
// Focused legacy-cache tests may intentionally use a byte budget smaller
// than one legal transport result. Production receipts never own bodies.
panic(fmt.Sprintf(
"mtprotoedge: legacy encoded rpc result exceeds inline-ledger budget: body=%d max=%d",
len(encoded.body), s.rpcResults.completedBytes.max,
))
}
priority, visible := prepareEncoded(encoded)
if owner != nil && !owner.HandOff() {
return ErrRPCResultFlightInvalid
}
started := time.Now()
encoded.markReplayable()
// Put may expose a completed result only after the old logical connection
// Complete may expose terminal execution only after the old connection
// is irreversibly unable to accept another same-generation request.
c.fenceUndeliveredRPCResult()
s.storeRPCResult(c, reqMsgID, encoded)
s.completeRPCResult(c, reqMsgID, encoded, false)
latency := time.Since(started)
if metrics, ok := s.metrics.(RPCResultMetrics); ok {
metrics.RPCResultDelivered(method, latency, len(encoded.body), admissionErr)
@ -989,7 +981,7 @@ func (s *Server) publishRPCResult(
if deliveryErr != nil {
encoded.markReplayable()
c.fenceUndeliveredRPCResult()
s.storeRPCResult(c, reqMsgID, encoded)
s.completeRPCResult(c, reqMsgID, encoded, true)
if checked := s.log.Check(resultLogLevel, "RPC result delivery fenced for replay"); checked != nil {
checked.Write(
zap.String("method", method), zap.Int64("req_msg_id", reqMsgID),
@ -1001,7 +993,7 @@ func (s *Server) publishRPCResult(
return
}
encoded.markDelivered()
s.storeRPCResult(c, reqMsgID, encoded)
s.completeRPCResult(c, reqMsgID, encoded, true)
if checked := s.log.Check(resultLogLevel, "RPC result delivered"); checked != nil {
checked.Write(
zap.String("method", method), zap.Int64("req_msg_id", reqMsgID),
@ -1054,24 +1046,24 @@ func (s *Server) sendResult(ctx context.Context, c *Conn, reqMsgID int64, result
// same-Conn duplicate would be ACKed while no result can ever arrive.
c.fenceUndeliveredRPCResult()
encoded.markReplayable()
s.storeRPCResult(c, reqMsgID, encoded)
s.completeRPCResult(c, reqMsgID, encoded, true)
return err
}
encoded.markDelivered()
// On a live Conn, completed means the rpc_result has reached the reliable byte
// stream. Same-physical duplicates can therefore be ACK-only without data loss.
s.storeRPCResult(c, reqMsgID, encoded)
s.completeRPCResult(c, reqMsgID, encoded, true)
return nil
}
// sendCachedRPCResult preserves the delivery half of the rpc_result invariant
// sendReplayedRPCResult preserves the delivery half of the rpc_result invariant
// for completed-flight replays: either the logical outbox result reaches this physical
// byte stream, or this logical Conn is fenced so a replacement may retry it.
func (s *Server) sendCachedRPCResult(ctx context.Context, c *Conn, encoded *encodedOutboundMessage) error {
return s.sendCachedRPCResultWithHook(ctx, c, encoded, nil)
func (s *Server) sendReplayedRPCResult(ctx context.Context, c *Conn, encoded *encodedOutboundMessage) error {
return s.sendReplayedRPCResultWithHook(ctx, c, encoded, nil)
}
func (s *Server) sendCachedRPCResultWithHook(
func (s *Server) sendReplayedRPCResultWithHook(
ctx context.Context,
c *Conn,
encoded *encodedOutboundMessage,
@ -1079,7 +1071,7 @@ func (s *Server) sendCachedRPCResultWithHook(
) error {
if encoded == nil {
c.fenceUndeliveredRPCResult()
return errors.New("nil cached rpc_result")
return errors.New("nil replayed rpc_result")
}
attempt, reserved, err := c.cloneRPCResultForRequestReserved(encoded, encoded.reqMsgID, false)
if err != nil {
@ -1096,7 +1088,7 @@ func (s *Server) sendCachedRPCResultWithHook(
finishRestore = c.beginRPCReplayRestore()
defer finishRestore()
}
// Cached replay owns its delivery-gated state synchronously. Calling the
// Outbox replay owns its delivery-gated state synchronously. Calling the
// lower send primitive avoids reserving the process-wide asynchronous hook
// executor; the logical hook is claimed only after this physical write wins.
if err := c.sendOutboundWithTerminalReserved(
@ -1117,10 +1109,10 @@ func (s *Server) sendCachedRPCResultWithHook(
// the sticky deferral). Fence before the deferred barrier is released; a
// later physical generation may wait for Done and replay the same bytes.
c.fenceUndeliveredRPCResult()
return fmt.Errorf("wait for cached rpc_result logical restore: %w", claimErr)
return fmt.Errorf("wait for replayed rpc_result logical restore: %w", claimErr)
}
return s.runBoundedRPCReplayRestore(
restoreCtx, c, "cached rpc_result", logicalRestore, afterSuccessfulDelivery,
restoreCtx, c, "replayed rpc_result", logicalRestore, afterSuccessfulDelivery,
)
}
@ -1320,11 +1312,11 @@ func (s *Server) encodeRPCResultWithoutSlot(ctx context.Context, c *Conn, reqMsg
}, nil
}
func (s *Server) cachedRPCResult(c *Conn, reqMsgID int64) (*encodedOutboundMessage, bool) {
func (s *Server) replayableRPCResult(c *Conn, reqMsgID int64) (*encodedOutboundMessage, bool) {
if s == nil || s.rpcResults == nil || c == nil {
return nil, false
}
return s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID)
return s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgID)
}
func (s *Server) replayRPCResultByRequest(ctx context.Context, c *Conn, reqMsgID int64) error {
@ -1335,24 +1327,26 @@ func (s *Server) replayRPCResultByRequest(ctx context.Context, c *Conn, reqMsgID
c.fenceUndeliveredRPCResult()
return err
} else if resent {
s.log.Debug("Resent connection cached rpc_result for duplicate msg_id", zap.Int64("msg_id", reqMsgID))
s.log.Debug("Resent connection-retained rpc_result for duplicate msg_id", zap.Int64("msg_id", reqMsgID))
return nil
}
if cached, ok := s.cachedRPCResult(c, reqMsgID); ok {
if err := s.sendCachedRPCResult(ctx, c, cached); err != nil {
if replayed, ok := s.replayableRPCResult(c, reqMsgID); ok {
if err := s.sendReplayedRPCResult(ctx, c, replayed); err != nil {
return err
}
s.log.Debug("Resent session cached rpc_result for duplicate msg_id", zap.Int64("msg_id", reqMsgID))
s.log.Debug("Resent logical-outbox rpc_result for duplicate msg_id", zap.Int64("msg_id", reqMsgID))
}
return nil
}
func (s *Server) storeRPCResult(c *Conn, reqMsgID int64, encoded *encodedOutboundMessage) {
func (s *Server) completeRPCResult(c *Conn, reqMsgID int64, encoded *encodedOutboundMessage, replayable bool) {
if s == nil || s.rpcResults == nil || c == nil {
return
}
s.conns.adoptLogicalSession(c)
s.rpcResults.Put(c.authKeyID, c.sessionID, reqMsgID, encoded)
if s.conns != nil {
s.conns.adoptLogicalSession(c)
}
s.rpcResults.Complete(c.authKeyID, c.sessionID, reqMsgID, encoded, replayable)
}
// sendPong 回复 mt.PingRequest / mt.PingDelayDisconnectRequest。

8
internal/mtprotoedge/inbound_layer_rpc.go
View file

@ -61,7 +61,7 @@ type layerRPCProfileEvidence struct {
// layerRPCAdmissionCursor is the wire-ordered, side-effect-free view used while
// decoding one MTProto container. evidenceMsgID is the last explicit
// invokeWithLayer proof, not merely the profile used by an arbitrary cached
// invokeWithLayer proof, not merely the profile used by an arbitrary retained
// request. It therefore advances only after generated admission reports
// ProfileEvidence.
type layerRPCAdmissionCursor struct {
@ -355,7 +355,7 @@ func (s *Server) prepareInboundLayerRPCBatch(ctx context.Context, c *Conn, plan
item.method = method
if profile, hasEvidence := admitted.ProfileEvidence(); hasEvidence {
if existing && profile != existingProfile {
return fmt.Errorf("%w: cached msg_id %d used Layer %d but replay selected Layer %d", ErrLayerProfileConflict, item.msgID, existingProfile, profile)
return fmt.Errorf("%w: retained msg_id %d used Layer %d but replay selected Layer %d", ErrLayerProfileConflict, item.msgID, existingProfile, profile)
}
evidence[index] = layerRPCProfileEvidence{profile: profile, present: true, fresh: item.profileEvidenceFresh()}
if evidence[index].fresh {
@ -436,7 +436,7 @@ func (s *Server) prepareInboundLayerRPCBatch(ctx context.Context, c *Conn, plan
case rpcResultAcquireCompleted:
// Transfer the materialization ticket to the plan before any
// profile-restore step can fail; plan.close is the universal abort
// path and sendCached releases it after outbound-budget handoff.
// path and sendReplayed releases it after outbound-budget handoff.
item.payload = claim.encoded
after, prepareErr := s.prepareAdmittedLayerRPCReplay(ctx, c, item.msgID, claim.admissionSeq, item.profileEvidenceFresh(), item.admitted)
if prepareErr != nil {
@ -1034,7 +1034,7 @@ func (s *Server) decodeInboundLayerRPCWithOptions(state LayerProfileSnapshot, bo
// commitLayerProfileEvidence publishes one generated invokeWithLayer proof.
// The exact-session registry is the cross-physical-connection linearization
// point; the Conn cursor then prevents a concurrent older admission from
// overwriting its local wire epoch. Older cached duplicates remain decodable
// overwriting its local wire epoch. Older retained duplicates remain decodable
// and request-bound, but cannot mutate session/profile state.
func (s *Server) commitLayerProfileEvidence(ctx context.Context, c *Conn, profile tlprofile.Profile, msgID int64) (bool, error) {
if s == nil || c == nil {

14
internal/mtprotoedge/inbound_layer_rpc_test.go
View file

@ -966,7 +966,7 @@ func TestDurabilityOutageInitializesOnlyCurrentConnection(t *testing.T) {
}
time.Sleep(time.Millisecond)
}
if _, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, msgID); !ok {
if _, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, msgID); !ok {
t.Fatal("successful outage-local init did not publish its exact RPC result")
}
if layer, found, err := router.ResolveInheritedAuthKeyLayer(ctx, c.authKeyID); err != nil || found || layer != 0 {
@ -1071,7 +1071,7 @@ func TestInvariantReplayNeverCachesInternalCanonicalProfile(t *testing.T) {
func TestSameMsgIDNakedReplayUsesWinnerAdmissionProfile(t *testing.T) {
handler := newAdmissionOnlyLayerRPC()
s := New(Options{DC: 2, LayerRPC: handler})
s.rpcResults = newRPCResultCacheWithFlightLimit(time.Now, 8)
s.rpcResults = newRPCExecutionLedgerForServerTest(s, time.Now, 8)
authKeyID := [8]byte{0x22, 0x99}
const sessionID = int64(2299)
body := exactOutboundLayerRPCBody(t, tlprofile.Profile225, &tg.MessagesGetHistoryRequest{
@ -1201,7 +1201,7 @@ func TestLayerEvidencePublicationBelongsOnlyToFreshFlightOwner(t *testing.T) {
func TestLayerEvidenceNotPublishedWhenBatchFlightCapacityRollsBack(t *testing.T) {
handler := newAdmissionOnlyLayerRPC()
s := New(Options{DC: 2, LayerRPC: handler})
s.rpcResults = newRPCResultCacheWithFlightLimit(time.Now, 1)
s.rpcResults = newRPCExecutionLedgerForServerTest(s, time.Now, 1)
c := &Conn{authKeyID: [8]byte{0x22, 0x97}, sessionID: 2297, metrics: NopMetrics{}}
c.startInboundRPCScheduler(s.rpcScheduler, 1, 8, time.Second)
defer c.Close()
@ -1285,7 +1285,7 @@ func TestLogicalSessionLayerWatermarkSurvivesResultExpiryAndOldContainer(t *test
now := newExpiryTestClock(time.Unix(1_900_000_000, 0))
router := rpc.New(rpc.Config{DC: 2}, rpc.Deps{}, zaptest.NewLogger(t), now)
s := New(Options{DC: 2, LayerRPC: router, Clock: now})
s.rpcResults = newRPCResultCacheWithFlightLimit(now.Now, 8)
s.rpcResults = newRPCExecutionLedgerForServerTest(s, now.Now, 8)
authKeyID := [8]byte{0x22, 0x95}
const sessionID = int64(2295)
newConn := func() *Conn {
@ -1331,7 +1331,7 @@ func TestLogicalSessionLayerWatermarkSurvivesResultExpiryAndOldContainer(t *test
// inherited default remains Layer 227, while an old inner request may execute
// request-bound but cannot recreate or roll back the exact-session watermark.
now.Advance(10 * time.Minute)
if _, ok := s.rpcResults.Get(authKeyID, sessionID, oldMsgID); ok {
if _, ok := s.rpcResults.Replay(authKeyID, sessionID, oldMsgID); ok {
t.Fatal("old completed result did not expire")
}
@ -1978,7 +1978,7 @@ func TestUnprofiledInvariantBindKeepsProfileUnknownAndReturnsExactBool(t *testin
func TestLayerRPCBatchCapacityKeepsExistingPendingReplay(t *testing.T) {
router := rpc.New(rpc.Config{DC: 2, IP: "127.0.0.1", Port: 2398}, rpc.Deps{}, zaptest.NewLogger(t), clock.System)
s := New(Options{DC: 2, LayerRPC: router})
s.rpcResults = newRPCResultCacheWithFlightLimit(time.Now, 1)
s.rpcResults = newRPCExecutionLedgerForServerTest(s, time.Now, 1)
c := &Conn{
authKeyID: [8]byte{7, 7, 1},
sessionID: 771,
@ -2033,7 +2033,7 @@ func TestLayerRPCBatchCapacityKeepsExistingPendingReplay(t *testing.T) {
}
func TestLayerRPCBatchCapacityAbortsRejectedRewrapOwner(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
cache := newRPCExecutionLedgerForTest(time.Now, 1)
authKeyID := [8]byte{7, 7, 9}
const (
sessionID = int64(779)

10
internal/mtprotoedge/inbound_preflight.go
View file

@ -41,7 +41,7 @@ const (
// It never dispatches business code a second time.
inboundItemRewrappedRPC
// inboundItemReplayRPC is a request first observed by this physical Conn whose
// terminal result already exists in the cross-connection cache. It is distinct
// terminal result already exists in the cross-connection execution ledger. It is distinct
// from inboundItemDuplicate: a duplicate already present in this Conn's seen
// table must only be ACKed. The original owner/result is already using the same
// reliable TCP stream, so replaying it once per retransmit wave amplifies a
@ -732,7 +732,7 @@ func preflightInboundItem(msgID int64, seqNo int32, typeID uint32, content bool,
// prepareInboundRPCBatch performs the whole container's count/byte admission
// before copying or scheduling any API RPC. Capacity exhaustion is converted
// into one consistent local 500 WORKER_BUSY_TOO_LONG_RETRY result per uncached
// into one consistent local 500 WORKER_BUSY_TOO_LONG_RETRY result per new
// RPC; no business handler from the batch is allowed to start in that case.
func (s *Server) prepareInboundRPCBatch(ctx context.Context, c *Conn, plan *inboundPlan) error {
if s.layerRPC != nil {
@ -846,7 +846,7 @@ func (s *Server) prepareInboundRPCBatch(ctx context.Context, c *Conn, plan *inbo
}
switch claim.state {
case rpcResultAcquireCompleted:
s.log.Info("RPC duplicate replay from session cache",
s.log.Info("RPC duplicate replay from logical-session outbox",
zap.String("method", method),
zap.Int64("msg_id", item.msgID),
zap.String("auth_key_id", c.authKeyHex),
@ -943,14 +943,14 @@ func (s *Server) executeInboundPlan(ctx context.Context, cs *connState, c *Conn,
case inboundItemServiceDuplicate:
// Classify from the originally committed connState record, never from the
// retransmitted body. This prevents same-id payload replacement. Only an
// already cached answer is eligible for resend (rpc_drop_answer today);
// already retained answer is eligible for resend (rpc_drop_answer today);
// other best-effort service traffic uses a later fresh request.
if err := s.replayRPCResultByRequest(ctx, c, item.msgID); err != nil {
return err
}
case inboundItemReplayRPC:
if encoded, _ := item.payload.(*encodedOutboundMessage); encoded != nil {
if err := s.sendCachedRPCResultWithHook(ctx, c, encoded, item.replayAfterSuccessfulDelivery); err != nil {
if err := s.sendReplayedRPCResultWithHook(ctx, c, encoded, item.replayAfterSuccessfulDelivery); err != nil {
return err
}
} else if err := s.replayRPCResultByRequest(ctx, c, item.msgID); err != nil {

2
internal/mtprotoedge/layer_admission_budget_test.go
View file

@ -279,7 +279,7 @@ func TestLayerRPCAdmissionNestedGZIPSiblingsShareFrameExpansionLimit(t *testing.
func TestLayerRPCAdmissionNestedGZIPReDecodeReusesMaterializationCharge(t *testing.T) {
handler := newAdmissionOnlyLayerRPC()
s := New(Options{DC: 2, LayerRPC: handler, Logger: zaptest.NewLogger(t)})
s.rpcResults = newRPCResultCacheWithFlightLimit(time.Now, 8)
s.rpcResults = newRPCExecutionLedgerForServerTest(s, time.Now, 8)
scheduler := newInboundRPCScheduler(1, 4, 1<<30)
s.rpcScheduler = scheduler
authKeyID := [8]byte{8, 24}

2
internal/mtprotoedge/layer_rpc_execution.go
View file

@ -82,7 +82,7 @@ var errLayerRPCResultIdentityMismatch = errors.New("layer RPC result does not ma
// generated dispatcher. A LayerRPCHandler implementation must return the
// result capability created from this exact admission; accepting a result from
// another request would pair the wrong result TypeRef/profile with this
// flight/cache identity even when both methods happen to share a Go type.
// execution-ledger identity even when both methods happen to share a Go type.
func bindAdmittedLayerRPCResult(request tlprofile.Admission, result tlprofile.Result) (*layerRPCResultEncoder, error) {
if result == nil {
return nil, nil

2
internal/mtprotoedge/layer_rpc_execution_test.go
View file

@ -138,7 +138,7 @@ func TestProjectionFailureCachesInternalWithoutRepeatingBusiness(t *testing.T) {
if rpcErr.ErrorCode != 500 || rpcErr.ErrorMessage != "INTERNAL" {
t.Fatalf("projection terminal = %+v", rpcErr)
}
if _, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID); !ok {
if _, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgID); !ok {
t.Fatal("projection receipt disappeared before duplicate acquire")
}
// A same-msg replay is served from the completed exact identity; there is no

2
internal/mtprotoedge/layer_updates.go
View file

@ -32,7 +32,7 @@ const (
// legacy constructions which do not declare a stronger ownership model.
outboundLayerBindingSession outboundLayerBindingKind = iota
// A request-bound result retains the profile captured by admission. A later
// invokeWithLayer correction must not invalidate that in-flight/cached result.
// invokeWithLayer correction must not invalidate that in-flight/retained result.
outboundLayerBindingRequest
)

29
internal/mtprotoedge/logical_session_test.go
View file

@ -69,7 +69,7 @@ func storeLogicalRPCResultForTest(
encoded.replayMsgID = frame.msgID
encoded.replaySeqNo = frame.seqNo
state.mu.Unlock()
s.rpcResults.Put(c.authKeyID, c.sessionID, reqMsgID, encoded)
s.rpcResults.Complete(c.authKeyID, c.sessionID, reqMsgID, encoded, true)
}
func acknowledgeLogicalRPCResultForTest(t *testing.T, s *Server, c *Conn, reqMsgID int64) {
@ -160,12 +160,10 @@ func TestLogicalSessionACKReleasesPayloadAndReceipt(t *testing.T) {
}
c.outboundState.mu.Unlock()
ledger := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
ledger := newRPCExecutionLedger(time.Now, rpcExecutionLedgerCapacity{
maxPending: 16, maxPendingPerAuth: 16,
globalMaxEntries: 16, globalMaxBytes: 16,
authMaxEntries: 16, authMaxBytes: 16,
sessionMaxEntries: 16, sessionMaxBytes: 16,
sessions: manager,
globalMaxEntries: 16, authMaxEntries: 16, sessionMaxEntries: 16,
replayStore: manager,
})
claim, err := ledger.Acquire(authKeyID, 43, 88)
if err != nil || claim.state != rpcResultAcquireOwner {
@ -173,17 +171,20 @@ func TestLogicalSessionACKReleasesPayloadAndReceipt(t *testing.T) {
}
claim.owner.CompleteExecution(true)
claim.owner.HandOff()
ledger.Put(authKeyID, 43, 88, &encodedOutboundMessage{body: body, typeID: proto.ResultTypeID, reqMsgID: 88})
ledger.Complete(authKeyID, 43, 88, &encodedOutboundMessage{body: body, typeID: proto.ResultTypeID, reqMsgID: 88}, true)
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: 43, reqMsgID: 88}
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: 43, reqMsgID: 88}
shard := ledger.shard(key)
shard.mu.Lock()
entry := shard.byKey[key].Value.(*rpcResultCacheEntry)
if entry.encoded != nil || entry.size != 1 {
entry := shard.byKey[key].Value.(*rpcExecutionReceipt)
if entry.unavailable {
shard.mu.Unlock()
t.Fatalf("completed ledger retained payload: encoded=%p size=%d", entry.encoded, entry.size)
t.Fatal("logical outbox receipt was marked unavailable")
}
shard.mu.Unlock()
if got := ledger.receiptBudgetBytes(); got != rpcExecutionReceiptBudgetBytes {
t.Fatalf("receipt budget bytes = %d, want %d", got, rpcExecutionReceiptBudgetBytes)
}
c.outboundState.mu.Lock()
acked := c.outboundState.ack([]int64{201})
@ -241,13 +242,13 @@ func TestLogicalSessionDestroyReleasesPayloadAndCompletedReceipt(t *testing.T) {
storeLogicalRPCResultForTest(t, s, c, 901, &encodedOutboundMessage{
body: make([]byte, 32), typeID: proto.ResultTypeID, reqMsgID: 901,
})
if _, ok := s.rpcResults.Get(authKeyID, 91, 901); !ok {
if _, ok := s.rpcResults.Replay(authKeyID, 91, 901); !ok {
t.Fatal("logical result fixture was not published")
}
if removed := s.conns.DestroySessionForAuthKey(authKeyID, 91); removed {
t.Fatal("construction-only logical session unexpectedly reported active")
}
if _, ok := s.rpcResults.Get(authKeyID, 91, 901); ok {
if _, ok := s.rpcResults.Replay(authKeyID, 91, 901); ok {
t.Fatal("destroy retained completed receipt")
}
if got := s.outboundTrackedBudget.snapshot(); got != 0 {
@ -272,7 +273,7 @@ func TestBusinessAuthRevocationReleasesOfflineTempLogicalSession(t *testing.T) {
if closed := s.conns.CloseSessionsForBusinessAuthKey(businessAuthKeyID); closed != 0 {
t.Fatalf("offline logical revocation closed %d physical conns", closed)
}
if _, ok := s.rpcResults.Get(rawAuthKeyID, 92, 902); ok {
if _, ok := s.rpcResults.Replay(rawAuthKeyID, 92, 902); ok {
t.Fatal("business auth revocation retained temp-key receipt")
}
if got := s.outboundTrackedBudget.snapshot(); got != 0 {

71
internal/mtprotoedge/production_compat_test.go
View file

@ -4,6 +4,7 @@ import (
"context"
"errors"
"sort"
"sync"
"time"
"go.uber.org/zap"
@ -25,10 +26,72 @@ func (c *Conn) ForceClose() {
c.waitOutboundShutdown()
}
// newRPCResultCache keeps older focused cache tests concise without exposing a
// second production constructor.
func newRPCResultCache(now func() time.Time) *rpcResultCache {
return newRPCResultCacheWithFlightLimit(now, rpcResultFlightDefaultMaxPending)
type rpcReplayStoreForTest struct {
mu sync.Mutex
results map[rpcExecutionKey]*encodedOutboundMessage
}
func newRPCReplayStoreForTest() *rpcReplayStoreForTest {
return &rpcReplayStoreForTest{results: make(map[rpcExecutionKey]*encodedOutboundMessage)}
}
func (s *rpcReplayStoreForTest) rpcResult(
authKeyID [8]byte,
sessionID, reqMsgID int64,
) (*encodedOutboundMessage, bool) {
if s == nil {
return nil, false
}
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s.mu.Lock()
result, ok := s.results[key]
s.mu.Unlock()
return result, ok
}
func (s *rpcReplayStoreForTest) put(
authKeyID [8]byte,
sessionID, reqMsgID int64,
encoded *encodedOutboundMessage,
) {
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s.mu.Lock()
s.results[key] = encoded
s.mu.Unlock()
}
func newRPCExecutionLedgerForTest(now func() time.Time, maxPending int) *rpcExecutionLedger {
return newRPCExecutionLedger(now, rpcExecutionLedgerCapacity{
maxPending: maxPending, maxPendingPerAuth: maxPending,
globalMaxEntries: rpcExecutionMaxEntries,
authMaxEntries: rpcExecutionMaxEntries, sessionMaxEntries: rpcExecutionMaxEntries,
replayStore: newRPCReplayStoreForTest(),
})
}
func newRPCExecutionLedgerForServerTest(s *Server, now func() time.Time, maxPending int) *rpcExecutionLedger {
if s == nil || s.conns == nil {
panic("newRPCExecutionLedgerForServerTest requires a server SessionManager")
}
return newRPCExecutionLedger(now, rpcExecutionLedgerCapacity{
maxPending: maxPending, maxPendingPerAuth: maxPending,
globalMaxEntries: rpcExecutionMaxEntries,
authMaxEntries: rpcExecutionMaxEntries, sessionMaxEntries: rpcExecutionMaxEntries,
replayStore: s.conns,
})
}
func (l *rpcExecutionLedger) completeReplayableForTest(
authKeyID [8]byte,
sessionID, reqMsgID int64,
encoded *encodedOutboundMessage,
) {
store, ok := l.replayStore.(*rpcReplayStoreForTest)
if !ok {
panic("completeReplayableForTest requires rpcReplayStoreForTest")
}
store.put(authKeyID, sessionID, reqMsgID, encoded)
l.Complete(authKeyID, sessionID, reqMsgID, encoded, true)
}
// Conns is a white-box test accessor. Production wires the shared manager

4
internal/mtprotoedge/rpc_delivery_coordinator_test.go
View file

@ -279,7 +279,7 @@ func TestCachedReplacementReplayBarrierWaitsForLogicalHookDone(t *testing.T) {
firstConn := newOutboundTestConn(t, &collectingSessionTransport{}, newOutboundTrackedBudget(1<<20))
firstResult := make(chan error, 1)
go func() {
firstResult <- s.sendCachedRPCResultWithHook(context.Background(), firstConn, encoded, func() error {
firstResult <- s.sendReplayedRPCResultWithHook(context.Background(), firstConn, encoded, func() error {
if !order.CompareAndSwap(0, 1) {
return errors.New("first replacement restore ran out of order")
}
@ -300,7 +300,7 @@ func TestCachedReplacementReplayBarrierWaitsForLogicalHookDone(t *testing.T) {
secondReplacement := make(chan struct{})
secondResult := make(chan error, 1)
go func() {
secondResult <- s.sendCachedRPCResultWithHook(context.Background(), secondConn, encoded, func() error {
secondResult <- s.sendReplayedRPCResultWithHook(context.Background(), secondConn, encoded, func() error {
if !order.CompareAndSwap(2, 3) {
return errors.New("second replacement restore passed logical hook completion")
}

206
internal/mtprotoedge/rpc_execution_budget.go Normal file
View file

@ -0,0 +1,206 @@
package mtprotoedge
import (
"encoding/binary"
"hash/maphash"
"sync"
)
const rpcExecutionBudgetShards = 64
type rpcExecutionBudgetUsage struct {
entries int64
pending int64
}
type rpcExecutionSessionBudgetKey struct {
authKeyID [8]byte
sessionID int64
}
type rpcExecutionAuthBudgetShard struct {
mu sync.Mutex
usage map[[8]byte]rpcExecutionBudgetUsage
}
type rpcExecutionSessionBudgetShard struct {
mu sync.Mutex
usage map[rpcExecutionSessionBudgetKey]rpcExecutionBudgetUsage
}
// rpcExecutionFairBudget accounts one bounded ledger slot at global, auth and
// session scopes. Pending owner and completed receipt are the same ownership:
// Complete transfers the reservation; Abort/ACK/TTL return it.
type rpcExecutionFairBudget struct {
seed maphash.Seed
globalEntries *rpcResultFlightLimit
authLimit int64
sessionLimit int64
pendingPerAuth int64
authShards [rpcExecutionBudgetShards]rpcExecutionAuthBudgetShard
sessionShards [rpcExecutionBudgetShards]rpcExecutionSessionBudgetShard
}
type rpcExecutionBudgetReservation struct {
budget *rpcExecutionFairBudget
key rpcExecutionKey
pending bool
released bool
}
func newRPCExecutionFairBudget(
seed maphash.Seed,
globalEntries *rpcResultFlightLimit,
authLimit int64,
sessionLimit int64,
pendingPerAuth int,
) *rpcExecutionFairBudget {
b := &rpcExecutionFairBudget{
seed: seed, globalEntries: globalEntries, authLimit: authLimit,
sessionLimit: sessionLimit, pendingPerAuth: int64(pendingPerAuth),
}
for i := range b.authShards {
b.authShards[i].usage = make(map[[8]byte]rpcExecutionBudgetUsage)
b.sessionShards[i].usage = make(map[rpcExecutionSessionBudgetKey]rpcExecutionBudgetUsage)
}
return b
}
func (b *rpcExecutionFairBudget) reserveOwner(key rpcExecutionKey) *rpcExecutionBudgetReservation {
return b.reserve(key, true)
}
func (b *rpcExecutionFairBudget) reserveCompleted(key rpcExecutionKey) *rpcExecutionBudgetReservation {
return b.reserve(key, false)
}
func (b *rpcExecutionFairBudget) reserve(key rpcExecutionKey, pending bool) *rpcExecutionBudgetReservation {
if b == nil || b.globalEntries == nil {
return nil
}
authShard := b.authShard(key.authKeyID)
sessionKey := rpcExecutionSessionBudgetKey{authKeyID: key.authKeyID, sessionID: key.sessionID}
sessionShard := b.sessionShard(sessionKey)
authShard.mu.Lock()
sessionShard.mu.Lock()
authUsage := authShard.usage[key.authKeyID]
sessionUsage := sessionShard.usage[sessionKey]
allowed := withinRPCExecutionBudget(authUsage.entries, 1, b.authLimit) &&
withinRPCExecutionBudget(sessionUsage.entries, 1, b.sessionLimit)
if pending {
allowed = allowed && withinRPCExecutionBudget(authUsage.pending, 1, b.pendingPerAuth)
}
if !allowed || !b.globalEntries.reserve() {
sessionShard.mu.Unlock()
authShard.mu.Unlock()
return nil
}
authUsage.entries++
sessionUsage.entries++
if pending {
authUsage.pending++
}
authShard.usage[key.authKeyID] = authUsage
sessionShard.usage[sessionKey] = sessionUsage
sessionShard.mu.Unlock()
authShard.mu.Unlock()
return &rpcExecutionBudgetReservation{budget: b, key: key, pending: pending}
}
func withinRPCExecutionBudget(used, delta, limit int64) bool {
return delta >= 0 && limit > 0 && used >= 0 && used <= limit-delta
}
func (r *rpcExecutionBudgetReservation) releasePending() {
if r == nil || r.budget == nil || r.released || !r.pending {
return
}
b := r.budget
shard := b.authShard(r.key.authKeyID)
shard.mu.Lock()
usage, ok := shard.usage[r.key.authKeyID]
if !ok || usage.pending < 1 {
shard.mu.Unlock()
panic("mtprotoedge: rpc execution per-auth pending budget underflow")
}
usage.pending--
shard.usage[r.key.authKeyID] = usage
r.pending = false
shard.mu.Unlock()
}
func (r *rpcExecutionBudgetReservation) release() {
if r == nil || r.budget == nil || r.released {
return
}
b := r.budget
authShard := b.authShard(r.key.authKeyID)
sessionKey := rpcExecutionSessionBudgetKey{authKeyID: r.key.authKeyID, sessionID: r.key.sessionID}
sessionShard := b.sessionShard(sessionKey)
authShard.mu.Lock()
sessionShard.mu.Lock()
authUsage, authOK := authShard.usage[r.key.authKeyID]
sessionUsage, sessionOK := sessionShard.usage[sessionKey]
if !authOK || !sessionOK || authUsage.entries < 1 || sessionUsage.entries < 1 ||
(r.pending && authUsage.pending < 1) {
sessionShard.mu.Unlock()
authShard.mu.Unlock()
panic("mtprotoedge: rpc execution fair budget underflow")
}
authUsage.entries--
sessionUsage.entries--
if r.pending {
authUsage.pending--
}
if authUsage == (rpcExecutionBudgetUsage{}) {
delete(authShard.usage, r.key.authKeyID)
} else {
authShard.usage[r.key.authKeyID] = authUsage
}
if sessionUsage == (rpcExecutionBudgetUsage{}) {
delete(sessionShard.usage, sessionKey)
} else {
sessionShard.usage[sessionKey] = sessionUsage
}
r.released = true
r.pending = false
b.globalEntries.release()
sessionShard.mu.Unlock()
authShard.mu.Unlock()
}
func (b *rpcExecutionFairBudget) authSnapshot(authKeyID [8]byte) rpcExecutionBudgetUsage {
if b == nil {
return rpcExecutionBudgetUsage{}
}
shard := b.authShard(authKeyID)
shard.mu.Lock()
usage := shard.usage[authKeyID]
shard.mu.Unlock()
return usage
}
func (b *rpcExecutionFairBudget) sessionSnapshot(authKeyID [8]byte, sessionID int64) rpcExecutionBudgetUsage {
if b == nil {
return rpcExecutionBudgetUsage{}
}
key := rpcExecutionSessionBudgetKey{authKeyID: authKeyID, sessionID: sessionID}
shard := b.sessionShard(key)
shard.mu.Lock()
usage := shard.usage[key]
shard.mu.Unlock()
return usage
}
func (b *rpcExecutionFairBudget) authShard(authKeyID [8]byte) *rpcExecutionAuthBudgetShard {
index := maphash.Bytes(b.seed, authKeyID[:]) & (rpcExecutionBudgetShards - 1)
return &b.authShards[index]
}
func (b *rpcExecutionFairBudget) sessionShard(key rpcExecutionSessionBudgetKey) *rpcExecutionSessionBudgetShard {
var raw [16]byte
copy(raw[:8], key.authKeyID[:])
binary.LittleEndian.PutUint64(raw[8:], uint64(key.sessionID))
index := maphash.Bytes(b.seed, raw[:]) & (rpcExecutionBudgetShards - 1)
return &b.sessionShards[index]
}

490
internal/mtprotoedge/rpc_execution_ledger.go Normal file
View file

@ -0,0 +1,490 @@
package mtprotoedge
import (
"container/list"
"context"
"encoding/binary"
"hash/maphash"
"sync"
"sync/atomic"
"time"
)
const (
// A valid client msg_id can be up to five minutes old or thirty seconds in
// the future. The extra second covers scheduler and boundary jitter. This is
// a no-ACK execution-receipt horizon, not a payload retention policy.
rpcExecutionReceiptTTL = 331 * time.Second
rpcExecutionMaxEntries = 1 << 18
rpcExecutionAuthMaxEntries = 1 << 15
rpcExecutionSessionMaxEntries = 1 << 14
rpcExecutionPendingPerAuth = 1 << 11
// Receipts contain only fixed-shape identity/outcome metadata. This
// conservative charge covers the receipt, list node, map bucket share and
// reservation bookkeeping. Payload bytes are accounted exclusively by the
// logical-session outbox.
rpcExecutionReceiptBudgetBytes = 384
// The complete replay identity is hashed with an instance-random seed. The
// shard count is a power of two.
rpcExecutionLedgerShards = 16
)
type rpcExecutionKey struct {
authKeyID [8]byte
sessionID int64
reqMsgID int64
}
// rpcReplayStore is the sole source of retained rpc_result payloads. The
// production implementation is SessionManager's logical-session outbox.
type rpcReplayStore interface {
rpcResult(authKeyID [8]byte, sessionID, reqMsgID int64) (*encodedOutboundMessage, bool)
}
type rpcExecutionReceipt struct {
key rpcExecutionKey
expiresAt time.Time
identity rpcResultRequestIdentity
admissionSeq uint64
executionKnown bool
executionOK bool
// acknowledged exists only for the ACK-before-Complete race. A completed
// receipt is removed immediately when the ACK wins.
acknowledged bool
// unavailable is a bounded execution tombstone. It prevents a completed
// business operation from running again when no exact outbox frame exists.
unavailable bool
// The pending owner transfers this same global/auth/session entry
// reservation to the completed receipt.
reservation *rpcExecutionBudgetReservation
}
type rpcResultDependency struct {
waiter *rpcResultWaiter
completed bool
success bool
}
// rpcExecutionLedger owns request execution identity and duplicate
// coordination. It never owns or copies rpc_result payload bytes; replay bytes
// are resolved from replayStore only while the logical-session outbox retains
// the exact unacknowledged frame.
type rpcExecutionLedger struct {
shards [rpcExecutionLedgerShards]rpcExecutionLedgerShard
hashSeed maphash.Seed
reservedEntries rpcResultFlightLimit
receiptCount atomic.Int64
fairBudget *rpcExecutionFairBudget
flightLimit rpcResultFlightLimit
subscriberBudget *rpcResultSubscriberBudget
subscriberPerFlight int
replayStore rpcReplayStore
nextAdmissionSeq atomic.Uint64
activeAdmissions rpcAdmissionTracker
}
func (l *rpcExecutionLedger) stableAdmissionSafeFloor() uint64 {
if l == nil {
return 0
}
return l.activeAdmissions.stableSafeFloor(&l.nextAdmissionSeq)
}
type rpcExecutionLedgerShard struct {
mu sync.Mutex
now func() time.Time
ttl time.Duration
receiptCount *atomic.Int64
order *list.List
byKey map[rpcExecutionKey]*list.Element
// In-flight owners are independent from receipt TTL and cannot disappear
// under completed-receipt pressure.
pending map[rpcExecutionKey]*rpcResultFlight
}
type rpcExecutionLedgerCapacity struct {
maxPending int
maxPendingPerAuth int
globalMaxEntries int
authMaxEntries int
sessionMaxEntries int
subscriberMaxGlobal int
subscriberMaxAuth int
subscriberMaxSession int
subscriberMaxPerFlight int
replayStore rpcReplayStore
}
func newRPCExecutionLedger(now func() time.Time, capacity rpcExecutionLedgerCapacity) *rpcExecutionLedger {
if now == nil {
now = time.Now
}
if capacity.replayStore == nil {
panic("mtprotoedge: rpc execution ledger requires a replay store")
}
if capacity.maxPending <= 0 {
capacity.maxPending = rpcResultFlightDefaultMaxPending
}
if capacity.maxPendingPerAuth <= 0 {
capacity.maxPendingPerAuth = capacity.maxPending
}
if capacity.globalMaxEntries <= 0 {
capacity.globalMaxEntries = rpcExecutionMaxEntries
}
if capacity.authMaxEntries <= 0 {
capacity.authMaxEntries = capacity.globalMaxEntries
}
if capacity.sessionMaxEntries <= 0 {
capacity.sessionMaxEntries = capacity.authMaxEntries
}
if capacity.subscriberMaxGlobal <= 0 {
capacity.subscriberMaxGlobal = rpcResultSubscriberMaxGlobal
}
if capacity.subscriberMaxAuth <= 0 {
capacity.subscriberMaxAuth = rpcResultSubscriberMaxAuth
}
if capacity.subscriberMaxSession <= 0 {
capacity.subscriberMaxSession = rpcResultSubscriberMaxSession
}
if capacity.subscriberMaxPerFlight <= 0 {
capacity.subscriberMaxPerFlight = rpcResultSubscriberMaxPerFlight
}
l := &rpcExecutionLedger{hashSeed: maphash.MakeSeed(), replayStore: capacity.replayStore}
l.reservedEntries.max = int64(capacity.globalMaxEntries)
l.flightLimit.max = int64(capacity.maxPending)
l.fairBudget = newRPCExecutionFairBudget(
l.hashSeed,
&l.reservedEntries,
int64(capacity.authMaxEntries),
int64(capacity.sessionMaxEntries),
capacity.maxPendingPerAuth,
)
l.subscriberBudget = newRPCResultSubscriberBudget(
l.hashSeed,
capacity.subscriberMaxGlobal,
capacity.subscriberMaxAuth,
capacity.subscriberMaxSession,
)
l.subscriberPerFlight = capacity.subscriberMaxPerFlight
for i := range l.shards {
s := &l.shards[i]
s.now = now
s.ttl = rpcExecutionReceiptTTL
s.receiptCount = &l.receiptCount
s.order = list.New()
s.byKey = make(map[rpcExecutionKey]*list.Element)
s.pending = make(map[rpcExecutionKey]*rpcResultFlight)
}
return l
}
func (l *rpcExecutionLedger) shard(key rpcExecutionKey) *rpcExecutionLedgerShard {
return &l.shards[l.shardIndex(key)]
}
func (l *rpcExecutionLedger) shardIndex(key rpcExecutionKey) uint64 {
var raw [24]byte
copy(raw[:8], key.authKeyID[:])
binary.LittleEndian.PutUint64(raw[8:16], uint64(key.sessionID))
binary.LittleEndian.PutUint64(raw[16:24], uint64(key.reqMsgID))
return maphash.Bytes(l.hashSeed, raw[:]) & (rpcExecutionLedgerShards - 1)
}
// Replay resolves an immutable result descriptor from the logical-session
// outbox. A receipt hit without an outbox frame is never treated as permission
// to execute the business handler again.
func (l *rpcExecutionLedger) Replay(authKeyID [8]byte, sessionID, reqMsgID int64) (*encodedOutboundMessage, bool) {
if l == nil || reqMsgID == 0 {
return nil, false
}
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s := l.shard(key)
now := s.now()
s.mu.Lock()
elem := s.byKey[key]
if elem == nil {
s.mu.Unlock()
return nil, false
}
receipt := elem.Value.(*rpcExecutionReceipt)
if !receipt.expiresAt.After(now) {
s.removeElement(elem)
s.mu.Unlock()
return nil, false
}
if receipt.unavailable || receipt.acknowledged {
s.mu.Unlock()
return nil, false
}
s.mu.Unlock()
return l.replayStore.rpcResult(authKeyID, sessionID, reqMsgID)
}
// Acknowledge removes a completed receipt immediately. reqMsgID must already
// have been resolved from the outbound actor's trusted server-msg-id mapping.
// A pending flight records the ACK so a racing completion cannot resurrect a
// receipt after the outbox body has been released.
func (l *rpcExecutionLedger) Acknowledge(authKeyID [8]byte, sessionID, reqMsgID int64) bool {
if l == nil || reqMsgID == 0 {
return false
}
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s := l.shard(key)
s.mu.Lock()
s.expireLocked(s.now())
if elem := s.byKey[key]; elem != nil {
s.removeElement(elem)
s.mu.Unlock()
return true
}
if flight := s.pending[key]; flight != nil {
flight.acknowledged = true
s.mu.Unlock()
return true
}
s.mu.Unlock()
return false
}
// ObserveDependency returns a waiter for an admitted in-flight dependency, a
// completed execution outcome, or ok=false for an unknown request. It never
// creates execution ownership.
func (l *rpcExecutionLedger) ObserveDependency(authKeyID [8]byte, sessionID, reqMsgID int64) (rpcResultDependency, bool) {
if l == nil || reqMsgID == 0 {
return rpcResultDependency{}, false
}
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s := l.shard(key)
now := s.now()
s.mu.Lock()
defer s.mu.Unlock()
if elem := s.byKey[key]; elem != nil {
receipt := elem.Value.(*rpcExecutionReceipt)
if receipt.expiresAt.After(now) {
if !receipt.executionKnown {
return rpcResultDependency{}, false
}
return rpcResultDependency{completed: true, success: receipt.executionOK}, true
}
s.removeElement(elem)
}
if flight := s.pending[key]; flight != nil {
if flight.executionDone {
return rpcResultDependency{completed: true, success: flight.executionOK}, true
}
return rpcResultDependency{waiter: &rpcResultWaiter{ledger: l, key: key, flight: flight}}, true
}
return rpcResultDependency{}, false
}
// Complete publishes terminal execution metadata and resolves current
// waiters. replayable is only a claim from the egress path; the ledger verifies
// that replayStore actually owns the exact frame before publishing a replayable
// receipt. encoded is passed transiently to joined waiters and is never stored.
func (l *rpcExecutionLedger) Complete(
authKeyID [8]byte,
sessionID, reqMsgID int64,
encoded *encodedOutboundMessage,
replayable bool,
) {
if l == nil || reqMsgID == 0 || encoded == nil {
return
}
if replayable {
_, replayable = l.replayStore.rpcResult(authKeyID, sessionID, reqMsgID)
}
if l.completeOnce(authKeyID, sessionID, reqMsgID, encoded, replayable) {
return
}
// An expired receipt in another shard may be the only global blocker.
l.expireReceipts()
_ = l.completeOnce(authKeyID, sessionID, reqMsgID, encoded, replayable)
}
// completeOnce returns false only when a cross-shard expiry reap may release
// the global reservation required by a defensive completion without an owner.
func (l *rpcExecutionLedger) completeOnce(
authKeyID [8]byte,
sessionID, reqMsgID int64,
encoded *encodedOutboundMessage,
replayable bool,
) bool {
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s := l.shard(key)
s.mu.Lock()
now := s.now()
s.expireLocked(now)
old := s.byKey[key]
flight := s.pending[key]
var oldReceipt *rpcExecutionReceipt
if old != nil {
oldReceipt = old.Value.(*rpcExecutionReceipt)
if flight == nil {
// Terminal publication is immutable. Late callbacks cannot extend TTL,
// replace replay identity or resurrect an acknowledged result.
s.mu.Unlock()
return true
}
}
identity, admissionSeq, executionKnown, executionOK, acknowledged := rpcResultFlightMetadataLocked(s, key)
var reservation *rpcExecutionBudgetReservation
switch {
case flight != nil:
reservation = flight.reservation
if reservation == nil {
s.mu.Unlock()
panic("mtprotoedge: pending rpc execution has no fair-budget reservation")
}
case oldReceipt != nil && oldReceipt.reservation != nil:
reservation = oldReceipt.reservation
default:
reservation = l.fairBudget.reserveCompleted(key)
if reservation == nil {
s.mu.Unlock()
return false
}
}
if old != nil {
s.unlinkElement(old)
if oldReceipt.reservation != nil && oldReceipt.reservation != reservation {
oldReceipt.reservation.release()
oldReceipt.reservation = nil
}
}
receipt := &rpcExecutionReceipt{
key: key,
expiresAt: now.Add(s.ttl),
identity: identity,
admissionSeq: admissionSeq,
executionKnown: executionKnown,
executionOK: executionOK,
acknowledged: acknowledged,
unavailable: !replayable,
reservation: reservation,
}
elem := s.order.PushBack(receipt)
s.byKey[key] = elem
s.incrementReceiptCount()
subscribers, executionSubscribers, terminalExecutionOK := l.completeRPCResultFlightLocked(s, key, encoded)
if acknowledged {
// ACK won before completion. Current subscribers still receive encoded,
// but no post-ACK receipt survives this critical section.
s.removeElement(elem)
}
s.mu.Unlock()
for _, subscriber := range subscribers {
subscriber(encoded, true)
}
for _, subscriber := range executionSubscribers {
subscriber(terminalExecutionOK)
}
return true
}
func rpcResultFlightMetadataLocked(s *rpcExecutionLedgerShard, key rpcExecutionKey) (
rpcResultRequestIdentity,
uint64,
bool,
bool,
bool,
) {
if flight := s.pending[key]; flight != nil {
return flight.identity, flight.admissionSeq, flight.executionDone, flight.executionOK, flight.acknowledged
}
return rpcResultRequestIdentity{}, 0, false, false, false
}
func (l *rpcExecutionLedger) expireReceipts() {
if l == nil {
return
}
for i := range l.shards {
s := &l.shards[i]
s.mu.Lock()
s.expireLocked(s.now())
s.mu.Unlock()
}
}
func (l *rpcExecutionLedger) receiptBudgetBytes() int64 {
if l == nil {
return 0
}
return l.receiptCount.Load() * rpcExecutionReceiptBudgetBytes
}
func (l *rpcExecutionLedger) Close() error { return nil }
func (l *rpcExecutionLedger) CloseContext(context.Context) error { return nil }
// forgetSession removes every terminal receipt for a destroyed logical
// session. SessionManager calls it only after physical producers converge.
func (l *rpcExecutionLedger) forgetSession(authKeyID [8]byte, sessionID int64) {
if l == nil {
return
}
for i := range l.shards {
s := &l.shards[i]
s.mu.Lock()
for elem := s.order.Front(); elem != nil; {
next := elem.Next()
receipt := elem.Value.(*rpcExecutionReceipt)
if receipt.key.authKeyID == authKeyID && receipt.key.sessionID == sessionID {
s.removeElement(elem)
}
elem = next
}
s.mu.Unlock()
}
}
func (s *rpcExecutionLedgerShard) expireLocked(now time.Time) {
for elem := s.order.Front(); elem != nil; {
next := elem.Next()
receipt := elem.Value.(*rpcExecutionReceipt)
if receipt.expiresAt.After(now) {
return
}
s.removeElement(elem)
elem = next
}
}
func (s *rpcExecutionLedgerShard) removeElement(elem *list.Element) {
receipt := s.unlinkElement(elem)
if receipt != nil && receipt.reservation != nil {
receipt.reservation.release()
receipt.reservation = nil
}
}
func (s *rpcExecutionLedgerShard) unlinkElement(elem *list.Element) *rpcExecutionReceipt {
if elem == nil {
return nil
}
receipt := elem.Value.(*rpcExecutionReceipt)
delete(s.byKey, receipt.key)
s.order.Remove(elem)
s.decrementReceiptCount()
return receipt
}
func (s *rpcExecutionLedgerShard) incrementReceiptCount() {
if s.receiptCount == nil {
panic("mtprotoedge: rpc execution receipt counter is unavailable")
}
s.receiptCount.Add(1)
}
func (s *rpcExecutionLedgerShard) decrementReceiptCount() {
if s.receiptCount == nil || s.receiptCount.Add(-1) < 0 {
panic("mtprotoedge: rpc execution receipt counter underflow")
}
}

346
internal/mtprotoedge/rpc_execution_ledger_test.go Normal file
View file

@ -0,0 +1,346 @@
package mtprotoedge
import (
"container/list"
"errors"
"sync"
"testing"
"time"
"unsafe"
)
func newRPCExecutionLedgerWithLimitsForTest(
now func() time.Time,
maxPending, maxPendingPerAuth, global, auth, session int,
) *rpcExecutionLedger {
return newRPCExecutionLedger(now, rpcExecutionLedgerCapacity{
maxPending: maxPending, maxPendingPerAuth: maxPendingPerAuth,
globalMaxEntries: global, authMaxEntries: auth, sessionMaxEntries: session,
replayStore: newRPCReplayStoreForTest(),
})
}
func TestRPCExecutionLedgerSessionCapacityIsolatesAnotherAuth(t *testing.T) {
ledger := newRPCExecutionLedgerWithLimitsForTest(time.Now, 8, 6, 8, 6, 2)
authA := [8]byte{0xa1}
authB := [8]byte{0xb1}
for i := 0; i < 2; i++ {
msgID := int64(1000 + i)
claim, err := ledger.Acquire(authA, 77, msgID)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("same-session admission %d = %#v, %v", i, claim, err)
}
ledger.completeReplayableForTest(authA, 77, msgID, &encodedOutboundMessage{body: []byte{1}})
}
if _, err := ledger.Acquire(authA, 77, 2000); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("admission beyond session limit = %v, want capacity", err)
}
other, err := ledger.Acquire(authB, 88, 3000)
if err != nil || other.state != rpcResultAcquireOwner {
t.Fatalf("other auth blocked by full session: %#v, %v", other, err)
}
other.owner.Abort()
}
func TestRPCExecutionLedgerAuthCapacityIsolatesAnotherAuth(t *testing.T) {
ledger := newRPCExecutionLedgerWithLimitsForTest(time.Now, 8, 4, 8, 2, 2)
authA := [8]byte{0xa2}
authB := [8]byte{0xb2}
for i := 0; i < 2; i++ {
claim, err := ledger.Acquire(authA, int64(10+i), int64(100+i))
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("auth A admission %d = %#v, %v", i, claim, err)
}
ledger.completeReplayableForTest(authA, int64(10+i), int64(100+i), &encodedOutboundMessage{body: []byte{1}})
}
if _, err := ledger.Acquire(authA, 12, 102); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("same auth beyond limit = %v, want capacity", err)
}
other, err := ledger.Acquire(authB, 20, 200)
if err != nil || other.state != rpcResultAcquireOwner {
t.Fatalf("other auth blocked by auth A: %#v, %v", other, err)
}
other.owner.Abort()
}
func TestRPCExecutionLedgerPendingLimitIsAdditional(t *testing.T) {
ledger := newRPCExecutionLedgerWithLimitsForTest(time.Now, 6, 2, 12, 6, 4)
authA := [8]byte{0xa3}
authB := [8]byte{0xb3}
owners := make([]*rpcResultOwnerLease, 0, 3)
for i := 0; i < 2; i++ {
claim, err := ledger.Acquire(authA, int64(i+1), int64(100+i))
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("pending auth A %d = %#v, %v", i, claim, err)
}
owners = append(owners, claim.owner)
}
if _, err := ledger.Acquire(authA, 3, 103); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("third pending owner for auth A = %v, want capacity", err)
}
other, err := ledger.Acquire(authB, 4, 104)
if err != nil || other.state != rpcResultAcquireOwner {
t.Fatalf("auth B blocked by auth A pending limit: %#v, %v", other, err)
}
owners = append(owners, other.owner)
for _, owner := range owners {
if !owner.Abort() {
t.Fatal("pending owner did not abort")
}
}
if usage := ledger.fairBudget.authSnapshot(authA); usage != (rpcExecutionBudgetUsage{}) {
t.Fatalf("auth A budget after abort = %#v", usage)
}
}
func TestRPCExecutionLedgerReceiptLifecycleACKAndTTL(t *testing.T) {
now := time.Unix(1000, 0)
ledger := newRPCExecutionLedgerWithLimitsForTest(func() time.Time { return now }, 4, 4, 6, 5, 3)
auth := [8]byte{0xc1}
claim, err := ledger.Acquire(auth, 1, 101)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("owner = %#v, %v", claim, err)
}
if usage := ledger.fairBudget.authSnapshot(auth); usage.entries != 1 || usage.pending != 1 {
t.Fatalf("pending reservation = %#v", usage)
}
claim.owner.CompleteExecution(true)
ledger.completeReplayableForTest(auth, 1, 101, &encodedOutboundMessage{body: make([]byte, 8<<20)})
if ledger.flightLimit.snapshot() != 0 || ledger.receiptCount.Load() != 1 || ledger.reservedEntries.snapshot() != 1 {
t.Fatalf("terminal counts owner=%d receipt=%d reserved=%d", ledger.flightLimit.snapshot(), ledger.receiptCount.Load(), ledger.reservedEntries.snapshot())
}
if got := ledger.receiptBudgetBytes(); got != rpcExecutionReceiptBudgetBytes {
t.Fatalf("8 MiB result charged %d receipt bytes, want fixed %d", got, rpcExecutionReceiptBudgetBytes)
}
if !ledger.Acknowledge(auth, 1, 101) {
t.Fatal("ACK did not remove receipt")
}
if ledger.receiptCount.Load() != 0 || ledger.reservedEntries.snapshot() != 0 || ledger.receiptBudgetBytes() != 0 {
t.Fatal("ACK leaked receipt reservation")
}
second, err := ledger.Acquire(auth, 2, 201)
if err != nil || second.state != rpcResultAcquireOwner {
t.Fatalf("second owner = %#v, %v", second, err)
}
ledger.completeReplayableForTest(auth, 2, 201, &encodedOutboundMessage{body: []byte{1}})
now = now.Add(rpcExecutionReceiptTTL + time.Second)
if _, ok := ledger.Replay(auth, 2, 201); ok {
t.Fatal("expired receipt remained replayable")
}
if ledger.receiptCount.Load() != 0 || ledger.reservedEntries.snapshot() != 0 {
t.Fatal("TTL leaked receipt reservation")
}
}
func TestRPCExecutionLedgerACKBeforeCompleteDoesNotResurrectReceipt(t *testing.T) {
ledger := newRPCExecutionLedgerForTest(time.Now, 2)
auth := [8]byte{0xd1}
claim, err := ledger.Acquire(auth, 1, 101)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("owner = %#v, %v", claim, err)
}
joined, err := ledger.Acquire(auth, 1, 101)
if err != nil || joined.state != rpcResultAcquirePending {
t.Fatalf("join = %#v, %v", joined, err)
}
if !ledger.Acknowledge(auth, 1, 101) {
t.Fatal("ACK did not mark pending owner")
}
want := &encodedOutboundMessage{body: []byte{1}, reqMsgID: 101}
ledger.completeReplayableForTest(auth, 1, 101, want)
if got, ok, waitErr := joined.waiter.Wait(t.Context()); waitErr != nil || !ok || got != want {
t.Fatalf("joined waiter = %p/%v/%v", got, ok, waitErr)
}
if ledger.receiptCount.Load() != 0 || ledger.reservedEntries.snapshot() != 0 {
t.Fatal("ACK-before-complete resurrected receipt")
}
newClaim, err := ledger.Acquire(auth, 1, 101)
if err != nil || newClaim.state != rpcResultAcquireOwner {
t.Fatalf("post-ACK request did not get a fresh owner: %#v, %v", newClaim, err)
}
newClaim.owner.Abort()
}
func TestRPCExecutionLedgerUnavailableTombstonePreventsReexecution(t *testing.T) {
now := time.Unix(1000, 0)
ledger := newRPCExecutionLedgerWithLimitsForTest(func() time.Time { return now }, 2, 2, 4, 4, 4)
auth := [8]byte{0xe1}
claim, err := ledger.Acquire(auth, 1, 101)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("owner = %#v, %v", claim, err)
}
claim.owner.CompleteExecution(true)
ledger.Complete(auth, 1, 101, &encodedOutboundMessage{body: make([]byte, 8<<20)}, false)
if _, ok := ledger.Replay(auth, 1, 101); ok {
t.Fatal("unavailable tombstone masqueraded as replayable")
}
if _, err := ledger.Acquire(auth, 1, 101); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("duplicate after unavailable completion = %v, want capacity", err)
}
if got := ledger.receiptBudgetBytes(); got != rpcExecutionReceiptBudgetBytes {
t.Fatalf("unavailable receipt budget = %d", got)
}
now = now.Add(rpcExecutionReceiptTTL + time.Second)
retry, err := ledger.Acquire(auth, 1, 101)
if err != nil || retry.state != rpcResultAcquireOwner {
t.Fatalf("admission after tombstone expiry = %#v, %v", retry, err)
}
retry.owner.Abort()
}
func TestRPCExecutionLedgerConcurrentReservationsNeverOvercommit(t *testing.T) {
const limit = 24
ledger := newRPCExecutionLedgerWithLimitsForTest(time.Now, limit, 4, limit, 8, 3)
const callers = 256
start := make(chan struct{})
var (
wg sync.WaitGroup
mu sync.Mutex
owners []*rpcResultOwnerLease
)
for i := 0; i < callers; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
<-start
auth := [8]byte{byte(i % 4)}
claim, err := ledger.Acquire(auth, int64(i%8), int64(1000+i))
if errors.Is(err, ErrRPCResultFlightCapacity) {
return
}
if err != nil || claim.state != rpcResultAcquireOwner {
t.Errorf("Acquire %d = %#v, %v", i, claim, err)
return
}
mu.Lock()
owners = append(owners, claim.owner)
mu.Unlock()
}(i)
}
close(start)
wg.Wait()
if got := ledger.reservedEntries.snapshot(); got > limit || got != int64(len(owners)) {
t.Fatalf("reserved=%d owners=%d limit=%d", got, len(owners), limit)
}
for i := 0; i < 4; i++ {
auth := [8]byte{byte(i)}
usage := ledger.fairBudget.authSnapshot(auth)
if usage.entries > 8 || usage.pending > 4 {
t.Fatalf("auth %d overcommitted: %#v", i, usage)
}
}
for _, owner := range owners {
owner.Abort()
}
if ledger.reservedEntries.snapshot() != 0 {
t.Fatal("concurrent abort leaked reservations")
}
}
func TestRPCExecutionLedgerFullKeyHashSpreadsOneSession(t *testing.T) {
first := newRPCExecutionLedgerForTest(time.Now, 64)
second := newRPCExecutionLedgerForTest(time.Now, 64)
auth := [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
seen := make(map[uint64]struct{})
differentInstance := false
for msgID := int64(1); msgID <= 256; msgID++ {
key := rpcExecutionKey{authKeyID: auth, sessionID: 99, reqMsgID: msgID}
firstIndex := first.shardIndex(key)
seen[firstIndex] = struct{}{}
if firstIndex != second.shardIndex(key) {
differentInstance = true
}
}
if len(seen) < rpcExecutionLedgerShards/2 {
t.Fatalf("one session used only %d/%d shards", len(seen), rpcExecutionLedgerShards)
}
if !differentInstance {
t.Fatal("two ledger instances used an identical shard stream")
}
}
func TestRPCExecutionLedgerForgetSessionReleasesReceipts(t *testing.T) {
ledger := newRPCExecutionLedgerForTest(time.Now, 8)
auth := [8]byte{0xf1}
for _, sessionID := range []int64{1, 1, 2} {
msgID := int64(100 + ledger.receiptCount.Load())
claim, err := ledger.Acquire(auth, sessionID, msgID)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("owner session=%d: %#v, %v", sessionID, claim, err)
}
ledger.completeReplayableForTest(auth, sessionID, msgID, &encodedOutboundMessage{body: []byte{1}})
}
ledger.forgetSession(auth, 1)
if got := ledger.receiptCount.Load(); got != 1 {
t.Fatalf("receipts after session forget = %d, want 1", got)
}
if _, ok := ledger.Replay(auth, 1, 100); ok {
t.Fatal("forgotten session remained replayable")
}
}
func TestRPCExecutionLedgerServerOptionsPropagateLimits(t *testing.T) {
s := New(Options{
RPCGlobalMaxTasks: 6,
RPCExecutionMaxEntries: 12,
RPCExecutionAuthMaxEntries: 8,
RPCExecutionSessionMaxEntries: 4,
RPCExecutionPendingPerAuth: 3,
})
if s.rpcResults.reservedEntries.max != 12 {
t.Fatalf("global option propagation = %d", s.rpcResults.reservedEntries.max)
}
budget := s.rpcResults.fairBudget
if budget.authLimit != 8 || budget.sessionLimit != 4 || budget.pendingPerAuth != 3 {
t.Fatalf("fair option propagation = auth:%d session:%d pending:%d", budget.authLimit, budget.sessionLimit, budget.pendingPerAuth)
}
}
func TestRPCExecutionLedgerServerOptionsFailFast(t *testing.T) {
base := Options{
RPCGlobalMaxTasks: 6,
RPCExecutionMaxEntries: 12,
RPCExecutionAuthMaxEntries: 8,
RPCExecutionSessionMaxEntries: 4,
RPCExecutionPendingPerAuth: 3,
}
tests := []struct {
name string
mutate func(*Options)
}{
{name: "entry hierarchy", mutate: func(o *Options) { o.RPCExecutionAuthMaxEntries = 13 }},
{name: "pending hierarchy", mutate: func(o *Options) { o.RPCExecutionPendingPerAuth = 7 }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
opts := base
test.mutate(&opts)
defer func() {
if recover() == nil {
t.Fatal("New accepted invalid rpc execution options")
}
}()
_ = New(opts)
})
}
}
func TestRPCExecutionLedgerRequiresReplayStore(t *testing.T) {
defer func() {
if recover() == nil {
t.Fatal("ledger accepted nil replay store")
}
}()
_ = newRPCExecutionLedger(time.Now, rpcExecutionLedgerCapacity{})
}
func TestRPCExecutionReceiptBudgetCoversOwnedFixedStructures(t *testing.T) {
fixed := unsafe.Sizeof(rpcExecutionReceipt{}) +
unsafe.Sizeof(list.Element{}) +
unsafe.Sizeof(rpcExecutionBudgetReservation{})
if fixed > rpcExecutionReceiptBudgetBytes {
t.Fatalf("fixed receipt structures use %d bytes, budget charge is %d", fixed, rpcExecutionReceiptBudgetBytes)
}
}

190
internal/mtprotoedge/rpc_flight.go
View file

@ -23,7 +23,7 @@ var (
)
// rpcResultIdentityMismatchError carries the winner's immutable admission
// profile from inside the cache shard critical section. A replacement Conn can
// profile from inside the ledger shard critical section. A replacement Conn can
// re-decode the same naked body under that grammar even if the winner aborts
// immediately after the mismatch is returned.
type rpcResultIdentityMismatchError struct {
@ -51,8 +51,8 @@ type rpcResultRequestIdentity struct {
func (i rpcResultRequestIdentity) matches(requested rpcResultRequestIdentity) bool {
if !requested.valid {
// Legacy service/test callers carry no API request identity and preserve
// the historical cache lookup behavior. Exact callers must always match.
// Service-message callers carry no API request identity. Exact API callers
// must always match the winner's full prepared identity.
return true
}
return i.valid && i.exact == requested.exact
@ -72,10 +72,10 @@ const (
//
// Exactly one state-specific field is non-nil:
// - completed: encoded contains the immutable completed rpc_result;
// - acknowledged: the client ACKed the completed rpc_result, so only the
// compact request receipt remains and the duplicate must be ACK-only;
// - acknowledged: ACK won while the owner was still pending, so the duplicate
// must be ACK-only until that owner completes;
// - pending: waiter joins the already-running owner;
// - owner: owner must eventually complete through rpcResultCache.Put or Abort.
// - owner: owner must eventually complete through ledger Complete or Abort.
type rpcResultAcquire struct {
state rpcResultAcquireState
admissionSeq uint64
@ -87,7 +87,7 @@ type rpcResultAcquire struct {
}
// rpcResultFlight is not part of the completed receipt TTL lifecycle. Its
// done channel is closed exactly once while holding the owning cache shard lock;
// done channel is closed exactly once while holding the owning ledger shard lock;
// channel close publishes encoded/ok to all waiters without a waiter goroutine.
type rpcResultFlight struct {
done chan struct{}
@ -98,9 +98,9 @@ type rpcResultFlight struct {
executionOK bool
executionSubscribers []func(bool)
// acknowledged is set only from the sole outbound actor's server-msg-id to
// request-msg-id mapping. It can win the race with the asynchronous Put;
// publication then keeps only the compact receipt while still resolving
// waiters with the immutable result that was already written on the wire.
// request-msg-id mapping. It can win the race with asynchronous completion;
// publication then drops the receipt while still resolving waiters with the
// immutable result that was already written on the wire.
acknowledged bool
// subscriberSlots counts callbacks retained by this pending flight. Result
// and execution callbacks are charged independently; a replay alias installs
@ -108,18 +108,18 @@ type rpcResultFlight struct {
subscriberSlots int
identity rpcResultRequestIdentity
admissionSeq uint64
// reservation owns one entry and at least one byte at global, raw-auth and
// session scopes. Put transfers it to a result/tombstone; Abort releases it.
reservation *rpcResultBudgetReservation
// reservation owns one entry at global, raw-auth and session scopes.
// Complete transfers it to a receipt/tombstone; Abort releases it.
reservation *rpcExecutionBudgetReservation
}
type rpcResultWaiter struct {
cache *rpcResultCache
key rpcResultCacheKey
ledger *rpcExecutionLedger
key rpcExecutionKey
flight *rpcResultFlight
}
// Wait blocks until the owner publishes through Put, aborts, or ctx expires.
// Wait blocks until the owner publishes through Complete, aborts, or ctx expires.
// ok=false with err=nil means the owner aborted without a result.
func (w *rpcResultWaiter) Wait(ctx context.Context) (encoded *encodedOutboundMessage, ok bool, err error) {
if w == nil || w.flight == nil || ctx == nil {
@ -148,7 +148,7 @@ func (w *rpcResultWaiter) Wait(ctx context.Context) (encoded *encodedOutboundMes
}
// Subscribe registers an event callback without creating a goroutine or
// occupying an RPC worker. The callback is invoked after the cache shard lock is
// occupying an RPC worker. The callback is invoked after the ledger shard lock is
// released; it must remain non-blocking.
func (w *rpcResultWaiter) Subscribe(fn func(*encodedOutboundMessage, bool)) error {
if fn == nil {
@ -186,10 +186,10 @@ func (w *rpcResultWaiter) subscribe(
resultFn func(*encodedOutboundMessage, bool),
executionFn func(bool),
) error {
if w == nil || w.cache == nil || w.flight == nil || (resultFn == nil && executionFn == nil) {
if w == nil || w.ledger == nil || w.flight == nil || (resultFn == nil && executionFn == nil) {
return ErrRPCResultFlightInvalid
}
s := w.cache.shard(w.key)
s := w.ledger.shard(w.key)
var (
encoded *encodedOutboundMessage
resultOK bool
@ -207,8 +207,8 @@ func (w *rpcResultWaiter) subscribe(
slots++
}
if slots > 0 {
if flight.subscriberSlots > w.cache.subscriberPerFlight-slots ||
!w.cache.subscriberBudget.reserve(w.key, slots) {
if flight.subscriberSlots > w.ledger.subscriberPerFlight-slots ||
!w.ledger.subscriberBudget.reserve(w.key, slots) {
s.mu.Unlock()
return ErrRPCResultSubscriberCapacity
}
@ -256,8 +256,8 @@ func (w *rpcResultWaiter) subscribe(
}
type rpcResultOwnerLease struct {
cache *rpcResultCache
key rpcResultCacheKey
ledger *rpcExecutionLedger
key rpcExecutionKey
flight *rpcResultFlight
delivery *rpcResultDelivery
hookMu sync.Mutex
@ -278,13 +278,13 @@ func (l *rpcResultOwnerLease) SetAbortHook(fn func()) {
}
// InstallAbortHook installs fn only while this lease still owns the pending
// flight. The shard lock linearizes installation with Abort/Put so a registry
// flight. The shard lock linearizes installation with Abort/Complete so a registry
// cannot publish a candidate after its owner has already disappeared.
func (l *rpcResultOwnerLease) InstallAbortHook(fn func()) bool {
if l == nil || l.cache == nil || l.flight == nil || fn == nil {
if l == nil || l.ledger == nil || l.flight == nil || fn == nil {
return false
}
s := l.cache.shard(l.key)
s := l.ledger.shard(l.key)
s.mu.Lock()
flight, ok := s.pending[l.key]
if !ok || flight != l.flight {
@ -299,10 +299,10 @@ func (l *rpcResultOwnerLease) InstallAbortHook(fn func()) bool {
}
func (l *rpcResultOwnerLease) Waiter() *rpcResultWaiter {
if l == nil || l.cache == nil || l.flight == nil {
if l == nil || l.ledger == nil || l.flight == nil {
return nil
}
return &rpcResultWaiter{cache: l.cache, key: l.key, flight: l.flight}
return &rpcResultWaiter{ledger: l.ledger, key: l.key, flight: l.flight}
}
func (l *rpcResultOwnerLease) TryRetarget(reqMsgID int64) bool {
@ -318,13 +318,13 @@ func (l *rpcResultOwnerLease) Delivery() *rpcResultDelivery {
// CompleteExecution publishes the handler outcome exactly once while this
// lease still owns the flight. success=false includes RPC errors, internal
// failures and dependency failures. Delivery/cache completion remains a
// failures and dependency failures. Delivery/ledger completion remains a
// separate later transition.
func (l *rpcResultOwnerLease) CompleteExecution(success bool) bool {
if l == nil || l.cache == nil || l.flight == nil {
if l == nil || l.ledger == nil || l.flight == nil {
return false
}
s := l.cache.shard(l.key)
s := l.ledger.shard(l.key)
s.mu.Lock()
flight, ok := s.pending[l.key]
if !ok || flight != l.flight || flight.executionDone {
@ -335,7 +335,7 @@ func (l *rpcResultOwnerLease) CompleteExecution(success bool) bool {
flight.executionOK = success
subscribers := append([]func(bool){}, flight.executionSubscribers...)
flight.executionSubscribers = nil
l.cache.releaseFlightSubscriberSlotsLocked(l.key, flight, len(subscribers))
l.ledger.releaseFlightSubscriberSlotsLocked(l.key, flight, len(subscribers))
s.mu.Unlock()
for _, subscriber := range subscribers {
subscriber(success)
@ -345,12 +345,12 @@ func (l *rpcResultOwnerLease) CompleteExecution(success bool) bool {
// HandOff transfers completion responsibility from the inbound RPC task to an
// already-admitted egress operation. The egress terminal callback must resolve
// the flight through Put on both successful delivery and fenced failure.
// the flight through Complete on both successful delivery and fenced failure.
func (l *rpcResultOwnerLease) HandOff() bool {
if l == nil || l.cache == nil || l.flight == nil {
if l == nil || l.ledger == nil || l.flight == nil {
return false
}
s := l.cache.shard(l.key)
s := l.ledger.shard(l.key)
s.mu.Lock()
defer s.mu.Unlock()
flight, ok := s.pending[l.key]
@ -365,13 +365,13 @@ func (l *rpcResultOwnerLease) HandOff() bool {
// result. Pointer identity prevents an old lease from deleting a later owner
// that reacquired the same key. It returns true only for the winning abort.
func (l *rpcResultOwnerLease) Abort() bool {
if l == nil || l.cache == nil || l.flight == nil {
if l == nil || l.ledger == nil || l.flight == nil {
return false
}
if l.handedOff.Load() {
return false
}
s := l.cache.shard(l.key)
s := l.ledger.shard(l.key)
s.mu.Lock()
if l.handedOff.Load() {
s.mu.Unlock()
@ -388,13 +388,13 @@ func (l *rpcResultOwnerLease) Abort() bool {
flight.reservation.release()
flight.reservation = nil
}
l.cache.flightLimit.release()
l.cache.activeAdmissions.retire(flight.admissionSeq)
l.ledger.flightLimit.release()
l.ledger.activeAdmissions.retire(flight.admissionSeq)
subscribers := append([]func(*encodedOutboundMessage, bool){}, flight.subscribers...)
flight.subscribers = nil
executionSubscribers := append([]func(bool){}, flight.executionSubscribers...)
flight.executionSubscribers = nil
l.cache.releaseFlightSubscriberSlotsLocked(
l.ledger.releaseFlightSubscriberSlotsLocked(
l.key, flight, len(subscribers)+len(executionSubscribers),
)
if !flight.executionDone {
@ -458,7 +458,7 @@ func (l *rpcResultFlightLimit) releaseN(delta int64) {
panic("mtproto rpc result counter release must be positive")
}
if remaining := l.used.Add(-delta); remaining < 0 {
// Put/Abort use map removal and lease identity to make double release
// Complete/Abort use map removal and lease identity to make double release
// impossible. Fail fast instead of masking a capacity-accounting bug that
// could otherwise admit more owners than the configured hard limit.
panic("mtproto rpc result in-flight counter underflow")
@ -475,29 +475,29 @@ func (l *rpcResultFlightLimit) snapshot() int64 {
// Acquire atomically returns a completed result, joins the existing in-flight
// owner, or installs the unique owner lease. Pending entries have a separate
// lifecycle from completed receipt TTL but reserve the same global/auth/session
// ownership that Put later transfers to a completed result or tombstone.
func (c *rpcResultCache) Acquire(authKeyID [8]byte, sessionID, reqMsgID int64) (rpcResultAcquire, error) {
return c.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{})
// ownership that Complete later transfers to a receipt or tombstone.
func (l *rpcExecutionLedger) Acquire(authKeyID [8]byte, sessionID, reqMsgID int64) (rpcResultAcquire, error) {
return l.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{})
}
func (c *rpcResultCache) AcquireIdentified(
func (l *rpcExecutionLedger) AcquireIdentified(
authKeyID [8]byte,
sessionID, reqMsgID int64,
identity tlprofile.PreparedIdentity,
) (rpcResultAcquire, error) {
return c.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{exact: identity, valid: true})
return l.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{exact: identity, valid: true})
}
// AcquireLayerIdentified is the production exact-RPC claim. In addition to the
// immutable full request identity it retains the admission profile required to
// decode a later same-msg_id naked replay under its original grammar.
func (c *rpcResultCache) AcquireLayerIdentified(
func (l *rpcExecutionLedger) AcquireLayerIdentified(
authKeyID [8]byte,
sessionID, reqMsgID int64,
profile tlprofile.Profile,
identity tlprofile.PreparedIdentity,
) (rpcResultAcquire, error) {
return c.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{
return l.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{
exact: identity, profile: profile, valid: true,
})
}
@ -506,17 +506,17 @@ func (c *rpcResultCache) AcquireLayerIdentified(
// owner/result. It does not create or join a flight. Callers still perform
// AcquireLayerIdentified after decode, which atomically rejects a same-msg_id
// body change by comparing the full prepared identity.
func (c *rpcResultCache) ExactAdmissionProfile(authKeyID [8]byte, sessionID, reqMsgID int64) (tlprofile.Profile, bool) {
if c == nil || reqMsgID == 0 {
func (l *rpcExecutionLedger) ExactAdmissionProfile(authKeyID [8]byte, sessionID, reqMsgID int64) (tlprofile.Profile, bool) {
if l == nil || reqMsgID == 0 {
return 0, false
}
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s := c.shard(key)
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s := l.shard(key)
now := s.now()
s.mu.Lock()
defer s.mu.Unlock()
if elem := s.byKey[key]; elem != nil {
entry := elem.Value.(*rpcResultCacheEntry)
entry := elem.Value.(*rpcExecutionReceipt)
if entry.expiresAt.After(now) {
if entry.identity.valid && entry.identity.profile != 0 {
return entry.identity.profile, true
@ -531,16 +531,16 @@ func (c *rpcResultCache) ExactAdmissionProfile(authKeyID [8]byte, sessionID, req
return 0, false
}
func (c *rpcResultCache) acquire(
func (l *rpcExecutionLedger) acquire(
authKeyID [8]byte,
sessionID, reqMsgID int64,
identity rpcResultRequestIdentity,
) (rpcResultAcquire, error) {
if c == nil || reqMsgID == 0 {
if l == nil || reqMsgID == 0 {
return rpcResultAcquire{}, ErrRPCResultFlightInvalid
}
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s := c.shard(key)
key := rpcExecutionKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s := l.shard(key)
reclaimedExpired := false
for {
now := s.now()
@ -548,7 +548,7 @@ func (c *rpcResultCache) acquire(
s.expireLocked(now)
if elem, ok := s.byKey[key]; ok {
entry := elem.Value.(*rpcResultCacheEntry)
entry := elem.Value.(*rpcExecutionReceipt)
if !entry.identity.matches(identity) {
s.mu.Unlock()
return rpcResultAcquire{}, identityMismatch(entry.identity)
@ -561,32 +561,20 @@ func (c *rpcResultCache) acquire(
s.mu.Unlock()
return result, nil
}
if entry.capacity {
if entry.unavailable {
s.mu.Unlock()
return rpcResultAcquire{}, ErrRPCResultFlightCapacity
}
if c.sessions != nil {
result := rpcResultAcquire{
state: rpcResultAcquireCompleted, admissionSeq: entry.admissionSeq,
executionKnown: entry.executionKnown, executionOK: entry.executionOK,
}
s.mu.Unlock()
encoded, replayable := c.sessions.rpcResult(authKeyID, sessionID, reqMsgID)
if !replayable {
return rpcResultAcquire{}, ErrRPCResultReplayUnavailable
}
result.encoded = encoded
return result, nil
}
if entry.encoded == nil {
s.mu.Unlock()
return rpcResultAcquire{}, ErrRPCResultReplayUnavailable
}
result := rpcResultAcquire{
state: rpcResultAcquireCompleted, admissionSeq: entry.admissionSeq, encoded: entry.encoded,
state: rpcResultAcquireCompleted, admissionSeq: entry.admissionSeq,
executionKnown: entry.executionKnown, executionOK: entry.executionOK,
}
s.mu.Unlock()
encoded, replayable := l.replayStore.rpcResult(authKeyID, sessionID, reqMsgID)
if !replayable {
return rpcResultAcquire{}, ErrRPCResultReplayUnavailable
}
result.encoded = encoded
return result, nil
}
if flight, ok := s.pending[key]; ok {
@ -605,22 +593,18 @@ func (c *rpcResultCache) acquire(
result := rpcResultAcquire{
state: rpcResultAcquirePending,
admissionSeq: flight.admissionSeq,
waiter: &rpcResultWaiter{cache: c, key: key, flight: flight},
waiter: &rpcResultWaiter{ledger: l, key: key, flight: flight},
}
s.mu.Unlock()
return result, nil
}
if s.maxEntries > 0 && len(s.byKey)+len(s.pending) >= s.maxEntries {
if !l.flightLimit.reserve() {
s.mu.Unlock()
return rpcResultAcquire{}, ErrRPCResultFlightCapacity
}
if !c.flightLimit.reserve() {
s.mu.Unlock()
return rpcResultAcquire{}, ErrRPCResultFlightCapacity
}
reservation := c.fairBudget.reserveOwner(key)
reservation := l.fairBudget.reserveOwner(key)
if reservation == nil {
c.flightLimit.release()
l.flightLimit.release()
s.mu.Unlock()
if reclaimedExpired {
return rpcResultAcquire{}, ErrRPCResultFlightCapacity
@ -628,17 +612,17 @@ func (c *rpcResultCache) acquire(
// Expired rows in another full-key shard may be the only consumers at
// the global, auth or session scope. Reap once, then retry every identity
// and capacity check because another goroutine may have won this key.
c.expireCompletedResults()
l.expireReceipts()
reclaimedExpired = true
continue
}
var admissionSeq uint64
if identity.valid {
var err error
admissionSeq, err = c.activeAdmissions.allocateAndRegister(&c.nextAdmissionSeq)
admissionSeq, err = l.activeAdmissions.allocateAndRegister(&l.nextAdmissionSeq)
if err != nil {
reservation.release()
c.flightLimit.release()
l.flightLimit.release()
s.mu.Unlock()
return rpcResultAcquire{}, err
}
@ -648,7 +632,7 @@ func (c *rpcResultCache) acquire(
reservation: reservation,
}
if s.pending == nil {
s.pending = make(map[rpcResultCacheKey]*rpcResultFlight)
s.pending = make(map[rpcExecutionKey]*rpcResultFlight)
}
s.pending[key] = flight
s.mu.Unlock()
@ -656,7 +640,7 @@ func (c *rpcResultCache) acquire(
state: rpcResultAcquireOwner,
admissionSeq: admissionSeq,
owner: &rpcResultOwnerLease{
cache: c, key: key, flight: flight, delivery: newRPCResultDelivery(reqMsgID),
ledger: l, key: key, flight: flight, delivery: newRPCResultDelivery(reqMsgID),
},
}, nil
}
@ -664,16 +648,16 @@ func (c *rpcResultCache) acquire(
// completeRPCResultFlightLocked publishes encoded to the current owner claim.
// The caller must hold s.mu and must publish the completed receipt first.
func (c *rpcResultCache) completeRPCResultFlightLocked(
s *rpcResultCacheShard,
key rpcResultCacheKey,
func (l *rpcExecutionLedger) completeRPCResultFlightLocked(
s *rpcExecutionLedgerShard,
key rpcExecutionKey,
encoded *encodedOutboundMessage,
) (
[]func(*encodedOutboundMessage, bool),
[]func(bool),
bool,
) {
if c == nil || s == nil || encoded == nil {
if l == nil || s == nil || encoded == nil {
return nil, nil, false
}
flight, ok := s.pending[key]
@ -688,8 +672,8 @@ func (c *rpcResultCache) completeRPCResultFlightLocked(
}
flight.encoded = encoded
flight.ok = true
c.flightLimit.release()
c.activeAdmissions.retire(flight.admissionSeq)
l.flightLimit.release()
l.activeAdmissions.retire(flight.admissionSeq)
subscribers := append([]func(*encodedOutboundMessage, bool){}, flight.subscribers...)
flight.subscribers = nil
executionSubscribers := append([]func(bool){}, flight.executionSubscribers...)
@ -697,13 +681,13 @@ func (c *rpcResultCache) completeRPCResultFlightLocked(
executionOK := flight.executionOK
if !flight.executionDone {
// A result without an explicit handler-completion proof cannot satisfy an
// invokeAfter dependency. Production completes execution before Put; this
// invokeAfter dependency. Production completes execution before Complete;
// branch is the conservative terminal cleanup for defensive callers.
flight.executionDone = true
flight.executionOK = false
executionOK = false
}
c.releaseFlightSubscriberSlotsLocked(
l.releaseFlightSubscriberSlotsLocked(
key, flight, len(subscribers)+len(executionSubscribers),
)
close(flight.done)
@ -711,19 +695,19 @@ func (c *rpcResultCache) completeRPCResultFlightLocked(
}
// releaseFlightSubscriberSlotsLocked releases callbacks detached from a flight.
// The caller holds that flight's cache shard lock, preserving the only lock
// The caller holds that flight's ledger shard lock, preserving the only lock
// order used by subscription: shard -> subscriber budget.
func (c *rpcResultCache) releaseFlightSubscriberSlotsLocked(
key rpcResultCacheKey,
func (l *rpcExecutionLedger) releaseFlightSubscriberSlotsLocked(
key rpcExecutionKey,
flight *rpcResultFlight,
slots int,
) {
if slots == 0 {
return
}
if c == nil || flight == nil || slots < 0 || flight.subscriberSlots < slots {
if l == nil || flight == nil || slots < 0 || flight.subscriberSlots < slots {
panic("mtproto rpc result subscriber slot underflow")
}
flight.subscriberSlots -= slots
c.subscriberBudget.release(key, slots)
l.subscriberBudget.release(key, slots)
}

121
internal/mtprotoedge/rpc_flight_test.go
View file

@ -32,25 +32,23 @@ func rpcFlightExactIdentity(t *testing.T, profile tlprofile.Profile, request bin
return admitted.Prepared().Identity()
}
func newRPCResultSubscriberTestCache(global, auth, session, perFlight int) *rpcResultCache {
return newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
func newRPCExecutionSubscriberTestLedger(global, auth, session, perFlight int) *rpcExecutionLedger {
return newRPCExecutionLedger(time.Now, rpcExecutionLedgerCapacity{
maxPending: 64,
maxPendingPerAuth: 64,
globalMaxBytes: rpcResultCacheMaxBytes,
globalMaxEntries: rpcResultCacheMaxEntries,
authMaxBytes: rpcResultCacheAuthMaxBytes,
authMaxEntries: rpcResultCacheAuthMaxEntries,
sessionMaxBytes: rpcResultCacheSessionMaxBytes,
sessionMaxEntries: rpcResultCacheSessionMaxEntries,
globalMaxEntries: rpcExecutionMaxEntries,
authMaxEntries: rpcExecutionAuthMaxEntries,
sessionMaxEntries: rpcExecutionSessionMaxEntries,
subscriberMaxGlobal: global,
subscriberMaxAuth: auth,
subscriberMaxSession: session,
subscriberMaxPerFlight: perFlight,
replayStore: newRPCReplayStoreForTest(),
})
}
func TestRPCResultFlightSubscriberPairCapacityFailureIsAtomic(t *testing.T) {
cache := newRPCResultSubscriberTestCache(8, 8, 8, 1)
cache := newRPCExecutionSubscriberTestLedger(8, 8, 8, 1)
authKeyID := rpcFlightTestAuthID(70)
claim, err := cache.Acquire(authKeyID, 70, 700)
if err != nil || claim.owner == nil {
@ -64,7 +62,7 @@ func TestRPCResultFlightSubscriberPairCapacityFailureIsAtomic(t *testing.T) {
if !errors.Is(err, ErrRPCResultSubscriberCapacity) {
t.Fatalf("pair subscription err=%v, want %v", err, ErrRPCResultSubscriberCapacity)
}
s := cache.shard(rpcResultCacheKey{authKeyID: authKeyID, sessionID: 70, reqMsgID: 700})
s := cache.shard(rpcExecutionKey{authKeyID: authKeyID, sessionID: 70, reqMsgID: 700})
s.mu.Lock()
if got := claim.owner.flight.subscriberSlots; got != 0 {
t.Fatalf("failed pair retained %d subscriber slots", got)
@ -74,7 +72,7 @@ func TestRPCResultFlightSubscriberPairCapacityFailureIsAtomic(t *testing.T) {
}
s.mu.Unlock()
claim.owner.CompleteExecution(true)
cache.Put(authKeyID, 70, 700, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 700})
cache.completeReplayableForTest(authKeyID, 70, 700, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 700})
if resultCalls != 0 || executionCalls != 0 {
t.Fatalf("failed pair callbacks ran: result=%d execution=%d", resultCalls, executionCalls)
}
@ -84,7 +82,7 @@ func TestRPCResultFlightSubscriberPairCapacityFailureIsAtomic(t *testing.T) {
}
func TestRPCResultFlightSubscriberBudgetsIsolateSessionAndAuth(t *testing.T) {
cache := newRPCResultSubscriberTestCache(3, 2, 1, 4)
cache := newRPCExecutionSubscriberTestLedger(3, 2, 1, 4)
authA := rpcFlightTestAuthID(71)
authB := rpcFlightTestAuthID(72)
type ownerKey struct {
@ -149,7 +147,7 @@ func TestRPCResultFlightSubscriberBudgetsIsolateSessionAndAuth(t *testing.T) {
}
func TestRPCResultFlightSubscriberSlotsReleasePerTerminalHalf(t *testing.T) {
cache := newRPCResultSubscriberTestCache(4, 4, 4, 4)
cache := newRPCExecutionSubscriberTestLedger(4, 4, 4, 4)
authKeyID := rpcFlightTestAuthID(73)
claim, err := cache.Acquire(authKeyID, 73, 730)
if err != nil || claim.owner == nil {
@ -175,7 +173,7 @@ func TestRPCResultFlightSubscriberSlotsReleasePerTerminalHalf(t *testing.T) {
if got := cache.subscriberBudget.sessionSnapshot(authKeyID, 73); got != 1 {
t.Fatalf("post-execution subscriber usage=%d, want 1", got)
}
cache.Put(authKeyID, 73, 730, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 730})
cache.completeReplayableForTest(authKeyID, 73, 730, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 730})
if ok := <-result; !ok {
t.Fatal("result callback reported failure")
}
@ -185,7 +183,7 @@ func TestRPCResultFlightSubscriberSlotsReleasePerTerminalHalf(t *testing.T) {
}
func TestRPCResultFlightRepeatedReplayJoinsStayBoundedAndPutCleansExecution(t *testing.T) {
cache := newRPCResultSubscriberTestCache(2, 2, 2, 2)
cache := newRPCExecutionSubscriberTestLedger(2, 2, 2, 2)
authKeyID := rpcFlightTestAuthID(74)
claim, err := cache.Acquire(authKeyID, 74, 740)
if err != nil || claim.owner == nil {
@ -212,7 +210,7 @@ func TestRPCResultFlightRepeatedReplayJoinsStayBoundedAndPutCleansExecution(t *t
t.Fatalf("join %d err=%v, want capacity", i, err)
}
}
cache.Put(authKeyID, 74, 740, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 740})
cache.completeReplayableForTest(authKeyID, 74, 740, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 740})
if resultCalls != 1 || executionCalls != 1 {
t.Fatalf("terminal callback counts result=%d execution=%d", resultCalls, executionCalls)
}
@ -222,7 +220,7 @@ func TestRPCResultFlightRepeatedReplayJoinsStayBoundedAndPutCleansExecution(t *t
}
func TestRPCResultFlightExactIdentityGuardsPendingAndCompletedReuse(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
cache := newRPCExecutionLedgerForTest(time.Now, 2)
authKeyID := rpcFlightTestAuthID(90)
firstIdentity := rpcFlightExactIdentity(t, tlprofile.Profile225, &tg.HelpGetConfigRequest{})
otherIdentity := rpcFlightExactIdentity(t, tlprofile.Profile225, &tg.HelpGetNearestDCRequest{})
@ -240,7 +238,7 @@ func TestRPCResultFlightExactIdentityGuardsPendingAndCompletedReuse(t *testing.T
}
want := &encodedOutboundMessage{body: []byte{1, 2, 3, 4}, reqMsgID: 900}
cache.Put(authKeyID, 90, 900, want)
cache.completeReplayableForTest(authKeyID, 90, 900, want)
if _, err := cache.AcquireIdentified(authKeyID, 90, 900, otherIdentity); !errors.Is(err, ErrRPCResultIdentityMismatch) {
t.Fatalf("completed mismatched Acquire err = %v, want %v", err, ErrRPCResultIdentityMismatch)
}
@ -254,7 +252,7 @@ func TestRPCResultFlightExactIdentityGuardsPendingAndCompletedReuse(t *testing.T
}
func TestRPCResultFlightAdmissionSequenceAllocatedOnceAndReplayed(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 4)
cache := newRPCExecutionLedgerForTest(time.Now, 4)
authKeyID := rpcFlightTestAuthID(89)
identity := rpcFlightExactIdentity(t, tlprofile.Profile225, &tg.HelpGetConfigRequest{})
owner, err := cache.AcquireLayerIdentified(authKeyID, 89, 890, tlprofile.Profile225, identity)
@ -267,7 +265,7 @@ func TestRPCResultFlightAdmissionSequenceAllocatedOnceAndReplayed(t *testing.T)
}
owner.owner.CompleteExecution(true)
encoded := &encodedOutboundMessage{body: []byte{1}, reqMsgID: 890}
cache.Put(authKeyID, 89, 890, encoded)
cache.completeReplayableForTest(authKeyID, 89, 890, encoded)
completed, err := cache.AcquireLayerIdentified(authKeyID, 89, 890, tlprofile.Profile225, identity)
if err != nil || completed.state != rpcResultAcquireCompleted || completed.admissionSeq != owner.admissionSeq {
t.Fatalf("completed = state:%d seq:%d err:%v, want seq:%d", completed.state, completed.admissionSeq, err, owner.admissionSeq)
@ -285,7 +283,7 @@ func TestRPCResultFlightAdmissionSequenceAllocatedOnceAndReplayed(t *testing.T)
}
func TestRPCAdmissionSafeFloorTracksOwnersUntilPutOrAbort(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 4)
cache := newRPCExecutionLedgerForTest(time.Now, 4)
authKeyID := rpcFlightTestAuthID(86)
identity := rpcFlightExactIdentity(t, tlprofile.Profile225, &tg.HelpGetConfigRequest{})
first, err := cache.AcquireLayerIdentified(authKeyID, 86, 860, tlprofile.Profile225, identity)
@ -306,14 +304,14 @@ func TestRPCAdmissionSafeFloorTracksOwnersUntilPutOrAbort(t *testing.T) {
t.Fatalf("post-abort safe floor=%d, want %d", floor, second.admissionSeq)
}
second.owner.CompleteExecution(true)
cache.Put(authKeyID, 86, 864, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 864})
cache.completeReplayableForTest(authKeyID, 86, 864, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 864})
if floor := cache.stableAdmissionSafeFloor(); floor != second.admissionSeq+1 {
t.Fatalf("terminal safe floor=%d, want %d", floor, second.admissionSeq+1)
}
}
func TestRPCAdmissionSequenceExhaustionCannotWrap(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
cache := newRPCExecutionLedgerForTest(time.Now, 2)
cache.nextAdmissionSeq.Store(^uint64(0) - 1)
authKeyID := rpcFlightTestAuthID(85)
identity := rpcFlightExactIdentity(t, tlprofile.Profile225, &tg.HelpGetConfigRequest{})
@ -331,7 +329,7 @@ func TestRPCAdmissionSequenceExhaustionCannotWrap(t *testing.T) {
}
func TestRPCIdentityMismatchCarriesWinnerProfileAcrossAbort(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
cache := newRPCExecutionLedgerForTest(time.Now, 2)
authKeyID := rpcFlightTestAuthID(88)
request := &tg.MessagesGetHistoryRequest{Peer: &tg.InputPeerSelf{}, Limit: 1}
winnerIdentity := rpcFlightExactIdentity(t, tlprofile.Profile225, request)
@ -357,7 +355,7 @@ func TestRPCIdentityMismatchCarriesWinnerProfileAcrossAbort(t *testing.T) {
func TestRPCAdmissionProfileHintSurvivesCompletedEvictionWindow(t *testing.T) {
now := time.Unix(1_900_000_000, 0)
cache := newRPCResultCacheWithFlightLimit(func() time.Time { return now }, 2)
cache := newRPCExecutionLedgerForTest(func() time.Time { return now }, 2)
authKeyID := rpcFlightTestAuthID(87)
identity := rpcFlightExactIdentity(t, tlprofile.Profile225, &tg.MessagesGetHistoryRequest{
Peer: &tg.InputPeerSelf{}, Limit: 1,
@ -367,7 +365,7 @@ func TestRPCAdmissionProfileHintSurvivesCompletedEvictionWindow(t *testing.T) {
t.Fatalf("owner err=%v", err)
}
claim.owner.CompleteExecution(true)
cache.Put(authKeyID, 87, 870, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 870})
cache.completeReplayableForTest(authKeyID, 87, 870, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 870})
profile, ok := cache.ExactAdmissionProfile(authKeyID, 87, 870)
if !ok || profile != tlprofile.Profile225 {
t.Fatalf("profile hint = (%d,%v)", profile, ok)
@ -375,7 +373,7 @@ func TestRPCAdmissionProfileHintSurvivesCompletedEvictionWindow(t *testing.T) {
// Admission already copied the hint into its local decoder cursor. Expiry
// between that probe and the atomic claim must not make it fall back to the
// connection's newer default; it simply becomes a fresh owner under 225.
now = now.Add(rpcResultCacheTTL + time.Second)
now = now.Add(rpcExecutionReceiptTTL + time.Second)
replacement, err := cache.AcquireLayerIdentified(authKeyID, 87, 870, profile, identity)
if err != nil || replacement.state != rpcResultAcquireOwner || replacement.owner == nil {
t.Fatalf("post-eviction owner = state:%d err:%v", replacement.state, err)
@ -384,7 +382,7 @@ func TestRPCAdmissionProfileHintSurvivesCompletedEvictionWindow(t *testing.T) {
}
func TestRPCInvariantIdentityDoesNotExposeCanonicalProfileHint(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
cache := newRPCExecutionLedgerForTest(time.Now, 2)
authKeyID := rpcFlightTestAuthID(84)
identity := rpcFlightExactIdentity(t, tlprofile.Profile227, &tg.AuthBindTempAuthKeyRequest{
PermAuthKeyID: 1, Nonce: 2, ExpiresAt: 3, EncryptedMessage: []byte("bind"),
@ -397,14 +395,14 @@ func TestRPCInvariantIdentityDoesNotExposeCanonicalProfileHint(t *testing.T) {
t.Fatalf("pending invariant profile hint=(%d,%v), want absent", profile, ok)
}
claim.owner.CompleteExecution(true)
cache.Put(authKeyID, 84, 840, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 840})
cache.completeReplayableForTest(authKeyID, 84, 840, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 840})
if profile, ok := cache.ExactAdmissionProfile(authKeyID, 84, 840); ok || profile != 0 {
t.Fatalf("completed invariant profile hint=(%d,%v), want absent", profile, ok)
}
}
func TestRPCResultExecutionCompletionIsExactlyOnceAndDurable(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
cache := newRPCExecutionLedgerForTest(time.Now, 2)
authKeyID := rpcFlightTestAuthID(91)
claim, err := cache.Acquire(authKeyID, 91, 910)
if err != nil || claim.state != rpcResultAcquireOwner || claim.owner == nil {
@ -436,7 +434,7 @@ func TestRPCResultExecutionCompletionIsExactlyOnceAndDurable(t *testing.T) {
}
want := &encodedOutboundMessage{body: []byte{9, 1, 0, 0}, reqMsgID: 910}
cache.Put(authKeyID, 91, 910, want)
cache.completeReplayableForTest(authKeyID, 91, 910, want)
dependency, ok := cache.ObserveDependency(authKeyID, 91, 910)
if !ok || !dependency.completed || !dependency.success || dependency.waiter != nil {
t.Fatalf("completed dependency = %#v ok:%v", dependency, ok)
@ -451,7 +449,7 @@ func TestRPCResultExecutionCompletionIsExactlyOnceAndDurable(t *testing.T) {
}
func TestRPCResultExecutionAbortPublishesFailure(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
cache := newRPCExecutionLedgerForTest(time.Now, 1)
authKeyID := rpcFlightTestAuthID(92)
claim, err := cache.Acquire(authKeyID, 92, 920)
if err != nil || claim.owner == nil {
@ -474,7 +472,7 @@ func TestRPCResultExecutionAbortPublishesFailure(t *testing.T) {
func TestRPCResultFlightConcurrentAcquireHasUniqueOwner(t *testing.T) {
const callers = 64
cache := newRPCResultCacheWithFlightLimit(time.Now, callers)
cache := newRPCExecutionLedgerForTest(time.Now, callers)
authKeyID := rpcFlightTestAuthID(1)
start := make(chan struct{})
results := make(chan rpcResultAcquire, callers)
@ -535,7 +533,7 @@ func TestRPCResultFlightConcurrentAcquireHasUniqueOwner(t *testing.T) {
func TestRPCResultFlightPutPublishesAndWakesAllWaiters(t *testing.T) {
const waiters = 24
cache := newRPCResultCacheWithFlightLimit(time.Now, 32)
cache := newRPCExecutionLedgerForTest(time.Now, 32)
authKeyID := rpcFlightTestAuthID(10)
owner, err := cache.Acquire(authKeyID, 20, 200)
if err != nil || owner.state != rpcResultAcquireOwner || owner.owner == nil {
@ -563,13 +561,13 @@ func TestRPCResultFlightPutPublishesAndWakesAllWaiters(t *testing.T) {
for _, waiter := range waiterClaims {
go func(w *rpcResultWaiter) {
encoded, ok, waitErr := w.Wait(ctx)
cached, _ := cache.Get(authKeyID, 20, 200)
cached, _ := cache.Replay(authKeyID, 20, 200)
results <- waiterResult{encoded: encoded, cached: cached, ok: ok, err: waitErr}
}(waiter)
}
want := &encodedOutboundMessage{body: []byte{1, 2, 3, 4}, typeID: 42, reqMsgID: 200}
cache.Put(authKeyID, 20, 200, want)
cache.completeReplayableForTest(authKeyID, 20, 200, want)
for i := 0; i < waiters; i++ {
got := <-results
if got.err != nil || !got.ok {
@ -592,7 +590,7 @@ func TestRPCResultFlightPutPublishesAndWakesAllWaiters(t *testing.T) {
}
func TestRPCResultFlightAbortWakesAndAllowsReclaim(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
cache := newRPCExecutionLedgerForTest(time.Now, 2)
authKeyID := rpcFlightTestAuthID(20)
first, err := cache.Acquire(authKeyID, 30, 300)
if err != nil || first.state != rpcResultAcquireOwner {
@ -627,22 +625,17 @@ func TestRPCResultFlightAbortWakesAndAllowsReclaim(t *testing.T) {
}
}
func TestRPCResultFlightCompletedCachePressureDoesNotEvictPending(t *testing.T) {
func TestRPCResultFlightReceiptLifecycleDoesNotEvictPending(t *testing.T) {
now := time.Unix(1_000, 0)
cache := newRPCResultCacheWithFlightLimit(func() time.Time { return now }, 4)
cache := newRPCExecutionLedgerForTest(func() time.Time { return now }, 4)
authKeyID := rpcFlightTestAuthID(30)
pending, err := cache.Acquire(authKeyID, 40, 400)
if err != nil || pending.state != rpcResultAcquireOwner {
t.Fatalf("pending Acquire = state:%d err:%v", pending.state, err)
}
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: 40, reqMsgID: 400}
shard := cache.shard(key)
shard.mu.Lock()
shard.maxEntries = 2
shard.mu.Unlock()
for i := int64(0); i < 16; i++ {
cache.Put(authKeyID, 40, 500+i, &encodedOutboundMessage{body: []byte{byte(i)}})
cache.completeReplayableForTest(authKeyID, 40, 500+i, &encodedOutboundMessage{body: []byte{byte(i)}})
}
if got := cache.flightLimit.snapshot(); got != 1 {
t.Fatalf("completed trim changed pending count to %d", got)
@ -652,9 +645,9 @@ func TestRPCResultFlightCompletedCachePressureDoesNotEvictPending(t *testing.T)
t.Fatalf("Acquire after completed trim = state:%d err:%v", joined.state, err)
}
// Expire the independent completed cache and prove the pending owner remains.
now = now.Add(rpcResultCacheTTL + time.Second)
_, _ = cache.Get(authKeyID, 40, 515)
// Expire independent completed receipts and prove the pending owner remains.
now = now.Add(rpcExecutionReceiptTTL + time.Second)
_, _ = cache.Replay(authKeyID, 40, 515)
joinedAfterTTL, err := cache.Acquire(authKeyID, 40, 400)
if err != nil || joinedAfterTTL.state != rpcResultAcquirePending {
t.Fatalf("Acquire after completed TTL = state:%d err:%v", joinedAfterTTL.state, err)
@ -665,7 +658,7 @@ func TestRPCResultFlightCompletedCachePressureDoesNotEvictPending(t *testing.T)
}
func TestRPCResultFlightCapacityAndCountReturn(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 2)
cache := newRPCExecutionLedgerForTest(time.Now, 2)
authKeyID := rpcFlightTestAuthID(40)
first, err := cache.Acquire(authKeyID, 50, 501)
if err != nil || first.state != rpcResultAcquireOwner {
@ -687,7 +680,7 @@ func TestRPCResultFlightCapacityAndCountReturn(t *testing.T) {
}
want := &encodedOutboundMessage{body: []byte{9}, reqMsgID: 501}
cache.Put(authKeyID, 50, 501, want)
cache.completeReplayableForTest(authKeyID, 50, 501, want)
if got := cache.flightLimit.snapshot(); got != 1 {
t.Fatalf("pending count after Put = %d, want 1", got)
}
@ -710,8 +703,8 @@ func TestRPCResultFlightCapacityAndCountReturn(t *testing.T) {
}
}
func TestRPCResultFlightLargePutPublishesCompletedBeforeResolvingWaiters(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
func TestRPCResultFlightLargeCompletionDoesNotChargeLedgerBodyBytes(t *testing.T) {
cache := newRPCExecutionLedgerForTest(time.Now, 1)
authKeyID := rpcFlightTestAuthID(50)
owner, err := cache.Acquire(authKeyID, 60, 600)
if err != nil || owner.state != rpcResultAcquireOwner {
@ -722,22 +715,22 @@ func TestRPCResultFlightLargePutPublishesCompletedBeforeResolvingWaiters(t *test
t.Fatalf("joined Acquire = state:%d err:%v", joined.state, err)
}
// This is larger than the removed 4 MiB per-shard partition but remains a
// legal outbound result and fits the global/auth/session fair byte budgets.
largeSize := rpcResultCacheMaxBytes/rpcResultCacheShards + 1
// Large file results remain owned by the replay store. The ledger retains
// only one fixed-shape receipt regardless of payload size.
largeSize := 8 << 20
want := &encodedOutboundMessage{body: make([]byte, largeSize), reqMsgID: 600}
cache.Put(authKeyID, 60, 600, want)
cache.completeReplayableForTest(authKeyID, 60, 600, want)
if encoded, ok, waitErr := joined.waiter.Wait(context.Background()); waitErr != nil || !ok || encoded != want {
t.Fatalf("large Wait = encoded:%p ok:%v err:%v", encoded, ok, waitErr)
}
if got, ok := cache.Get(authKeyID, 60, 600); !ok || got != want {
t.Fatalf("large completed Get = encoded:%p ok:%v", got, ok)
if got, ok := cache.Replay(authKeyID, 60, 600); !ok || got != want {
t.Fatalf("large completed Replay = encoded:%p ok:%v", got, ok)
}
if got := cache.flightLimit.snapshot(); got != 0 {
t.Fatalf("large Put leaked pending count %d", got)
}
if owner.owner.Abort() {
t.Fatal("large Put left its old owner abortable")
t.Fatal("large completion left its old owner abortable")
}
completed, err := cache.Acquire(authKeyID, 60, 600)
if err != nil || completed.state != rpcResultAcquireCompleted || completed.encoded != want {
@ -746,13 +739,13 @@ func TestRPCResultFlightLargePutPublishesCompletedBeforeResolvingWaiters(t *test
if completed.owner != nil {
t.Fatal("large completed result incorrectly returned a new owner")
}
if got := cache.completedBytes.snapshot(); got != int64(largeSize) {
t.Fatalf("completed byte budget = %d, want %d", got, largeSize)
if got := cache.receiptBudgetBytes(); got != rpcExecutionReceiptBudgetBytes {
t.Fatalf("receipt budget bytes = %d, want %d", got, rpcExecutionReceiptBudgetBytes)
}
}
func TestRPCResultFlightWaitContextDoesNotReleaseOwner(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
cache := newRPCExecutionLedgerForTest(time.Now, 1)
authKeyID := rpcFlightTestAuthID(60)
owner, err := cache.Acquire(authKeyID, 70, 700)
if err != nil {
@ -780,7 +773,7 @@ func TestRPCResultFlightConcurrentCapacityReturnsAllSlots(t *testing.T) {
limit = 32
callers = 512
)
cache := newRPCResultCacheWithFlightLimit(time.Now, limit)
cache := newRPCExecutionLedgerForTest(time.Now, limit)
authKeyID := rpcFlightTestAuthID(70)
start := make(chan struct{})
errs := make(chan error, callers)
@ -804,7 +797,7 @@ func TestRPCResultFlightConcurrentCapacityReturnsAllSlots(t *testing.T) {
return
}
if i%2 == 0 {
cache.Put(authKeyID, 80+int64(i%4), 1_000+int64(i), &encodedOutboundMessage{body: []byte{1}})
cache.completeReplayableForTest(authKeyID, 80+int64(i%4), 1_000+int64(i), &encodedOutboundMessage{body: []byte{1}})
} else if !claim.owner.Abort() {
errs <- errors.New("owner Abort lost")
}

292
internal/mtprotoedge/rpc_result_budget.go
View file

@ -1,292 +0,0 @@
package mtprotoedge
import (
"encoding/binary"
"hash/maphash"
"sync"
)
const rpcResultBudgetShards = 64
type rpcResultBudgetLimit struct {
entries int64
bytes int64
}
type rpcResultBudgetUsage struct {
entries int64
bytes int64
pending int64
}
type rpcResultSessionBudgetKey struct {
authKeyID [8]byte
sessionID int64
}
type rpcResultAuthBudgetShard struct {
mu sync.Mutex
usage map[[8]byte]rpcResultBudgetUsage
}
type rpcResultSessionBudgetShard struct {
mu sync.Mutex
usage map[rpcResultSessionBudgetKey]rpcResultBudgetUsage
}
// rpcResultFairBudget accounts one ownership reservation at all three scopes.
// A pending owner and its completed result are the same ownership: admission
// reserves one entry plus one byte, Put resizes that byte reservation and moves
// it to the completed row, while Abort/TTL return the whole reservation.
//
// Auth and session maps are striped independently. Every operation takes the
// auth stripe before the session stripe; global counters remain atomic. This
// keeps unrelated auth keys off one process-wide mutex while preserving hard
// limits at every hierarchy level.
type rpcResultFairBudget struct {
seed maphash.Seed
globalEntries *rpcResultFlightLimit
globalBytes *rpcResultCacheByteBudget
authLimit rpcResultBudgetLimit
sessionLimit rpcResultBudgetLimit
pendingPerAuth int64
authShards [rpcResultBudgetShards]rpcResultAuthBudgetShard
sessionShards [rpcResultBudgetShards]rpcResultSessionBudgetShard
}
type rpcResultBudgetReservation struct {
budget *rpcResultFairBudget
key rpcResultCacheKey
bytes int
pending bool
released bool
}
func newRPCResultFairBudget(
seed maphash.Seed,
globalEntries *rpcResultFlightLimit,
globalBytes *rpcResultCacheByteBudget,
authLimit rpcResultBudgetLimit,
sessionLimit rpcResultBudgetLimit,
pendingPerAuth int,
) *rpcResultFairBudget {
b := &rpcResultFairBudget{
seed: seed,
globalEntries: globalEntries,
globalBytes: globalBytes,
authLimit: authLimit,
sessionLimit: sessionLimit,
pendingPerAuth: int64(pendingPerAuth),
}
for i := range b.authShards {
b.authShards[i].usage = make(map[[8]byte]rpcResultBudgetUsage)
b.sessionShards[i].usage = make(map[rpcResultSessionBudgetKey]rpcResultBudgetUsage)
}
return b
}
func (b *rpcResultFairBudget) reserveOwner(key rpcResultCacheKey) *rpcResultBudgetReservation {
return b.reserve(key, 1, true)
}
func (b *rpcResultFairBudget) reserveCompleted(key rpcResultCacheKey, bytes int) *rpcResultBudgetReservation {
return b.reserve(key, bytes, false)
}
func (b *rpcResultFairBudget) reserve(key rpcResultCacheKey, bytes int, pending bool) *rpcResultBudgetReservation {
if b == nil || b.globalEntries == nil || b.globalBytes == nil || bytes < 1 {
return nil
}
authShard := b.authShard(key.authKeyID)
sessionKey := rpcResultSessionBudgetKey{authKeyID: key.authKeyID, sessionID: key.sessionID}
sessionShard := b.sessionShard(sessionKey)
authShard.mu.Lock()
sessionShard.mu.Lock()
authUsage := authShard.usage[key.authKeyID]
sessionUsage := sessionShard.usage[sessionKey]
bytes64 := int64(bytes)
canReserve := withinRPCResultBudget(authUsage.entries, 1, b.authLimit.entries) &&
withinRPCResultBudget(authUsage.bytes, bytes64, b.authLimit.bytes) &&
withinRPCResultBudget(sessionUsage.entries, 1, b.sessionLimit.entries) &&
withinRPCResultBudget(sessionUsage.bytes, bytes64, b.sessionLimit.bytes)
if pending {
canReserve = canReserve && withinRPCResultBudget(authUsage.pending, 1, b.pendingPerAuth)
}
if !canReserve || !b.globalEntries.reserve() {
sessionShard.mu.Unlock()
authShard.mu.Unlock()
return nil
}
if !b.globalBytes.reserve(bytes) {
b.globalEntries.release()
sessionShard.mu.Unlock()
authShard.mu.Unlock()
return nil
}
authUsage.entries++
authUsage.bytes += bytes64
sessionUsage.entries++
sessionUsage.bytes += bytes64
if pending {
authUsage.pending++
}
authShard.usage[key.authKeyID] = authUsage
sessionShard.usage[sessionKey] = sessionUsage
sessionShard.mu.Unlock()
authShard.mu.Unlock()
return &rpcResultBudgetReservation{budget: b, key: key, bytes: bytes, pending: pending}
}
func withinRPCResultBudget(used, delta, limit int64) bool {
return delta >= 0 && limit > 0 && used >= 0 && used <= limit-delta
}
func (r *rpcResultBudgetReservation) resizeBytes(bytes int) bool {
if r == nil || r.budget == nil || r.released || bytes < 1 {
return false
}
if bytes == r.bytes {
return true
}
b := r.budget
authShard := b.authShard(r.key.authKeyID)
sessionKey := rpcResultSessionBudgetKey{authKeyID: r.key.authKeyID, sessionID: r.key.sessionID}
sessionShard := b.sessionShard(sessionKey)
authShard.mu.Lock()
sessionShard.mu.Lock()
authUsage, authOK := authShard.usage[r.key.authKeyID]
sessionUsage, sessionOK := sessionShard.usage[sessionKey]
if !authOK || !sessionOK || authUsage.entries < 1 || sessionUsage.entries < 1 {
sessionShard.mu.Unlock()
authShard.mu.Unlock()
panic("mtprotoedge: rpc result budget reservation disappeared during resize")
}
delta := int64(bytes) - int64(r.bytes)
if delta > 0 {
if !withinRPCResultBudget(authUsage.bytes, delta, b.authLimit.bytes) ||
!withinRPCResultBudget(sessionUsage.bytes, delta, b.sessionLimit.bytes) ||
!b.globalBytes.reserve(int(delta)) {
sessionShard.mu.Unlock()
authShard.mu.Unlock()
return false
}
} else {
if authUsage.bytes < -delta || sessionUsage.bytes < -delta {
sessionShard.mu.Unlock()
authShard.mu.Unlock()
panic("mtprotoedge: rpc result byte reservation underflow during resize")
}
}
authUsage.bytes += delta
sessionUsage.bytes += delta
authShard.usage[r.key.authKeyID] = authUsage
sessionShard.usage[sessionKey] = sessionUsage
r.bytes = bytes
if delta < 0 {
b.globalBytes.release(int(-delta))
}
sessionShard.mu.Unlock()
authShard.mu.Unlock()
return true
}
func (r *rpcResultBudgetReservation) releasePending() {
if r == nil || r.budget == nil || r.released || !r.pending {
return
}
b := r.budget
authShard := b.authShard(r.key.authKeyID)
authShard.mu.Lock()
authUsage, ok := authShard.usage[r.key.authKeyID]
if !ok || authUsage.pending < 1 {
authShard.mu.Unlock()
panic("mtprotoedge: rpc result per-auth pending budget underflow")
}
authUsage.pending--
authShard.usage[r.key.authKeyID] = authUsage
r.pending = false
authShard.mu.Unlock()
}
func (r *rpcResultBudgetReservation) release() {
if r == nil || r.budget == nil || r.released {
return
}
b := r.budget
authShard := b.authShard(r.key.authKeyID)
sessionKey := rpcResultSessionBudgetKey{authKeyID: r.key.authKeyID, sessionID: r.key.sessionID}
sessionShard := b.sessionShard(sessionKey)
authShard.mu.Lock()
sessionShard.mu.Lock()
authUsage, authOK := authShard.usage[r.key.authKeyID]
sessionUsage, sessionOK := sessionShard.usage[sessionKey]
bytes64 := int64(r.bytes)
if !authOK || !sessionOK || authUsage.entries < 1 || sessionUsage.entries < 1 ||
authUsage.bytes < bytes64 || sessionUsage.bytes < bytes64 ||
(r.pending && authUsage.pending < 1) {
sessionShard.mu.Unlock()
authShard.mu.Unlock()
panic("mtprotoedge: rpc result fair budget underflow")
}
authUsage.entries--
authUsage.bytes -= bytes64
sessionUsage.entries--
sessionUsage.bytes -= bytes64
if r.pending {
authUsage.pending--
}
if authUsage == (rpcResultBudgetUsage{}) {
delete(authShard.usage, r.key.authKeyID)
} else {
authShard.usage[r.key.authKeyID] = authUsage
}
if sessionUsage == (rpcResultBudgetUsage{}) {
delete(sessionShard.usage, sessionKey)
} else {
sessionShard.usage[sessionKey] = sessionUsage
}
r.released = true
r.pending = false
r.bytes = 0
b.globalBytes.release(int(bytes64))
b.globalEntries.release()
sessionShard.mu.Unlock()
authShard.mu.Unlock()
}
func (b *rpcResultFairBudget) authSnapshot(authKeyID [8]byte) rpcResultBudgetUsage {
if b == nil {
return rpcResultBudgetUsage{}
}
shard := b.authShard(authKeyID)
shard.mu.Lock()
usage := shard.usage[authKeyID]
shard.mu.Unlock()
return usage
}
func (b *rpcResultFairBudget) sessionSnapshot(authKeyID [8]byte, sessionID int64) rpcResultBudgetUsage {
if b == nil {
return rpcResultBudgetUsage{}
}
key := rpcResultSessionBudgetKey{authKeyID: authKeyID, sessionID: sessionID}
shard := b.sessionShard(key)
shard.mu.Lock()
usage := shard.usage[key]
shard.mu.Unlock()
return usage
}
func (b *rpcResultFairBudget) authShard(authKeyID [8]byte) *rpcResultAuthBudgetShard {
index := maphash.Bytes(b.seed, authKeyID[:]) & (rpcResultBudgetShards - 1)
return &b.authShards[index]
}
func (b *rpcResultFairBudget) sessionShard(key rpcResultSessionBudgetKey) *rpcResultSessionBudgetShard {
var raw [16]byte
copy(raw[:8], key.authKeyID[:])
binary.LittleEndian.PutUint64(raw[8:], uint64(key.sessionID))
index := maphash.Bytes(b.seed, raw[:]) & (rpcResultBudgetShards - 1)
return &b.sessionShards[index]
}

670
internal/mtprotoedge/rpc_result_cache.go
View file

@ -1,670 +0,0 @@
package mtprotoedge
import (
"container/list"
"context"
"encoding/binary"
"hash/maphash"
"sync"
"sync/atomic"
"time"
)
const (
// Telegram accepts client msg_id values up to five minutes old and up to
// thirty seconds in the future. Retain the result across that complete
// replay horizon, plus one second for boundary/scheduler jitter, so a valid
// duplicate cannot rerun its handler merely because our cache expired first.
rpcResultCacheTTL = 331 * time.Second
// Completed results cover the complete replay horizon under explicit global,
// auth and session hard ceilings. At the default 331-second TTL, the 1<<18
// global entries permit about 792 unique RPC/s process-wide before bounded
// backpressure; lower scopes provide noisy-neighbor isolation.
rpcResultCacheMaxEntries = 1 << 18
rpcResultCacheMaxBytes = 64 << 20
rpcResultCacheAuthMaxEntries = 1 << 15
rpcResultCacheAuthMaxBytes = 32 << 20
rpcResultCacheSessionMaxEntries = 1 << 14
rpcResultCacheSessionMaxBytes = 16 << 20
rpcResultFlightMaxPendingPerAuth = 1 << 11
// rpcResultCacheShards hashes the complete replay identity with a random
// per-instance maphash seed. Including req_msg_id spreads one hot session's
// independent requests instead of forcing them through one mutex. The shard
// count is a power of two.
rpcResultCacheShards = 16
)
type rpcResultCacheKey struct {
authKeyID [8]byte
sessionID int64
reqMsgID int64
}
type rpcResultCacheEntry struct {
key rpcResultCacheKey
// encoded is used only by legacy focused constructors whose cache has no
// SessionManager. Production rows are metadata receipts and always leave it
// nil; exact bytes have one owner in the logical-session outbox.
encoded *encodedOutboundMessage
size int
expiresAt time.Time
identity rpcResultRequestIdentity
admissionSeq uint64
executionKnown bool
executionOK bool
// acknowledged exists only for the ACK-before-Put race. A completed row is
// removed immediately on ACK; unlike the previous design, success receipts
// are not retained for the rest of the 331-second horizon.
acknowledged bool
// capacity marks a bounded replay tombstone. The original owner and its
// already-joined waiters received encoded, but the byte budget could not
// retain that body. Keeping the immutable identity until TTL prevents a
// duplicate from rerunning business; Acquire returns a capacity error.
capacity bool
// reservation is the same global+auth+session ownership acquired before the
// handler ran. Put transfers it from the pending flight; TTL returns it.
reservation *rpcResultBudgetReservation
}
type rpcResultDependency struct {
waiter *rpcResultWaiter
completed bool
success bool
}
// rpcResultCache 缓存已有交付证明的 rpc_result（按 auth_key+session+req_msg_id），
// 用于跨连接重放重复请求。Put 的调用方必须先证明结果已物理写出，或原 logical Conn
// 已不可逆 fenced；绝不能发布“Conn 仍 current/open 但结果尚未上 wire”的完成态。
// encodedOutboundMessage 构造后不可变（push fan-out 与 pending resend 均依赖该契约），
// 因此 Get/Put 直接共享指针，不做防御性拷贝。
type rpcResultCache struct {
shards [rpcResultCacheShards]rpcResultCacheShard
hashSeed maphash.Seed
completedBytes rpcResultCacheByteBudget
completedEntries rpcResultFlightLimit
fairBudget *rpcResultFairBudget
flightLimit rpcResultFlightLimit
subscriberBudget *rpcResultSubscriberBudget
subscriberPerFlight int
// sessions is the production payload source. Completed ledger rows retain
// only request identity/outcome/TTL metadata; exact bytes stay solely in the
// logical-session unacked outbox. Nil is kept as a legacy unit-test seam.
sessions *SessionManager
// nextAdmissionSeq is the process-wide ordering authority for auth-key
// shared Layer defaults. Exact owners allocate once; joins/replays retain the
// owner's value from their flight/completed descriptor.
nextAdmissionSeq atomic.Uint64
activeAdmissions rpcAdmissionTracker
}
func (c *rpcResultCache) stableAdmissionSafeFloor() uint64 {
if c == nil {
return 0
}
return c.activeAdmissions.stableSafeFloor(&c.nextAdmissionSeq)
}
type rpcResultCacheShard struct {
mu sync.Mutex
now func() time.Time
ttl time.Duration
// maxEntries is a focused-test seam for one physical shard. Production leaves
// it zero and uses the explicit global/auth/session fair-budget hierarchy.
maxEntries int
bytes int
order *list.List
byKey map[rpcResultCacheKey]*list.Element
// pending is deliberately independent from the completed-result order/byKey
// cache. In-flight owners and waiters must not disappear when completed
// results expire or are trimmed under entry/byte pressure.
pending map[rpcResultCacheKey]*rpcResultFlight
}
func newRPCResultCacheWithFlightLimit(now func() time.Time, maxPending int) *rpcResultCache {
if maxPending <= 0 {
maxPending = rpcResultFlightDefaultMaxPending
}
pendingPerAuth := rpcResultFlightMaxPendingPerAuth
if pendingPerAuth > maxPending {
pendingPerAuth = maxPending
}
return newRPCResultCacheWithFairCapacity(now, rpcResultCacheCapacity{
maxPending: maxPending,
maxPendingPerAuth: pendingPerAuth,
globalMaxBytes: rpcResultCacheMaxBytes,
globalMaxEntries: rpcResultCacheMaxEntries,
authMaxBytes: rpcResultCacheAuthMaxBytes,
authMaxEntries: rpcResultCacheAuthMaxEntries,
sessionMaxBytes: rpcResultCacheSessionMaxBytes,
sessionMaxEntries: rpcResultCacheSessionMaxEntries,
})
}
func newRPCResultCacheWithLimits(now func() time.Time, maxPending, maxCompletedBytes int) *rpcResultCache {
return newRPCResultCacheWithCapacity(now, maxPending, int64(maxCompletedBytes), rpcResultCacheMaxEntries)
}
func newRPCResultCacheWithCapacity(
now func() time.Time,
maxPending int,
maxCompletedBytes int64,
maxCompletedEntries int,
) *rpcResultCache {
// Compatibility/test constructor: the caller supplied only global limits, so
// keep every fairness scope equal to that global ceiling. Production always
// calls newRPCResultCacheWithFairCapacity with explicit auth/session limits.
return newRPCResultCacheWithFairCapacity(now, rpcResultCacheCapacity{
maxPending: maxPending,
maxPendingPerAuth: maxPending,
globalMaxBytes: maxCompletedBytes,
globalMaxEntries: maxCompletedEntries,
authMaxBytes: maxCompletedBytes,
authMaxEntries: maxCompletedEntries,
sessionMaxBytes: maxCompletedBytes,
sessionMaxEntries: maxCompletedEntries,
})
}
type rpcResultCacheCapacity struct {
maxPending int
maxPendingPerAuth int
globalMaxBytes int64
globalMaxEntries int
authMaxBytes int64
authMaxEntries int
sessionMaxBytes int64
sessionMaxEntries int
subscriberMaxGlobal int
subscriberMaxAuth int
subscriberMaxSession int
subscriberMaxPerFlight int
sessions *SessionManager
}
func newRPCResultCacheWithFairCapacity(now func() time.Time, capacity rpcResultCacheCapacity) *rpcResultCache {
if now == nil {
now = time.Now
}
if capacity.maxPending <= 0 {
capacity.maxPending = rpcResultFlightDefaultMaxPending
}
if capacity.maxPendingPerAuth <= 0 {
capacity.maxPendingPerAuth = capacity.maxPending
}
if capacity.globalMaxBytes <= 0 {
capacity.globalMaxBytes = rpcResultCacheMaxBytes
}
if capacity.globalMaxEntries <= 0 {
capacity.globalMaxEntries = rpcResultCacheMaxEntries
}
if capacity.authMaxBytes <= 0 {
capacity.authMaxBytes = capacity.globalMaxBytes
}
if capacity.authMaxEntries <= 0 {
capacity.authMaxEntries = capacity.globalMaxEntries
}
if capacity.sessionMaxBytes <= 0 {
capacity.sessionMaxBytes = capacity.authMaxBytes
}
if capacity.sessionMaxEntries <= 0 {
capacity.sessionMaxEntries = capacity.authMaxEntries
}
if capacity.subscriberMaxGlobal <= 0 {
capacity.subscriberMaxGlobal = rpcResultSubscriberMaxGlobal
}
if capacity.subscriberMaxAuth <= 0 {
capacity.subscriberMaxAuth = rpcResultSubscriberMaxAuth
}
if capacity.subscriberMaxSession <= 0 {
capacity.subscriberMaxSession = rpcResultSubscriberMaxSession
}
if capacity.subscriberMaxPerFlight <= 0 {
capacity.subscriberMaxPerFlight = rpcResultSubscriberMaxPerFlight
}
c := &rpcResultCache{hashSeed: maphash.MakeSeed(), sessions: capacity.sessions}
c.completedBytes.max = capacity.globalMaxBytes
c.completedEntries.max = int64(capacity.globalMaxEntries)
c.flightLimit.max = int64(capacity.maxPending)
c.fairBudget = newRPCResultFairBudget(
c.hashSeed,
&c.completedEntries,
&c.completedBytes,
rpcResultBudgetLimit{entries: int64(capacity.authMaxEntries), bytes: capacity.authMaxBytes},
rpcResultBudgetLimit{entries: int64(capacity.sessionMaxEntries), bytes: capacity.sessionMaxBytes},
capacity.maxPendingPerAuth,
)
c.subscriberBudget = newRPCResultSubscriberBudget(
c.hashSeed,
capacity.subscriberMaxGlobal,
capacity.subscriberMaxAuth,
capacity.subscriberMaxSession,
)
c.subscriberPerFlight = capacity.subscriberMaxPerFlight
for i := range c.shards {
s := &c.shards[i]
s.now = now
s.ttl = rpcResultCacheTTL
s.maxEntries = 0
s.order = list.New()
s.byKey = make(map[rpcResultCacheKey]*list.Element)
s.pending = make(map[rpcResultCacheKey]*rpcResultFlight)
}
return c
}
func (c *rpcResultCache) shard(key rpcResultCacheKey) *rpcResultCacheShard {
return &c.shards[c.shardIndex(key)]
}
func (c *rpcResultCache) shardIndex(key rpcResultCacheKey) uint64 {
var raw [24]byte
copy(raw[:8], key.authKeyID[:])
binary.LittleEndian.PutUint64(raw[8:16], uint64(key.sessionID))
binary.LittleEndian.PutUint64(raw[16:24], uint64(key.reqMsgID))
return maphash.Bytes(c.hashSeed, raw[:]) & (rpcResultCacheShards - 1)
}
func (c *rpcResultCache) Get(authKeyID [8]byte, sessionID, reqMsgID int64) (*encodedOutboundMessage, bool) {
if c == nil || reqMsgID == 0 {
return nil, false
}
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s := c.shard(key)
now := s.now()
s.mu.Lock()
elem, ok := s.byKey[key]
if !ok {
s.mu.Unlock()
return nil, false
}
entry := elem.Value.(*rpcResultCacheEntry)
if !entry.expiresAt.After(now) {
s.removeElement(elem)
s.mu.Unlock()
return nil, false
}
if entry.capacity || entry.acknowledged {
s.mu.Unlock()
return nil, false
}
if c.sessions != nil {
s.mu.Unlock()
return c.sessions.rpcResult(authKeyID, sessionID, reqMsgID)
}
encoded := entry.encoded
s.mu.Unlock()
return encoded, encoded != nil
}
// Acknowledge removes a completed rpc_result receipt immediately.
// The caller must have resolved reqMsgID from a client msgs_ack through the
// sole outbound actor's tracked server-message mapping; an untrusted request
// msg_id must never call this method directly.
//
// ACK can race the asynchronous terminal Put. A pending flight records the
// event so Put resolves current waiters but does not retain a completed row.
// This follows the logical-session model: once the client has acknowledged the
// exact server message, neither its payload nor its duplicate receipt is useful.
func (c *rpcResultCache) Acknowledge(authKeyID [8]byte, sessionID, reqMsgID int64) bool {
if c == nil || reqMsgID == 0 {
return false
}
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s := c.shard(key)
now := s.now()
s.mu.Lock()
s.expireLocked(now)
if elem := s.byKey[key]; elem != nil {
s.removeElement(elem)
s.mu.Unlock()
return true
}
if flight := s.pending[key]; flight != nil {
flight.acknowledged = true
s.mu.Unlock()
return true
}
s.mu.Unlock()
return false
}
// ObserveDependency returns a waiter for an admitted in-flight dependency, a
// nil waiter for an already completed dependency, or ok=false when the
// referenced message never established API-RPC ownership. It never creates a
// flight and therefore cannot turn a forged invokeAfterMsg into authority to
// run another request.
func (c *rpcResultCache) ObserveDependency(authKeyID [8]byte, sessionID, reqMsgID int64) (rpcResultDependency, bool) {
if c == nil || reqMsgID == 0 {
return rpcResultDependency{}, false
}
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s := c.shard(key)
now := s.now()
s.mu.Lock()
defer s.mu.Unlock()
if elem, exists := s.byKey[key]; exists {
entry := elem.Value.(*rpcResultCacheEntry)
if entry.expiresAt.After(now) {
if !entry.executionKnown {
return rpcResultDependency{}, false
}
return rpcResultDependency{completed: true, success: entry.executionOK}, true
}
s.removeElement(elem)
}
if flight := s.pending[key]; flight != nil {
if flight.executionDone {
return rpcResultDependency{completed: true, success: flight.executionOK}, true
}
return rpcResultDependency{waiter: &rpcResultWaiter{cache: c, key: key, flight: flight}}, true
}
return rpcResultDependency{}, false
}
func (c *rpcResultCache) Put(authKeyID [8]byte, sessionID, reqMsgID int64, encoded *encodedOutboundMessage) {
if c == nil || reqMsgID == 0 || encoded == nil {
return
}
if c.putOnce(authKeyID, sessionID, reqMsgID, encoded) {
return
}
// A direct Put has no pre-reserved owner slot. Expired entries in another
// shard may be its only blocker; reap once without holding a shard and retry.
// Production owner publication already carries both reservations and never
// needs this cold path.
c.expireCompletedResults()
_ = c.putOnce(authKeyID, sessionID, reqMsgID, encoded)
}
// putOnce returns false only when a cross-shard expiry reap may release the
// process-wide entry/body capacity needed by a defensive direct Put.
func (c *rpcResultCache) putOnce(authKeyID [8]byte, sessionID, reqMsgID int64, encoded *encodedOutboundMessage) bool {
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
s := c.shard(key)
logicalOutbox := c.sessions != nil
logicalReplayable := false
if logicalOutbox {
_, logicalReplayable = c.sessions.rpcResult(authKeyID, sessionID, reqMsgID)
}
accountedSize := len(encoded.body)
if logicalOutbox {
accountedSize = 1
}
if accountedSize < 1 {
// Every owner reserves one byte at admission. Keeping zero-length results
// at the same minimum makes entry and byte capacity linearizable.
accountedSize = 1
}
// Publication never evicts another unexpired result. A production owner has
// already reserved its entry slot and one byte. If its actual result cannot
// expand that reservation, publish a one-byte identity tombstone: the owner
// and current waiters still receive the immutable result, while later
// duplicates fail admission instead of rerunning the handler.
s.mu.Lock()
now := s.now()
s.expireLocked(now)
old := s.byKey[key]
flight := s.pending[key]
if old == nil && flight == nil && s.maxEntries > 0 && len(s.byKey) >= s.maxEntries {
// Defensive direct Put callers do not own a reserved admission slot.
// Preserve every existing unexpired result and decline the new cache row.
s.mu.Unlock()
return true
}
identity, admissionSeq, executionKnown, executionOK, acknowledged := rpcResultFlightMetadataLocked(s, key)
var oldEntry *rpcResultCacheEntry
if old != nil {
oldEntry = old.Value.(*rpcResultCacheEntry)
if flight == nil {
// The first terminal publication is immutable, whether it retained an
// legacy inline body, receipt or capacity tombstone. A
// stale callback must not replace exact bytes, extend TTL, resurrect an
// ACKed body or valid receipt.
s.mu.Unlock()
return true
}
}
var reservation *rpcResultBudgetReservation
switch {
case flight != nil:
reservation = flight.reservation
if reservation == nil {
s.mu.Unlock()
panic("mtprotoedge: pending rpc result has no fair-budget reservation")
}
case oldEntry != nil && oldEntry.reservation != nil:
reservation = oldEntry.reservation
default:
reservation = c.fairBudget.reserveCompleted(key, accountedSize)
if reservation == nil {
s.mu.Unlock()
return false
}
}
retainedSize := accountedSize
retained := encoded
capacity := false
if acknowledged {
const receiptSize = 1
if !reservation.resizeBytes(receiptSize) {
s.mu.Unlock()
panic("mtprotoedge: acknowledged rpc result owner lost its receipt reservation")
}
retainedSize = receiptSize
retained = nil
} else if logicalOutbox {
const receiptSize = 1
if !reservation.resizeBytes(receiptSize) {
s.mu.Unlock()
panic("mtprotoedge: logical rpc result owner lost its receipt reservation")
}
retainedSize = receiptSize
retained = nil
capacity = !logicalReplayable
} else if !reservation.resizeBytes(accountedSize) {
if flight == nil {
// A direct replacement cannot discard the prior replay body. Leave it
// untouched and let Put perform one cross-shard expiry reap before its
// final bounded failure.
if oldEntry == nil {
reservation.release()
}
s.mu.Unlock()
return false
}
// Owner admission already reserved one byte at all three scopes. When the
// actual body cannot expand, transfer that reservation to an identity
// tombstone so a duplicate never reruns business.
const tombstoneSize = 1
if !reservation.resizeBytes(tombstoneSize) {
s.mu.Unlock()
panic("mtprotoedge: rpc result owner lost its one-byte tombstone reservation")
}
retainedSize = tombstoneSize
retained = nil
capacity = true
}
if old != nil {
s.unlinkElement(old)
if oldEntry.reservation != nil && oldEntry.reservation != reservation {
oldEntry.reservation.release()
oldEntry.reservation = nil
}
}
entry := &rpcResultCacheEntry{
key: key,
encoded: retained,
size: retainedSize,
expiresAt: now.Add(s.ttl),
identity: identity,
admissionSeq: admissionSeq,
executionKnown: executionKnown,
executionOK: executionOK,
acknowledged: acknowledged,
capacity: capacity,
reservation: reservation,
}
elem := s.order.PushBack(entry)
s.byKey[key] = elem
s.bytes += retainedSize
// Resolve the independent in-flight entry only after either the completed
// result or its replay tombstone is published under the same shard lock.
subscribers, executionSubscribers, executionOK := c.completeRPCResultFlightLocked(s, key, encoded)
if acknowledged {
// ACK won before terminal publication. Current subscribers still receive
// encoded below, but no post-ACK receipt survives this critical section.
s.removeElement(elem)
}
s.mu.Unlock()
for _, subscriber := range subscribers {
subscriber(encoded, true)
}
for _, subscriber := range executionSubscribers {
subscriber(executionOK)
}
return true
}
func rpcResultFlightMetadataLocked(s *rpcResultCacheShard, key rpcResultCacheKey) (
rpcResultRequestIdentity,
uint64,
bool,
bool,
bool,
) {
if flight := s.pending[key]; flight != nil {
return flight.identity, flight.admissionSeq, flight.executionDone, flight.executionOK, flight.acknowledged
}
return rpcResultRequestIdentity{}, 0, false, false, false
}
// expireCompletedResults performs the cold-path cross-shard reap used only
// after a one-byte admission reservation fails. The caller must hold no shard
// lock. Each shard is reaped independently so ordinary result publication on
// the other shards remains parallel.
func (c *rpcResultCache) expireCompletedResults() {
if c == nil {
return
}
for i := range c.shards {
s := &c.shards[i]
s.mu.Lock()
s.expireLocked(s.now())
s.mu.Unlock()
}
}
func (c *rpcResultCache) Close() error {
return nil
}
func (c *rpcResultCache) CloseContext(context.Context) error {
return nil
}
// forgetCompletedSession removes every terminal receipt for a logical session.
// SessionManager invokes it only after the session's physical producers have
// converged, so no pending owner should remain and no later terminal callback
// can recreate a receipt for the destroyed outbox.
func (c *rpcResultCache) forgetCompletedSession(authKeyID [8]byte, sessionID int64) {
if c == nil {
return
}
for i := range c.shards {
s := &c.shards[i]
s.mu.Lock()
for elem := s.order.Front(); elem != nil; {
next := elem.Next()
entry := elem.Value.(*rpcResultCacheEntry)
if entry.key.authKeyID == authKeyID && entry.key.sessionID == sessionID {
s.removeElement(elem)
}
elem = next
}
s.mu.Unlock()
}
}
func (s *rpcResultCacheShard) expireLocked(now time.Time) {
for elem := s.order.Front(); elem != nil; {
next := elem.Next()
entry := elem.Value.(*rpcResultCacheEntry)
if entry.expiresAt.After(now) {
return
}
s.removeElement(elem)
elem = next
}
}
func (s *rpcResultCacheShard) removeElement(elem *list.Element) {
entry := s.unlinkElement(elem)
if entry != nil && entry.reservation != nil {
entry.reservation.release()
entry.reservation = nil
}
}
func (s *rpcResultCacheShard) unlinkElement(elem *list.Element) *rpcResultCacheEntry {
if elem == nil {
return nil
}
entry := elem.Value.(*rpcResultCacheEntry)
delete(s.byKey, entry.key)
s.bytes -= entry.size
if s.bytes < 0 {
s.bytes = 0
}
s.order.Remove(elem)
return entry
}
type rpcResultCacheByteBudget struct {
max int64
used atomic.Int64
}
func (b *rpcResultCacheByteBudget) reserve(n int) bool {
if n <= 0 {
return true
}
bytes := int64(n)
if b == nil || bytes > b.max {
return false
}
for {
used := b.used.Load()
if used > b.max-bytes {
return false
}
if b.used.CompareAndSwap(used, used+bytes) {
return true
}
}
}
func (b *rpcResultCacheByteBudget) release(n int) {
if b == nil || n <= 0 {
return
}
if remaining := b.used.Add(-int64(n)); remaining < 0 {
panic("mtprotoedge: rpc result completed-byte budget underflow")
}
}
func (b *rpcResultCacheByteBudget) snapshot() int64 {
if b == nil {
return 0
}
return b.used.Load()
}

822
internal/mtprotoedge/rpc_result_cache_test.go
View file

@ -1,822 +0,0 @@
package mtprotoedge
import (
"errors"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestRPCResultCacheFullSessionDoesNotBlockAnotherAuth(t *testing.T) {
now := time.Unix(1000, 0)
cache := newRPCResultCacheWithFairCapacity(func() time.Time { return now }, rpcResultCacheCapacity{
maxPending: 8, maxPendingPerAuth: 6,
globalMaxEntries: 8, globalMaxBytes: 64,
authMaxEntries: 6, authMaxBytes: 48,
sessionMaxEntries: 2, sessionMaxBytes: 16,
})
authA := [8]byte{0xa1}
authB := [8]byte{0xb1}
const sessionA = int64(77)
for i := 0; i < 2; i++ {
msgID := int64(1000 + i)
claim, err := cache.Acquire(authA, sessionA, msgID)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("same-session admission %d = %#v, %v", i, claim, err)
}
cache.Put(authA, sessionA, msgID, &encodedOutboundMessage{body: []byte{1}})
}
if _, err := cache.Acquire(authA, sessionA, 2000); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("admission beyond session entry limit = %v, want capacity", err)
}
otherAuth, err := cache.Acquire(authB, 88, 3000)
if err != nil || otherAuth.state != rpcResultAcquireOwner {
t.Fatalf("other auth blocked by full session: %#v, %v", otherAuth, err)
}
if !otherAuth.owner.Abort() {
t.Fatal("other-auth owner did not abort")
}
if _, ok := cache.Get(authA, sessionA, 1000); !ok {
t.Fatal("session capacity pressure evicted an unexpired result")
}
}
func TestRPCResultCacheFullAuthDoesNotBlockAnotherAuth(t *testing.T) {
cache := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
maxPending: 8, maxPendingPerAuth: 4,
globalMaxEntries: 8, globalMaxBytes: 64,
authMaxEntries: 2, authMaxBytes: 32,
sessionMaxEntries: 2, sessionMaxBytes: 16,
})
authA := [8]byte{0xa2}
authB := [8]byte{0xb2}
for i := 0; i < 2; i++ {
claim, err := cache.Acquire(authA, int64(10+i), int64(100+i))
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("auth A admission %d = %#v, %v", i, claim, err)
}
cache.Put(authA, int64(10+i), int64(100+i), &encodedOutboundMessage{body: []byte{1}})
}
if _, err := cache.Acquire(authA, 12, 102); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("same-auth new session at auth limit = %v, want capacity", err)
}
other, err := cache.Acquire(authB, 20, 200)
if err != nil || other.state != rpcResultAcquireOwner {
t.Fatalf("other auth blocked by full auth A: %#v, %v", other, err)
}
other.owner.Abort()
}
func TestRPCResultCacheAuthAndSessionByteLimitsAreIndependent(t *testing.T) {
cache := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
maxPending: 8, maxPendingPerAuth: 6,
globalMaxEntries: 10, globalMaxBytes: 10,
authMaxEntries: 8, authMaxBytes: 4,
sessionMaxEntries: 6, sessionMaxBytes: 2,
})
authA := [8]byte{0xa4}
authB := [8]byte{0xb4}
first, err := cache.Acquire(authA, 1, 101)
if err != nil || first.state != rpcResultAcquireOwner {
t.Fatalf("first owner = %#v, %v", first, err)
}
cache.Put(authA, 1, 101, &encodedOutboundMessage{body: []byte{1, 2}})
if _, err := cache.Acquire(authA, 1, 102); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("same session beyond byte limit = %v, want capacity", err)
}
second, err := cache.Acquire(authA, 2, 201)
if err != nil || second.state != rpcResultAcquireOwner {
t.Fatalf("second session owner = %#v, %v", second, err)
}
cache.Put(authA, 2, 201, &encodedOutboundMessage{body: []byte{3, 4}})
if _, err := cache.Acquire(authA, 3, 301); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("same auth beyond byte limit = %v, want capacity", err)
}
other, err := cache.Acquire(authB, 3, 302)
if err != nil || other.state != rpcResultAcquireOwner {
t.Fatalf("other auth blocked by auth A byte limit: %#v, %v", other, err)
}
other.owner.Abort()
}
func TestRPCResultCachePerAuthPendingLimitIsAdditional(t *testing.T) {
cache := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
maxPending: 6, maxPendingPerAuth: 2,
globalMaxEntries: 12, globalMaxBytes: 64,
authMaxEntries: 6, authMaxBytes: 32,
sessionMaxEntries: 4, sessionMaxBytes: 16,
})
authA := [8]byte{0xa3}
authB := [8]byte{0xb3}
owners := make([]*rpcResultOwnerLease, 0, 3)
for i := 0; i < 2; i++ {
claim, err := cache.Acquire(authA, int64(i+1), int64(100+i))
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("pending auth A %d = %#v, %v", i, claim, err)
}
owners = append(owners, claim.owner)
}
if _, err := cache.Acquire(authA, 3, 103); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("third pending owner for auth A = %v, want capacity", err)
}
other, err := cache.Acquire(authB, 4, 104)
if err != nil || other.state != rpcResultAcquireOwner {
t.Fatalf("auth B blocked by auth A pending limit: %#v, %v", other, err)
}
owners = append(owners, other.owner)
for _, owner := range owners {
if !owner.Abort() {
t.Fatal("pending owner did not abort")
}
}
if usage := cache.fairBudget.authSnapshot(authA); usage != (rpcResultBudgetUsage{}) {
t.Fatalf("auth A budget after abort = %#v", usage)
}
}
func TestRPCResultCacheFairReservationLifecycleReturnsEveryScope(t *testing.T) {
now := time.Unix(1000, 0)
cache := newRPCResultCacheWithFairCapacity(func() time.Time { return now }, rpcResultCacheCapacity{
maxPending: 4, maxPendingPerAuth: 3,
globalMaxEntries: 6, globalMaxBytes: 10,
authMaxEntries: 5, authMaxBytes: 8,
sessionMaxEntries: 3, sessionMaxBytes: 6,
})
auth := [8]byte{0xc1}
aborted, err := cache.Acquire(auth, 1, 101)
if err != nil || aborted.state != rpcResultAcquireOwner {
t.Fatalf("aborted owner = %#v, %v", aborted, err)
}
if usage := cache.fairBudget.authSnapshot(auth); usage.entries != 1 || usage.bytes != 1 || usage.pending != 1 {
t.Fatalf("pending auth reservation = %#v", usage)
}
if !aborted.owner.Abort() {
t.Fatal("owner Abort lost")
}
if usage := cache.fairBudget.authSnapshot(auth); usage != (rpcResultBudgetUsage{}) {
t.Fatalf("Abort leaked auth reservation %#v", usage)
}
body, err := cache.Acquire(auth, 1, 102)
if err != nil || body.state != rpcResultAcquireOwner {
t.Fatalf("body owner = %#v, %v", body, err)
}
cache.Put(auth, 1, 102, &encodedOutboundMessage{body: make([]byte, 4)})
if usage := cache.fairBudget.sessionSnapshot(auth, 1); usage.entries != 1 || usage.bytes != 4 || usage.pending != 0 {
t.Fatalf("body session reservation = %#v", usage)
}
tombstone, err := cache.Acquire(auth, 2, 201)
if err != nil || tombstone.state != rpcResultAcquireOwner {
t.Fatalf("tombstone owner = %#v, %v", tombstone, err)
}
// This cannot fit the 10-byte global or 8-byte auth ceiling. Put must not
// panic or lose ownership; it transfers the one-byte token to a tombstone.
cache.Put(auth, 2, 201, &encodedOutboundMessage{body: make([]byte, 20)})
if usage := cache.fairBudget.sessionSnapshot(auth, 2); usage.entries != 1 || usage.bytes != 1 || usage.pending != 0 {
t.Fatalf("tombstone session reservation = %#v", usage)
}
if got := cache.completedEntries.snapshot(); got != 2 {
t.Fatalf("global entries after body+tombstone = %d, want 2", got)
}
if got := cache.completedBytes.snapshot(); got != 5 {
t.Fatalf("global bytes after body+tombstone = %d, want 5", got)
}
cache.Put(auth, 1, 102, &encodedOutboundMessage{body: make([]byte, 2)})
if got := cache.completedBytes.snapshot(); got != 5 {
t.Fatalf("duplicate publication changed immutable global bytes: %d", got)
}
now = now.Add(rpcResultCacheTTL + time.Second)
_, _ = cache.Get(auth, 1, 102)
_, _ = cache.Get(auth, 2, 201)
if got := cache.completedEntries.snapshot(); got != 0 {
t.Fatalf("TTL leaked global entries %d", got)
}
if got := cache.completedBytes.snapshot(); got != 0 {
t.Fatalf("TTL leaked global bytes %d", got)
}
if usage := cache.fairBudget.authSnapshot(auth); usage != (rpcResultBudgetUsage{}) {
t.Fatalf("TTL leaked auth reservation %#v", usage)
}
}
func TestRPCResultCacheFullKeyMaphashSpreadsOneSession(t *testing.T) {
first := newRPCResultCacheWithFlightLimit(time.Now, 64)
second := newRPCResultCacheWithFlightLimit(time.Now, 64)
auth := [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
const sessionID = int64(99)
seen := make(map[uint64]struct{})
differentInstance := false
for msgID := int64(1); msgID <= 256; msgID++ {
key := rpcResultCacheKey{authKeyID: auth, sessionID: sessionID, reqMsgID: msgID}
firstIndex := first.shardIndex(key)
seen[firstIndex] = struct{}{}
if firstIndex != second.shardIndex(key) {
differentInstance = true
}
}
if len(seen) < rpcResultCacheShards/2 {
t.Fatalf("one session used only %d/%d full-key shards", len(seen), rpcResultCacheShards)
}
if !differentInstance {
t.Fatal("two cache instances produced an identical shard stream; seed is not instance-random")
}
}
func TestRPCResultCacheConcurrentFairReservationsNeverOvercommit(t *testing.T) {
cache := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
maxPending: 24, maxPendingPerAuth: 4,
globalMaxEntries: 24, globalMaxBytes: 24,
authMaxEntries: 8, authMaxBytes: 8,
sessionMaxEntries: 3, sessionMaxBytes: 3,
})
const callers = 256
start := make(chan struct{})
var (
wg sync.WaitGroup
mu sync.Mutex
owners []*rpcResultOwnerLease
)
for i := 0; i < callers; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
<-start
auth := [8]byte{byte(i % 4)}
claim, err := cache.Acquire(auth, int64(i%8), int64(1000+i))
if errors.Is(err, ErrRPCResultFlightCapacity) {
return
}
if err != nil || claim.state != rpcResultAcquireOwner {
t.Errorf("Acquire %d = %#v, %v", i, claim, err)
return
}
mu.Lock()
owners = append(owners, claim.owner)
mu.Unlock()
}(i)
}
close(start)
wg.Wait()
if got := cache.completedEntries.snapshot(); got > 24 || got != int64(len(owners)) {
t.Fatalf("global entry usage=%d owners=%d limit=24", got, len(owners))
}
if got := cache.completedBytes.snapshot(); got > 24 || got != int64(len(owners)) {
t.Fatalf("global byte usage=%d owners=%d limit=24", got, len(owners))
}
for i := 0; i < 4; i++ {
auth := [8]byte{byte(i)}
usage := cache.fairBudget.authSnapshot(auth)
if usage.entries > 8 || usage.bytes > 8 || usage.pending > 4 {
t.Fatalf("auth %d overcommitted: %#v", i, usage)
}
for sessionID := int64(0); sessionID < 8; sessionID++ {
session := cache.fairBudget.sessionSnapshot(auth, sessionID)
if session.entries > 3 || session.bytes > 3 {
t.Fatalf("auth %d session %d overcommitted: %#v", i, sessionID, session)
}
}
}
for _, owner := range owners {
if !owner.Abort() {
t.Fatal("concurrent owner did not abort")
}
}
if cache.completedEntries.snapshot() != 0 || cache.completedBytes.snapshot() != 0 {
t.Fatal("concurrent Abort leaked global fair budget")
}
}
func TestRPCResultCacheConcurrentOwnerPublicationAcrossShards(t *testing.T) {
const publications = 256
now := time.Unix(1000, 0)
cache := newRPCResultCacheWithFairCapacity(func() time.Time { return now }, rpcResultCacheCapacity{
maxPending: publications, maxPendingPerAuth: 4,
globalMaxEntries: publications, globalMaxBytes: publications * 4,
authMaxEntries: 4, authMaxBytes: 16,
sessionMaxEntries: 1, sessionMaxBytes: 4,
})
type publication struct {
auth [8]byte
session int64
msgID int64
owner *rpcResultOwnerLease
}
publicationsByKey := make([]publication, 0, publications)
for i := 0; i < publications; i++ {
auth := [8]byte{byte(i), byte(i >> 8), 0xa5}
sessionID := int64(10_000 + i)
msgID := int64(20_000 + i)
claim, err := cache.Acquire(auth, sessionID, msgID)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("Acquire %d = %#v, %v", i, claim, err)
}
publicationsByKey = append(publicationsByKey, publication{
auth: auth, session: sessionID, msgID: msgID, owner: claim.owner,
})
}
start := make(chan struct{})
var wg sync.WaitGroup
for i := range publicationsByKey {
item := publicationsByKey[i]
wg.Add(1)
go func() {
defer wg.Done()
<-start
if !item.owner.CompleteExecution(true) {
t.Errorf("CompleteExecution(%d) lost owner", item.msgID)
return
}
cache.Put(item.auth, item.session, item.msgID, &encodedOutboundMessage{body: []byte{1, 2, 3, 4}})
}()
}
close(start)
wg.Wait()
for _, item := range publicationsByKey {
encoded, ok := cache.Get(item.auth, item.session, item.msgID)
if !ok || encoded == nil || len(encoded.body) != 4 {
t.Fatalf("completed publication %d missing: ok=%v encoded=%#v", item.msgID, ok, encoded)
}
}
if got := cache.completedEntries.snapshot(); got != publications {
t.Fatalf("completed entries=%d, want %d", got, publications)
}
if got := cache.completedBytes.snapshot(); got != publications*4 {
t.Fatalf("completed bytes=%d, want %d", got, publications*4)
}
now = now.Add(rpcResultCacheTTL + time.Second)
cache.expireCompletedResults()
if cache.completedEntries.snapshot() != 0 || cache.completedBytes.snapshot() != 0 {
t.Fatal("parallel publications leaked fair-budget reservations after TTL")
}
}
func BenchmarkRPCResultCacheParallelShardPut(b *testing.B) {
cache := newRPCResultCacheWithFlightLimit(time.Now, rpcResultFlightDefaultMaxPending)
var nextWorker atomic.Uint64
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
id := nextWorker.Add(1)
auth := [8]byte{
byte(id), byte(id >> 8), byte(id >> 16), byte(id >> 24),
byte(id >> 32), byte(id >> 40), byte(id >> 48), byte(id >> 56),
}
sessionID := int64(id)
msgID := int64(1_000_000 + id)
encoded := &encodedOutboundMessage{body: []byte{1, 2, 3, 4}}
for pb.Next() {
cache.Put(auth, sessionID, msgID, encoded)
}
})
}
func TestRPCResultCacheEntryReservationTransfersAndReturns(t *testing.T) {
now := time.Unix(1000, 0)
cache := newRPCResultCacheWithCapacity(func() time.Time { return now }, 4, 2, 2)
authKeyID := [8]byte{0xa2}
first, err := cache.Acquire(authKeyID, 1, 101)
if err != nil || first.state != rpcResultAcquireOwner || cache.completedEntries.snapshot() != 1 {
t.Fatalf("first pending reservation = %#v entries=%d err=%v", first, cache.completedEntries.snapshot(), err)
}
cache.Put(authKeyID, 1, 101, &encodedOutboundMessage{body: []byte{1}})
if got := cache.completedEntries.snapshot(); got != 1 {
t.Fatalf("pending -> body changed entry count to %d", got)
}
second, err := cache.Acquire(authKeyID, 2, 202)
if err != nil || second.state != rpcResultAcquireOwner || cache.completedEntries.snapshot() != 2 {
t.Fatalf("second pending reservation = %#v entries=%d err=%v", second, cache.completedEntries.snapshot(), err)
}
// The byte budget has only the second owner's one-byte token remaining.
// Publication therefore leaves an identity tombstone, which still owns its
// real process-wide entry slot.
cache.Put(authKeyID, 2, 202, &encodedOutboundMessage{body: []byte{2, 2, 2}})
if got := cache.completedEntries.snapshot(); got != 2 {
t.Fatalf("pending -> tombstone changed entry count to %d", got)
}
if _, err := cache.Acquire(authKeyID, 3, 303); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("third admission at entry limit = %v, want capacity", err)
}
now = now.Add(rpcResultCacheTTL + time.Second)
firstShard := cache.shardIndex(rpcResultCacheKey{authKeyID: authKeyID, sessionID: 1, reqMsgID: 101})
secondShard := cache.shardIndex(rpcResultCacheKey{authKeyID: authKeyID, sessionID: 2, reqMsgID: 202})
thirdMsgID := rpcResultTestMsgIDOutsideShards(t, cache, authKeyID, 3, 303, firstShard, secondShard)
third, err := cache.Acquire(authKeyID, 3, thirdMsgID)
if err != nil || third.state != rpcResultAcquireOwner {
t.Fatalf("admission after global expiry reap = %#v, %v", third, err)
}
if got := cache.completedEntries.snapshot(); got != 1 {
t.Fatalf("expired entries were not returned before new owner: %d", got)
}
if !third.owner.Abort() || cache.completedEntries.snapshot() != 0 {
t.Fatalf("Abort did not return entry reservation: entries=%d", cache.completedEntries.snapshot())
}
}
func TestRPCResultCacheConcurrentGlobalEntryReservationNeverOvercommits(t *testing.T) {
const limit = 8
cache := newRPCResultCacheWithCapacity(time.Now, 128, 1<<20, limit)
authKeyID := [8]byte{0xa3}
var (
wg sync.WaitGroup
mu sync.Mutex
owners []*rpcResultOwnerLease
)
for i := 0; i < 64; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
claim, err := cache.Acquire(authKeyID, int64(i+1), int64(1000+i))
if err != nil {
if !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Errorf("Acquire %d: %v", i, err)
}
return
}
if claim.state != rpcResultAcquireOwner {
t.Errorf("Acquire %d state = %d", i, claim.state)
return
}
mu.Lock()
owners = append(owners, claim.owner)
mu.Unlock()
}(i)
}
wg.Wait()
if len(owners) != limit || cache.completedEntries.snapshot() != limit {
t.Fatalf("concurrent owners=%d entries=%d, want %d", len(owners), cache.completedEntries.snapshot(), limit)
}
for _, owner := range owners {
if !owner.Abort() {
t.Fatal("reserved owner failed to abort")
}
}
if got := cache.completedEntries.snapshot(); got != 0 {
t.Fatalf("entry reservations after abort = %d", got)
}
}
func TestRPCResultCacheRoundTripAndTTL(t *testing.T) {
if rpcResultCacheTTL != 331*time.Second {
t.Fatalf("replay TTL = %v, want full 300s past + 30s future window + 1s", rpcResultCacheTTL)
}
now := time.Unix(1000, 0)
cache := newRPCResultCache(func() time.Time { return now })
var keyID [8]byte
keyID[0] = 0xab
encoded := &encodedOutboundMessage{body: []byte{1, 2, 3, 4}, typeID: 42, reqMsgID: 7}
if _, ok := cache.Get(keyID, 5, 7); ok {
t.Fatal("unexpected hit on empty cache")
}
cache.Put(keyID, 5, 7, encoded)
if got := cache.completedEntries.snapshot(); got != 1 {
t.Fatalf("direct Put entry reservation = %d, want 1", got)
}
if usage := cache.fairBudget.sessionSnapshot(keyID, 5); usage.entries != 1 || usage.bytes != 4 || usage.pending != 0 {
t.Fatalf("direct Put session reservation = %#v", usage)
}
got, ok := cache.Get(keyID, 5, 7)
if !ok {
t.Fatal("expected hit")
}
// encodedOutboundMessage 不可变契约下 Get/Put 共享指针，不做防御性拷贝。
if got != encoded {
t.Fatal("expected shared pointer, got clone")
}
// 不同 session / msg_id 不串。
if _, ok := cache.Get(keyID, 6, 7); ok {
t.Fatal("hit with wrong session id")
}
if _, ok := cache.Get(keyID, 5, 8); ok {
t.Fatal("hit with wrong msg id")
}
// TTL 过期。
now = now.Add(rpcResultCacheTTL + time.Second)
if _, ok := cache.Get(keyID, 5, 7); ok {
t.Fatal("expected expiry after TTL")
}
if got := cache.completedEntries.snapshot(); got != 0 {
t.Fatalf("direct Put expiry left %d entry reservations", got)
}
if usage := cache.fairBudget.authSnapshot(keyID); usage != (rpcResultBudgetUsage{}) {
t.Fatalf("direct Put expiry leaked auth reservation %#v", usage)
}
}
func TestRPCResultCacheDuplicatePutPreservesCompletedExecutionMetadata(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
keyID := [8]byte{1, 9, 8, 4}
const sessionID, reqMsgID = int64(11), int64(12)
claim, err := cache.Acquire(keyID, sessionID, reqMsgID)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("Acquire owner = %#v, %v", claim, err)
}
if !claim.owner.CompleteExecution(true) || !claim.owner.HandOff() {
t.Fatal("complete owner metadata")
}
first := &encodedOutboundMessage{body: []byte{1, 2, 3, 4}, typeID: 42, reqMsgID: reqMsgID}
cache.Put(keyID, sessionID, reqMsgID, first)
second := &encodedOutboundMessage{body: []byte{5, 6, 7, 8}, typeID: 42, reqMsgID: reqMsgID}
cache.Put(keyID, sessionID, reqMsgID, second)
replay, err := cache.Acquire(keyID, sessionID, reqMsgID)
if err != nil || replay.state != rpcResultAcquireCompleted || replay.encoded != first ||
!replay.executionKnown || !replay.executionOK {
t.Fatalf("duplicate Put metadata = %#v, err=%v", replay, err)
}
}
func TestRPCResultCacheShardCapacityNeverEvictsUnexpiredResult(t *testing.T) {
now := time.Unix(1000, 0)
cache := newRPCResultCache(func() time.Time { return now })
var keyID [8]byte
firstKey := rpcResultCacheKey{authKeyID: keyID, sessionID: 1, reqMsgID: 100}
shard := cache.shard(firstKey)
shard.mu.Lock()
shard.maxEntries = 1
shard.mu.Unlock()
claim, err := cache.Acquire(keyID, 1, 100)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("first admission = %#v, %v", claim, err)
}
first := &encodedOutboundMessage{body: []byte{1}}
cache.Put(keyID, 1, 100, first)
secondMsgID := rpcResultTestMsgIDForShard(t, cache, keyID, 1, 101, cache.shardIndex(firstKey))
if _, err := cache.Acquire(keyID, 1, secondMsgID); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("full-shard admission = %v, want capacity", err)
}
if got, ok := cache.Get(keyID, 1, 100); !ok || got != first {
t.Fatalf("unexpired first result was displaced: got=%p ok=%v", got, ok)
}
now = now.Add(rpcResultCacheTTL + time.Second)
claim, err = cache.Acquire(keyID, 1, secondMsgID)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("admission after expiry = %#v, %v", claim, err)
}
claim.owner.Abort()
}
func TestRPCResultCacheGlobalByteCapacityNeverEvictsUnexpiredResults(t *testing.T) {
now := time.Unix(1000, 0)
cache := newRPCResultCacheWithLimits(func() time.Time { return now }, 32, 10)
var keyID [8]byte
// Five two-byte results consume the global budget. The sixth admission must
// fail bounded; none of the retained results may be sacrificed for it.
for sessionID := int64(1); sessionID <= 5; sessionID++ {
claim, err := cache.Acquire(keyID, sessionID, 100+sessionID)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("admission %d = %#v, %v", sessionID, claim, err)
}
cache.Put(keyID, sessionID, 100+sessionID, &encodedOutboundMessage{body: []byte{1, 2}})
}
if got := cache.completedBytes.snapshot(); got != 10 {
t.Fatalf("completed bytes at capacity = %d, want 10", got)
}
if _, err := cache.Acquire(keyID, 6, 106); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("byte-full admission = %v, want capacity", err)
}
for sessionID := int64(1); sessionID <= 5; sessionID++ {
if _, ok := cache.Get(keyID, sessionID, 100+sessionID); !ok {
t.Fatalf("unexpired result %d was evicted", sessionID)
}
}
}
func TestRPCResultCacheByteBudgetReturnsOnReplaceExpiryAndCapacity(t *testing.T) {
now := time.Unix(1000, 0)
cache := newRPCResultCacheWithLimits(func() time.Time { return now }, 32, 32)
var keyID [8]byte
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 4)})
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 7)})
if got := cache.completedBytes.snapshot(); got != 4 {
t.Fatalf("completed bytes after duplicate publication = %d, want 4", got)
}
if usage := cache.fairBudget.sessionSnapshot(keyID, 1); usage.entries != 1 || usage.bytes != 4 {
t.Fatalf("immutable fair reservation after duplicate = %#v", usage)
}
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 2)})
if got := cache.completedBytes.snapshot(); got != 4 {
t.Fatalf("completed bytes after second duplicate = %d, want 4", got)
}
if usage := cache.fairBudget.sessionSnapshot(keyID, 1); usage.entries != 1 || usage.bytes != 4 {
t.Fatalf("immutable fair reservation after second duplicate = %#v", usage)
}
now = now.Add(rpcResultCacheTTL + time.Second)
if _, ok := cache.Get(keyID, 1, 101); ok {
t.Fatal("replacement should expire")
}
if got := cache.completedBytes.snapshot(); got != 0 {
t.Fatalf("completed bytes after expiry = %d, want 0", got)
}
key := rpcResultCacheKey{authKeyID: keyID, sessionID: 2, reqMsgID: 201}
shard := cache.shard(key)
shard.mu.Lock()
shard.maxEntries = 1
shard.mu.Unlock()
claim, err := cache.Acquire(keyID, 2, 201)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("entry-capacity first admission = %#v, %v", claim, err)
}
cache.Put(keyID, 2, 201, &encodedOutboundMessage{body: make([]byte, 3)})
secondMsgID := rpcResultTestMsgIDForShard(t, cache, keyID, 2, 202, cache.shardIndex(key))
if _, err := cache.Acquire(keyID, 2, secondMsgID); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("entry-capacity second admission = %v", err)
}
if got := cache.completedBytes.snapshot(); got != 3 {
t.Fatalf("completed bytes after capacity rejection = %d, want 3", got)
}
if _, ok := cache.Get(keyID, 2, 201); !ok {
t.Fatal("capacity rejection displaced the first result")
}
}
func TestRPCResultCachePublicationOverflowLeavesReplayCapacityTombstone(t *testing.T) {
now := time.Unix(1000, 0)
cache := newRPCResultCacheWithLimits(func() time.Time { return now }, 32, 4)
var keyID [8]byte
claim, err := cache.Acquire(keyID, 1, 101)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("owner admission = %#v, %v", claim, err)
}
claim.owner.CompleteExecution(true)
tooLarge := &encodedOutboundMessage{body: make([]byte, 5)}
cache.Put(keyID, 1, 101, tooLarge)
if got := cache.completedBytes.snapshot(); got != 1 {
t.Fatalf("tombstone bytes = %d, want 1", got)
}
if _, ok := cache.Get(keyID, 1, 101); ok {
t.Fatal("capacity tombstone must not masquerade as a replayable body")
}
if _, err := cache.Acquire(keyID, 1, 101); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("duplicate after publication overflow = %v, want capacity", err)
}
now = now.Add(rpcResultCacheTTL + time.Second)
retry, err := cache.Acquire(keyID, 1, 101)
if err != nil || retry.state != rpcResultAcquireOwner {
t.Fatalf("admission after tombstone expiry = %#v, %v", retry, err)
}
retry.owner.Abort()
}
func TestRPCResultCacheByteCapacityReclaimsExpiredAcrossShards(t *testing.T) {
now := time.Unix(1000, 0)
cache := newRPCResultCacheWithLimits(func() time.Time { return now }, 32, 2)
var keyID [8]byte
first, err := cache.Acquire(keyID, 1, 101)
if err != nil || first.state != rpcResultAcquireOwner {
t.Fatalf("first admission = %#v, %v", first, err)
}
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: []byte{1, 2}})
if got := cache.completedBytes.snapshot(); got != 2 {
t.Fatalf("full budget = %d, want 2", got)
}
// Select a key in another full-key shard. Its failed one-byte reservation
// must trigger the cold-path global expiry reap before returning capacity.
now = now.Add(rpcResultCacheTTL + time.Second)
firstKey := rpcResultCacheKey{authKeyID: keyID, sessionID: 1, reqMsgID: 101}
secondMsgID := rpcResultTestMsgIDOutsideShard(t, cache, keyID, 2, 202, cache.shardIndex(firstKey))
second, err := cache.Acquire(keyID, 2, secondMsgID)
if err != nil || second.state != rpcResultAcquireOwner {
t.Fatalf("cross-shard admission after expiry = %#v, %v", second, err)
}
second.owner.Abort()
if got := cache.completedBytes.snapshot(); got != 0 {
t.Fatalf("bytes after expired reap and abort = %d, want 0", got)
}
}
func TestRPCResultCacheServerOptionsPropagateFairLimits(t *testing.T) {
s := New(Options{
RPCGlobalMaxTasks: 6,
RPCResultCacheMaxEntries: 12,
RPCResultCacheAuthMaxEntries: 8,
RPCResultCacheSessionMaxEntries: 4,
RPCResultPendingPerAuth: 3,
})
if s.rpcResults.completedEntries.max != 12 || s.rpcResults.completedBytes.max != 12 {
t.Fatalf("global option propagation = %d/%d", s.rpcResults.completedEntries.max, s.rpcResults.completedBytes.max)
}
budget := s.rpcResults.fairBudget
if budget.authLimit.entries != 8 || budget.authLimit.bytes != 8 ||
budget.sessionLimit.entries != 4 || budget.sessionLimit.bytes != 4 || budget.pendingPerAuth != 3 {
t.Fatalf("fair option propagation = auth:%#v session:%#v pending:%d",
budget.authLimit, budget.sessionLimit, budget.pendingPerAuth)
}
}
func TestRPCResultCacheServerOptionsFailFast(t *testing.T) {
base := Options{
RPCGlobalMaxTasks: 6,
RPCResultCacheMaxEntries: 12,
RPCResultCacheAuthMaxEntries: 8,
RPCResultCacheSessionMaxEntries: 4,
RPCResultPendingPerAuth: 3,
}
tests := []struct {
name string
mutate func(*Options)
}{
{name: "entry hierarchy", mutate: func(o *Options) { o.RPCResultCacheAuthMaxEntries = 13 }},
{name: "pending hierarchy", mutate: func(o *Options) { o.RPCResultPendingPerAuth = 7 }},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
opts := base
test.mutate(&opts)
defer func() {
if recover() == nil {
t.Fatal("New accepted invalid rpc_result cache options")
}
}()
_ = New(opts)
})
}
}
func rpcResultTestMsgIDForShard(
t *testing.T,
cache *rpcResultCache,
authKeyID [8]byte,
sessionID, start int64,
target uint64,
) int64 {
t.Helper()
for msgID := start; msgID < start+1_000_000; msgID++ {
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: msgID}
if cache.shardIndex(key) == target {
return msgID
}
}
t.Fatal("failed to find rpc_result key for target shard")
return 0
}
func rpcResultTestMsgIDOutsideShard(
t *testing.T,
cache *rpcResultCache,
authKeyID [8]byte,
sessionID, start int64,
excluded uint64,
) int64 {
t.Helper()
for msgID := start; msgID < start+1_000_000; msgID++ {
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: msgID}
if cache.shardIndex(key) != excluded {
return msgID
}
}
t.Fatal("failed to find rpc_result key outside excluded shard")
return 0
}
func rpcResultTestMsgIDOutsideShards(
t *testing.T,
cache *rpcResultCache,
authKeyID [8]byte,
sessionID, start int64,
excluded ...uint64,
) int64 {
t.Helper()
for msgID := start; msgID < start+1_000_000; msgID++ {
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: msgID}
index := cache.shardIndex(key)
allowed := true
for _, blocked := range excluded {
if index == blocked {
allowed = false
break
}
}
if allowed {
return msgID
}
}
t.Fatal("failed to find rpc_result key outside excluded shards")
return 0
}

8
internal/mtprotoedge/rpc_result_delivery_test.go
View file

@ -92,7 +92,7 @@ func (t *blockingCloseRPCResultTransport) Close() error {
return nil
}
func TestRPCResultCachePublishesOnlyAfterPhysicalWrite(t *testing.T) {
func TestRPCExecutionLedgerPublishesOnlyAfterPhysicalWrite(t *testing.T) {
tr := newGatedRequiredControlTransport(nil)
s := New(Options{WriteTimeout: time.Second})
key := newTestAuthKey(t)
@ -113,7 +113,7 @@ func TestRPCResultCachePublishesOnlyAfterPhysicalWrite(t *testing.T) {
case <-time.After(time.Second):
t.Fatal("rpc_result did not reach physical writer")
}
if _, ok := s.rpcResults.Get(key.ID, c.sessionID, reqMsgID); ok {
if _, ok := s.rpcResults.Replay(key.ID, c.sessionID, reqMsgID); ok {
t.Fatal("rpc_result became completed before physical write")
}
pending, err := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
@ -280,7 +280,7 @@ func TestRouterUpdateCursorDoesNotCommitAfterPhysicalWriteFailure(t *testing.T)
deadline := time.Now().Add(time.Second)
replayable := false
for time.Now().Before(deadline) {
if cached, ok := s.rpcResults.Get(key.ID, c.sessionID, reqMsgID); ok && cached.deliveryState() == rpcResultDeliveryReplayable {
if cached, ok := s.rpcResults.Replay(key.ID, c.sessionID, reqMsgID); ok && cached.deliveryState() == rpcResultDeliveryReplayable {
replayable = true
break
}
@ -334,7 +334,7 @@ func TestRouterUpdateCursorDoesNotCommitWhenResultEncodingFails(t *testing.T) {
}
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if cached, ok := s.rpcResults.Get(key.ID, c.sessionID, reqMsgID); ok && cached.deliveryState() == rpcResultDeliveryDelivered {
if cached, ok := s.rpcResults.Replay(key.ID, c.sessionID, reqMsgID); ok && cached.deliveryState() == rpcResultDeliveryDelivered {
break
}
time.Sleep(time.Millisecond)

61
internal/mtprotoedge/rpc_result_egress_test.go
View file

@ -203,7 +203,7 @@ func (h *saturatedSlotWaveRPC) Dispatch(context.Context, [8]byte, int64, *bin.Bu
func (*saturatedSlotWaveRPC) NegotiatedLayer([8]byte, int64) (int, bool) { return 227, true }
func TestPublishRPCResultSaturatedBudgetRetainsExactResultsAcrossSlotWaves(t *testing.T) {
func TestPublishRPCResultSaturatedBudgetLeavesExecutionTombstonesAcrossSlotWaves(t *testing.T) {
slotCount := cap(outboundEncodeSlots)
requestCount := slotCount*2 + 1
gate := &saturatedSlotWaveGate{
@ -214,7 +214,7 @@ func TestPublishRPCResultSaturatedBudgetRetainsExactResultsAcrossSlotWaves(t *te
handler := &saturatedSlotWaveRPC{gate: gate}
s := New(Options{legacyRPC: handler})
now := time.Unix(1_700_000_000, 0)
s.rpcResults = newRPCResultCacheWithFlightLimit(func() time.Time { return now }, requestCount+1)
s.rpcResults = newRPCExecutionLedgerForServerTest(s, func() time.Time { return now }, requestCount+1)
const primaryMax = 1 << 20
primary := newOutboundTrackedBudget(primaryMax)
@ -302,7 +302,6 @@ func TestPublishRPCResultSaturatedBudgetRetainsExactResultsAcrossSlotWaves(t *te
t.Fatalf("primary budget changed under saturation = %d, want %d", got, primaryMax)
}
var completedBytes int64
for i, c := range conns {
if !errors.Is(errs[i], ErrOutboundTrackedBudget) {
t.Fatalf("publish request %d error = %v, want terminal budget saturation", i, errs[i])
@ -311,49 +310,31 @@ func TestPublishRPCResultSaturatedBudgetRetainsExactResultsAcrossSlotWaves(t *te
t.Fatalf("request %d connection was not explicitly fenced", i)
}
if !owners[i].handedOff.Load() {
t.Fatalf("request %d owner was not handed to completed cache", i)
t.Fatalf("request %d owner was not handed to execution ledger", i)
}
cached, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgIDs[i])
if !ok || cached == nil {
t.Fatalf("request %d exact result missing from completed cache", i)
if cached, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgIDs[i]); ok || cached != nil {
t.Fatalf("request %d retained a payload despite outbox saturation", i)
}
completedBytes += int64(len(cached.body))
var envelope proto.Result
if err := envelope.Decode(&bin.Buffer{Buf: cached.body}); err != nil {
t.Fatalf("decode request %d cached rpc_result: %v", i, err)
}
if envelope.RequestMessageID != reqMsgIDs[i] {
t.Fatalf("request %d cached req_msg_id = %d, want %d", i, envelope.RequestMessageID, reqMsgIDs[i])
}
var result tg.DataJSON
if err := result.Decode(&bin.Buffer{Buf: envelope.Result}); err != nil {
t.Fatalf("decode request %d exact business result (possibly INTERNAL): %v", i, err)
}
if result.Data != saturatedSlotWaveResultData {
t.Fatalf("request %d cached result = %q, want %q", i, result.Data, saturatedSlotWaveResultData)
}
retry, err := s.rpcResults.Acquire(c.authKeyID, c.sessionID, reqMsgIDs[i])
if err != nil || retry.state != rpcResultAcquireCompleted || retry.encoded != cached {
t.Fatalf("retry request %d = %+v err=%v, want exact completed result", i, retry, err)
if _, err := s.rpcResults.Acquire(c.authKeyID, c.sessionID, reqMsgIDs[i]); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("retry request %d err=%v, want execution tombstone capacity", i, err)
}
}
if got := handler.calls.Load(); got != int32(requestCount) {
t.Fatalf("business executions after retries = %d, want unchanged %d", got, requestCount)
}
if got := s.rpcResults.completedBytes.snapshot(); got != completedBytes {
t.Fatalf("completed-cache charge = %d, want exact retained bytes %d", got, completedBytes)
if got := s.rpcResults.receiptBudgetBytes(); got != int64(requestCount*rpcExecutionReceiptBudgetBytes) {
t.Fatalf("receipt budget = %d, want %d fixed metadata bytes", got, requestCount*rpcExecutionReceiptBudgetBytes)
}
// Expiry is the completed cache's ownership release point. Force it
// deterministically and prove every retained byte is returned exactly once.
now = now.Add(rpcResultCacheTTL + time.Second)
// Expiry releases only execution receipts; no result body was retained.
now = now.Add(rpcExecutionReceiptTTL + time.Second)
for i, c := range conns {
if _, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgIDs[i]); ok {
t.Fatalf("request %d remained cached after forced expiry", i)
if _, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgIDs[i]); ok {
t.Fatalf("request %d remained replayable after forced expiry", i)
}
}
if got := s.rpcResults.completedBytes.snapshot(); got != 0 {
t.Fatalf("completed-cache bytes after expiry = %d, want 0", got)
if got := s.rpcResults.receiptBudgetBytes(); got != 0 {
t.Fatalf("receipt budget after expiry = %d, want 0", got)
}
primary.release(primaryMax)
if got := primary.snapshot(); got != 0 {
@ -398,7 +379,7 @@ func TestCachedReplayRestoreIsSynchronousAndIndependentOfGlobalHookExecutor(t *t
})
var restored atomic.Bool
if err := s.sendCachedRPCResultWithHook(context.Background(), c, encoded, func() error {
if err := s.sendReplayedRPCResultWithHook(context.Background(), c, encoded, func() error {
if got := len(transport.snapshot()); got != 1 {
return errors.New("replay restore ran before physical write")
}
@ -672,7 +653,7 @@ func TestWrappedConvergenceMethodDrivesEgressAndReplayPriority(t *testing.T) {
var cached *encodedOutboundMessage
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if got, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID); ok {
if got, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgID); ok {
cached = got
break
}
@ -725,7 +706,7 @@ func TestRPCWorkerReleasesAfterEgressAdmissionWhileWriteBlocked(t *testing.T) {
tr.once.Do(func() { close(tr.release) })
deadline := time.Now().Add(time.Second)
for {
if _, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID); ok {
if _, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgID); ok {
break
}
if time.Now().After(deadline) {
@ -754,7 +735,7 @@ func TestDeliveryHookRunsOnceAfterReplayNotFailedWrite(t *testing.T) {
deadline := time.Now().Add(time.Second)
var cached *encodedOutboundMessage
for time.Now().Before(deadline) {
if got, ok := s.rpcResults.Get(oldConn.authKeyID, oldConn.sessionID, reqMsgID); ok {
if got, ok := s.rpcResults.Replay(oldConn.authKeyID, oldConn.sessionID, reqMsgID); ok {
cached = got
break
}
@ -772,7 +753,7 @@ func TestDeliveryHookRunsOnceAfterReplayNotFailedWrite(t *testing.T) {
replayTransport := &failAfterTransport{}
replayConn := newOutboundTestConn(t, replayTransport, newOutboundTrackedBudget(1<<20))
if err := s.sendCachedRPCResult(context.Background(), replayConn, cached); err != nil {
if err := s.sendReplayedRPCResult(context.Background(), replayConn, cached); err != nil {
t.Fatalf("replay result: %v", err)
}
deadline = time.Now().Add(time.Second)
@ -789,7 +770,7 @@ func TestDeliveryHookRunsOnceAfterReplayNotFailedWrite(t *testing.T) {
cached.delivery.coordinator.hookState() != rpcResultDeliveryHookDone {
t.Fatal("successful replay did not complete shared delivery coordinator")
}
if err := s.sendCachedRPCResult(context.Background(), replayConn, cached); err != nil {
if err := s.sendReplayedRPCResult(context.Background(), replayConn, cached); err != nil {
t.Fatalf("second replay: %v", err)
}
time.Sleep(20 * time.Millisecond)

2
internal/mtprotoedge/rpc_result_retention_budget_test.go
View file

@ -80,7 +80,7 @@ func TestCachedRPCResultReplayUsesPreReservedBodyWithoutDoubleCharge(t *testing.
done := make(chan error, 1)
go func() {
done <- s.sendCachedRPCResult(context.Background(), c, encoded)
done <- s.sendReplayedRPCResult(context.Background(), c, encoded)
}()
select {
case <-tr.started:

28
internal/mtprotoedge/rpc_result_subscriber_budget.go
View file

@ -27,8 +27,8 @@ type rpcResultSubscriberBudget struct {
global rpcResultFlightLimit
authLimit int64
sessionLimit int64
authShards [rpcResultBudgetShards]rpcResultSubscriberBudgetShard[[8]byte]
sessionShards [rpcResultBudgetShards]rpcResultSubscriberBudgetShard[rpcResultSessionBudgetKey]
authShards [rpcExecutionBudgetShards]rpcResultSubscriberBudgetShard[[8]byte]
sessionShards [rpcExecutionBudgetShards]rpcResultSubscriberBudgetShard[rpcExecutionSessionBudgetKey]
}
func newRPCResultSubscriberBudget(
@ -43,25 +43,25 @@ func newRPCResultSubscriberBudget(
b.global.max = int64(globalLimit)
for i := range b.authShards {
b.authShards[i].usage = make(map[[8]byte]int64)
b.sessionShards[i].usage = make(map[rpcResultSessionBudgetKey]int64)
b.sessionShards[i].usage = make(map[rpcExecutionSessionBudgetKey]int64)
}
return b
}
func (b *rpcResultSubscriberBudget) reserve(key rpcResultCacheKey, slots int) bool {
func (b *rpcResultSubscriberBudget) reserve(key rpcExecutionKey, slots int) bool {
if b == nil || slots <= 0 {
return false
}
authShard := b.authShard(key.authKeyID)
sessionKey := rpcResultSessionBudgetKey{authKeyID: key.authKeyID, sessionID: key.sessionID}
sessionKey := rpcExecutionSessionBudgetKey{authKeyID: key.authKeyID, sessionID: key.sessionID}
sessionShard := b.sessionShard(sessionKey)
delta := int64(slots)
authShard.mu.Lock()
sessionShard.mu.Lock()
authUsed := authShard.usage[key.authKeyID]
sessionUsed := sessionShard.usage[sessionKey]
if !withinRPCResultBudget(authUsed, delta, b.authLimit) ||
!withinRPCResultBudget(sessionUsed, delta, b.sessionLimit) ||
if !withinRPCExecutionBudget(authUsed, delta, b.authLimit) ||
!withinRPCExecutionBudget(sessionUsed, delta, b.sessionLimit) ||
!b.global.reserveN(delta) {
sessionShard.mu.Unlock()
authShard.mu.Unlock()
@ -74,12 +74,12 @@ func (b *rpcResultSubscriberBudget) reserve(key rpcResultCacheKey, slots int) bo
return true
}
func (b *rpcResultSubscriberBudget) release(key rpcResultCacheKey, slots int) {
func (b *rpcResultSubscriberBudget) release(key rpcExecutionKey, slots int) {
if b == nil || slots <= 0 {
panic("mtproto rpc result subscriber release must be positive")
}
authShard := b.authShard(key.authKeyID)
sessionKey := rpcResultSessionBudgetKey{authKeyID: key.authKeyID, sessionID: key.sessionID}
sessionKey := rpcExecutionSessionBudgetKey{authKeyID: key.authKeyID, sessionID: key.sessionID}
sessionShard := b.sessionShard(sessionKey)
delta := int64(slots)
authShard.mu.Lock()
@ -123,7 +123,7 @@ func (b *rpcResultSubscriberBudget) sessionSnapshot(authKeyID [8]byte, sessionID
if b == nil {
return 0
}
key := rpcResultSessionBudgetKey{authKeyID: authKeyID, sessionID: sessionID}
key := rpcExecutionSessionBudgetKey{authKeyID: authKeyID, sessionID: sessionID}
shard := b.sessionShard(key)
shard.mu.Lock()
used := shard.usage[key]
@ -134,16 +134,16 @@ func (b *rpcResultSubscriberBudget) sessionSnapshot(authKeyID [8]byte, sessionID
func (b *rpcResultSubscriberBudget) authShard(
authKeyID [8]byte,
) *rpcResultSubscriberBudgetShard[[8]byte] {
index := maphash.Bytes(b.seed, authKeyID[:]) & (rpcResultBudgetShards - 1)
index := maphash.Bytes(b.seed, authKeyID[:]) & (rpcExecutionBudgetShards - 1)
return &b.authShards[index]
}
func (b *rpcResultSubscriberBudget) sessionShard(
key rpcResultSessionBudgetKey,
) *rpcResultSubscriberBudgetShard[rpcResultSessionBudgetKey] {
key rpcExecutionSessionBudgetKey,
) *rpcResultSubscriberBudgetShard[rpcExecutionSessionBudgetKey] {
var raw [16]byte
copy(raw[:8], key.authKeyID[:])
binary.LittleEndian.PutUint64(raw[8:], uint64(key.sessionID))
index := maphash.Bytes(b.seed, raw[:]) & (rpcResultBudgetShards - 1)
index := maphash.Bytes(b.seed, raw[:]) & (rpcExecutionBudgetShards - 1)
return &b.sessionShards[index]
}

12
internal/mtprotoedge/rpc_rewrap.go
View file

@ -446,7 +446,7 @@ func (a *rpcRewrapAlias) storeResultOnce(s *Server, encoded *encodedOutboundMess
if a == nil || s == nil || encoded == nil || !a.resultStoreClaimed.CompareAndSwap(false, true) {
return
}
s.storeRPCResult(a.conn, a.newReqID, encoded)
s.completeRPCResult(a.conn, a.newReqID, encoded, true)
}
func claimRPCRewrapLogicalHook(
@ -465,8 +465,8 @@ func claimRPCRewrapLogicalHook(
// completeDeliveredRPCRewrapResult is safe after a watchdog has already fenced
// this physical generation. The caller has independent proof that the
// retargeted bytes reached the stream; deliveredFinalizeOnce, the shared hook
// coordinator and cache publication make late/concurrent invocations converge
// while preserving replacement -> logical -> cache -> barrier order.
// coordinator and ledger publication make late/concurrent invocations converge
// while preserving replacement -> logical -> ledger -> barrier order.
func (s *Server) completeDeliveredRPCRewrapResult(
ctx context.Context,
a *rpcRewrapAlias,
@ -618,7 +618,7 @@ func (c *rpcRewrapDeliveryControl) timeout() bool {
// commit records successful physical delivery (or an already-proven retarget)
// without disarming the watchdog. The same absolute deadline covers the
// replacement/logical restore and cache/barrier terminal path; complete is the
// replacement/logical restore and ledger/barrier terminal path; complete is the
// only successful transition that stops the timer.
func (c *rpcRewrapDeliveryControl) commit() bool {
if c == nil || !c.transition(rpcRewrapJobRunning, rpcRewrapJobCommitted) {
@ -840,7 +840,7 @@ func (s *Server) failRPCRewrapResultJob(
publish := a.newOwner.HandOff()
encoded.markReplayable()
encoded.releaseDeferredLogicalDeliveryHook()
// Release the connection-local scheduler before any defensive cache panic;
// Release the connection-local scheduler before any defensive ledger panic;
// the physical generation is already fenced, so no following task can run.
a.releaseReplayRestoreBarrier()
if publish {
@ -1093,7 +1093,7 @@ func (s *Server) publishRewrappedRPCResult(
}
ctx, cancel := context.WithDeadline(context.Background(), deadline)
defer cancel()
// attachRPCRewrapReplayPreparation froze cache-copied scheduling metadata
// attachRPCRewrapReplayPreparation froze replay-copied scheduling metadata
// before the watchdog started. The delivery path must not mutate plain fields
// that a concurrent timeout publication can copy into the replay ledger.
encoded.markQueued()

58
internal/mtprotoedge/rpc_rewrap_test.go
View file

@ -165,7 +165,7 @@ func TestRPCRewrapFailedSourceAttemptPhysicallyDeliversAliasOnce(t *testing.T) {
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
cached, ok := s.rpcResults.Get(key.ID, sessionID, newReqID)
cached, ok := s.rpcResults.Replay(key.ID, sessionID, newReqID)
if ok && cached.deliveryState() == rpcResultDeliveryDelivered && hooks.Load() == 1 {
if got := cached.writtenRequestID(); got != newReqID {
t.Fatalf("alias physical request ID = %d, want %d", got, newReqID)
@ -173,7 +173,7 @@ func TestRPCRewrapFailedSourceAttemptPhysicallyDeliversAliasOnce(t *testing.T) {
if got := len(aliasTransport.snapshot()); got != 1 {
t.Fatalf("alias physical writes = %d, want 1", got)
}
sourceCached, sourceOK := s.rpcResults.Get(key.ID, sessionID, oldReqID)
sourceCached, sourceOK := s.rpcResults.Replay(key.ID, sessionID, oldReqID)
if !sourceOK || sourceCached.deliveryState() != rpcResultDeliveryReplayable {
t.Fatalf("source physical attempt = cached:%v state:%d, want replayable", sourceOK, sourceCached.deliveryState())
}
@ -181,13 +181,13 @@ func TestRPCRewrapFailedSourceAttemptPhysicallyDeliversAliasOnce(t *testing.T) {
}
time.Sleep(time.Millisecond)
}
cached, ok := s.rpcResults.Get(key.ID, sessionID, newReqID)
cached, ok := s.rpcResults.Replay(key.ID, sessionID, newReqID)
t.Fatalf("alias result = cached:%v state:%v hooks:%d writes:%d", ok, cached.deliveryState(), hooks.Load(), len(aliasTransport.snapshot()))
}
func TestRPCRewrapRepeatedReplacementSubscriberCapacityStaysBounded(t *testing.T) {
s := New(Options{WriteTimeout: time.Second})
s.rpcResults = newRPCResultSubscriberTestCache(2, 2, 2, 2)
s.rpcResults = newRPCExecutionSubscriberTestLedger(2, 2, 2, 2)
transport := &collectingSessionTransport{}
key := newTestAuthKey(t)
const sessionID, oldReqID, newReqID = int64(188), int64(15101), int64(15201)
@ -289,7 +289,7 @@ func TestRPCRewrapRetargetFailureRequiresReplacementAliasWrite(t *testing.T) {
deadline := time.Now().Add(2 * time.Second)
var aliasCached *encodedOutboundMessage
for time.Now().Before(deadline) {
if cached, ok := s.rpcResults.Get(key.ID, sessionID, newReqID); ok && cached.deliveryState() == rpcResultDeliveryReplayable {
if cached, ok := s.rpcResults.Replay(key.ID, sessionID, newReqID); ok && cached.deliveryState() == rpcResultDeliveryReplayable {
aliasCached = cached
break
}
@ -306,7 +306,7 @@ func TestRPCRewrapRetargetFailureRequiresReplacementAliasWrite(t *testing.T) {
replacement := s.newConn(replacementTransport, key, sessionID, 1)
legacyCanonicalTestConn(t, replacement)
t.Cleanup(replacement.ForceClose)
if err := s.sendCachedRPCResult(context.Background(), replacement, aliasCached); err != nil {
if err := s.sendReplayedRPCResult(context.Background(), replacement, aliasCached); err != nil {
t.Fatalf("replacement alias replay: %v", err)
}
deadline = time.Now().Add(time.Second)
@ -319,7 +319,7 @@ func TestRPCRewrapRetargetFailureRequiresReplacementAliasWrite(t *testing.T) {
}
func TestRPCResultWaiterSubscribeIsEventDriven(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 8)
cache := newRPCExecutionLedgerForTest(time.Now, 8)
claim, err := cache.Acquire([8]byte{1}, 2, 3)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("Acquire owner = %+v, %v", claim, err)
@ -337,14 +337,14 @@ func TestRPCResultWaiterSubscribeIsEventDriven(t *testing.T) {
t.Fatal("Subscribe waited for or fabricated a result")
}
encoded := &encodedOutboundMessage{typeID: mt.RPCResultTypeID, body: make([]byte, 16), reqMsgID: 3}
cache.Put([8]byte{1}, 2, 3, encoded)
cache.completeReplayableForTest([8]byte{1}, 2, 3, encoded)
if !called.Load() {
t.Fatal("completion event did not invoke subscriber")
}
}
func TestRPCResultOwnerAbortHookInstallationIsFlightBound(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 8)
cache := newRPCExecutionLedgerForTest(time.Now, 8)
claim, err := cache.Acquire([8]byte{2}, 3, 4)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("Acquire owner = %+v, %v", claim, err)
@ -362,7 +362,7 @@ func TestRPCResultOwnerAbortHookInstallationIsFlightBound(t *testing.T) {
}
func TestRPCRewrapRegistryIsPlatformAgnosticAndAckBound(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 8)
cache := newRPCExecutionLedgerForTest(time.Now, 8)
claim, err := cache.Acquire([8]byte{4}, 5, 6)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("Acquire owner = %+v, %v", claim, err)
@ -451,7 +451,7 @@ func TestInitRewrapAfterWritingReplaysWithoutBusinessExecution(t *testing.T) {
deadline := time.Now().Add(2 * time.Second)
var replayed *encodedOutboundMessage
for time.Now().Before(deadline) {
if got, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, newReqID); ok {
if got, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, newReqID); ok {
replayed = got
break
}
@ -543,7 +543,7 @@ func TestInitRewrapAliasesExecutionAndRetargetsQueuedResult(t *testing.T) {
)
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if aliased, ok = s.rpcResults.Get(c.authKeyID, c.sessionID, newReqID); ok {
if aliased, ok = s.rpcResults.Replay(c.authKeyID, c.sessionID, newReqID); ok {
break
}
time.Sleep(time.Millisecond)
@ -609,8 +609,8 @@ func TestRPCRewrapDeliveryJobPanicAndDeadlineReleaseBarrier(t *testing.T) {
}
}
func TestExpiredRPCRewrapResultJobPublishesCompletedAliasExactlyOnce(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
func TestExpiredRPCRewrapResultJobPublishesUnavailableAliasExactlyOnce(t *testing.T) {
cache := newRPCExecutionLedgerForTest(time.Now, 1)
s := &Server{log: zaptest.NewLogger(t), rpcResults: cache}
c := &Conn{
metrics: NopMetrics{},
@ -668,27 +668,22 @@ func TestExpiredRPCRewrapResultJobPublishesCompletedAliasExactlyOnce(t *testing.
t.Fatalf("pending result flights = %d, want 0", used)
}
completed, ok := cache.Get(c.authKeyID, c.sessionID, reqMsgID)
if !ok || completed != encoded {
t.Fatalf("completed aliased result = (%p, %v), want (%p, true)", completed, ok, encoded)
completed, ok := cache.Replay(c.authKeyID, c.sessionID, reqMsgID)
if ok || completed != nil {
t.Fatalf("failed alias retained payload = (%p, %v)", completed, ok)
}
replay, err := cache.Acquire(c.authKeyID, c.sessionID, reqMsgID)
if err != nil {
t.Fatalf("reacquire completed aliased result: %v", err)
}
if replay.state != rpcResultAcquireCompleted || replay.encoded != encoded ||
!replay.executionKnown || !replay.executionOK {
t.Fatalf("completed aliased result metadata = %#v", replay)
if _, err := cache.Acquire(c.authKeyID, c.sessionID, reqMsgID); !errors.Is(err, ErrRPCResultFlightCapacity) {
t.Fatalf("reacquire unavailable aliased result = %v, want capacity", err)
}
// A defensive duplicate failure report must not republish or underflow the
// completed flight. The first handoff/cache completion is the sole winner.
// completed flight. The first handoff/ledger completion is the sole winner.
s.failRPCRewrapResultJob(alias, encoded, context.DeadlineExceeded)
if used := cache.flightLimit.snapshot(); used != 0 {
t.Fatalf("pending result flights after duplicate failure = %d, want 0", used)
}
if got, ok := cache.Get(c.authKeyID, c.sessionID, reqMsgID); !ok || got != encoded {
t.Fatalf("completed result changed after duplicate failure = (%p, %v)", got, ok)
if got, ok := cache.Replay(c.authKeyID, c.sessionID, reqMsgID); ok || got != nil {
t.Fatalf("unavailable result changed after duplicate failure = (%p, %v)", got, ok)
}
}
@ -788,7 +783,7 @@ func TestRetargetedRPCRestoreIsOrderedAndIndependentOfGlobalHookExecutor(t *test
if pending != 0 {
t.Fatalf("retarget restore barriers = %d, want 0", pending)
}
if _, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, newReqID); !ok {
if _, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, newReqID); !ok {
t.Fatal("retargeted result was not cached under new req_msg_id")
}
}
@ -980,7 +975,7 @@ func TestRPCRewrapPhysicalSuccessAfterWatchdogStillRunsLogicalRestore(t *testing
}
func TestConcurrentRPCRewrapDeliveredFinalizationPublishesOnceWithMetadata(t *testing.T) {
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
cache := newRPCExecutionLedgerForTest(time.Now, 1)
s := &Server{log: zaptest.NewLogger(t), rpcResults: cache}
c := &Conn{
metrics: NopMetrics{},
@ -1006,6 +1001,7 @@ func TestConcurrentRPCRewrapDeliveredFinalizationPublishesOnceWithMetadata(t *te
encoded := encodedRPCResultForPriorityTest(reqMsgID, 0)
encoded.delivery = claim.owner.Delivery()
encoded.setDeliveryHook(func() { logical.Add(1) })
cache.replayStore.(*rpcReplayStoreForTest).put(c.authKeyID, c.sessionID, reqMsgID, encoded)
alias := &rpcRewrapAlias{
conn: c, newReqID: reqMsgID, method: "help.getConfig", newOwner: claim.owner,
afterSuccessfulDelivery: func() error {
@ -1036,7 +1032,7 @@ func TestConcurrentRPCRewrapDeliveredFinalizationPublishesOnceWithMetadata(t *te
t.Fatalf("logical finalizations = %d, want 1", got)
}
if got := subscribers.Load(); got != 1 {
t.Fatalf("cache subscriber calls = %d, want 1", got)
t.Fatalf("ledger subscriber calls = %d, want 1", got)
}
replay, err := cache.Acquire(c.authKeyID, c.sessionID, reqMsgID)
if err != nil || replay.state != rpcResultAcquireCompleted ||
@ -1109,7 +1105,7 @@ func TestRPCRewrapSubscriberPanicCannotLoseClaimedLogicalHook(t *testing.T) {
if pending != 0 {
t.Fatalf("restore barriers after subscriber panic = %d, want 0", pending)
}
if cached, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID); !ok ||
if cached, ok := s.rpcResults.Replay(c.authKeyID, c.sessionID, reqMsgID); !ok ||
cached.delivery != encoded.delivery || !bytes.Equal(cached.body, encoded.body) || cached.replayMsgID == 0 {
t.Fatalf("logical outbox result after subscriber panic = (%p, %v), source=%p", cached, ok, encoded)
}

66
internal/mtprotoedge/runtime_metrics.go
View file

@ -7,34 +7,35 @@ package mtprotoedge
// concurrent transition; every individual budget/count remains internally
// consistent.
type RuntimeSnapshot struct {
RawConnections int64
RawConnectionLimit int64
Handshakes int64
HandshakeLimit int64
ActiveSessions int64
ProvisionalSessions int64
LogicalSessions int64
OfflineLogicalSessions int64
LogicalOutboxFrames int64
LogicalOutboxBytes int64
PendingPushBytes int64
InboundRPCTasks int64
InboundRPCBytes int64
InboundRPCReadyConnections int64
InboundRPCMaxTasks int64
InboundRPCMaxBytes int64
InboundFrameBytes int64
InboundFrameMaxBytes int64
OutboundTrackedBytes int64
OutboundTrackedMaxBytes int64
OutboundControlBytes int64
OutboundControlMaxBytes int64
OutboundWriteBytes int64
OutboundWriteMaxBytes int64
RPCResultOwners int64
RPCResultReceipts int64
RPCResultReceiptBytes int64
RPCResultSubscribers int64
RawConnections int64
RawConnectionLimit int64
Handshakes int64
HandshakeLimit int64
ActiveSessions int64
ProvisionalSessions int64
LogicalSessions int64
OfflineLogicalSessions int64
LogicalOutboxFrames int64
LogicalOutboxBytes int64
PendingPushBytes int64
InboundRPCTasks int64
InboundRPCBytes int64
InboundRPCReadyConnections int64
InboundRPCMaxTasks int64
InboundRPCMaxBytes int64
InboundFrameBytes int64
InboundFrameMaxBytes int64
OutboundTrackedBytes int64
OutboundTrackedMaxBytes int64
OutboundControlBytes int64
OutboundControlMaxBytes int64
OutboundWriteBytes int64
OutboundWriteMaxBytes int64
RPCExecutionOwners int64
RPCExecutionReservedEntries int64
RPCExecutionReceipts int64
RPCExecutionReceiptBudgetBytes int64
RPCExecutionSubscribers int64
}
type sessionManagerRuntimeSnapshot struct {
@ -175,11 +176,12 @@ func (s *Server) RuntimeSnapshot() RuntimeSnapshot {
result.OutboundWriteMaxBytes = s.outboundScratchPool.budget.maxBytes
}
if s.rpcResults != nil {
result.RPCResultOwners = s.rpcResults.flightLimit.snapshot()
result.RPCResultReceipts = s.rpcResults.completedEntries.snapshot()
result.RPCResultReceiptBytes = s.rpcResults.completedBytes.snapshot()
result.RPCExecutionOwners = s.rpcResults.flightLimit.snapshot()
result.RPCExecutionReservedEntries = s.rpcResults.reservedEntries.snapshot()
result.RPCExecutionReceipts = s.rpcResults.receiptCount.Load()
result.RPCExecutionReceiptBudgetBytes = s.rpcResults.receiptBudgetBytes()
if s.rpcResults.subscriberBudget != nil {
result.RPCResultSubscribers = s.rpcResults.subscriberBudget.global.snapshot()
result.RPCExecutionSubscribers = s.rpcResults.subscriberBudget.global.snapshot()
}
}
return result

32
internal/mtprotoedge/runtime_metrics_test.go
View file

@ -1,6 +1,9 @@
package mtprotoedge
import "testing"
import (
"testing"
"time"
)
func TestRuntimeSnapshotIsNilSafeAndReportsConfiguredLimits(t *testing.T) {
if got := (*Server)(nil).RuntimeSnapshot(); got != (RuntimeSnapshot{}) {
@ -25,3 +28,30 @@ func TestRuntimeSnapshotIsNilSafeAndReportsConfiguredLimits(t *testing.T) {
t.Fatalf("fresh server reported live ownership: %#v", snapshot)
}
}
func TestRuntimeSnapshotSeparatesExecutionOwnersReceiptsAndBudget(t *testing.T) {
server := New(Options{})
server.rpcResults = newRPCExecutionLedgerForTest(time.Now, 4)
auth := [8]byte{1, 2, 3, 4}
claim, err := server.rpcResults.Acquire(auth, 5, 6)
if err != nil || claim.state != rpcResultAcquireOwner {
t.Fatalf("owner = %#v, %v", claim, err)
}
pending := server.RuntimeSnapshot()
if pending.RPCExecutionOwners != 1 || pending.RPCExecutionReservedEntries != 1 ||
pending.RPCExecutionReceipts != 0 || pending.RPCExecutionReceiptBudgetBytes != 0 {
t.Fatalf("pending execution snapshot = %#v", pending)
}
server.rpcResults.completeReplayableForTest(auth, 5, 6, &encodedOutboundMessage{body: make([]byte, 4<<20)})
completed := server.RuntimeSnapshot()
if completed.RPCExecutionOwners != 0 || completed.RPCExecutionReservedEntries != 1 ||
completed.RPCExecutionReceipts != 1 || completed.RPCExecutionReceiptBudgetBytes != rpcExecutionReceiptBudgetBytes {
t.Fatalf("completed execution snapshot = %#v", completed)
}
server.rpcResults.Acknowledge(auth, 5, 6)
released := server.RuntimeSnapshot()
if released.RPCExecutionReservedEntries != 0 || released.RPCExecutionReceipts != 0 ||
released.RPCExecutionReceiptBudgetBytes != 0 {
t.Fatalf("released execution snapshot = %#v", released)
}
}

89
internal/mtprotoedge/server.go
View file

@ -56,7 +56,7 @@ type legacyRPCHandlerWithMethod interface {
// LayerRPCHandler is the production API-RPC boundary. Admission is a separate
// allocation-bounded phase so the edge can freeze the connection profile,
// validate wrapper dependencies and establish exact request identity before
// flight/cache/scheduler ownership is acquired.
// execution-ledger/scheduler ownership is acquired.
type LayerRPCHandler interface {
AdmitLayer(profile tlprofile.Profile, b *bin.Buffer, limits tlprofile.Limits) (tlprofile.Admission, error)
AdmitUnprofiled(b *bin.Buffer, limits tlprofile.Limits) (tlprofile.Admission, error)
@ -103,7 +103,7 @@ type LayerRPCSessionProfileResolver interface {
// LayerRPCOrderedSessionProfileResolver restores both the selected Layer and
// the newest invokeWithLayer client msg_id which proved it. The cursor prevents
// an old cached request replay on a replacement physical connection from
// an old retained request replay on a replacement physical connection from
// rolling the logical session back to an older profile.
type LayerRPCOrderedSessionProfileResolver interface {
NegotiatedSessionLayerEvidence(authKeyID [8]byte, sessionID int64) (layer int, msgID int64, ok bool)
@ -175,7 +175,7 @@ type LayerRPCDurableSessionProfileDeleter interface {
// LayerRPCReplayPreparer reapplies connection-local wrapper state for an
// already-executed exact request without consuming its one-shot business
// dispatch lease. The returned callback is safe to run only after a successful
// cached rpc_result reaches the replacement physical connection.
// replayed rpc_result reaches the replacement physical connection.
type LayerRPCReplayPreparer interface {
PrepareAdmittedReplay(
ctx context.Context,
@ -201,7 +201,7 @@ type LayerRPCProfileEvidenceContext interface {
// LayerRPCAdmissionProfilePublisher advances the auth-key-wide inherited
// default for fresh explicit evidence. admissionSeq is allocated once by the
// edge's exact flight owner and globally orders different MTProto sessions;
// cached joins/replays never call this hook again.
// joined/replayed requests never call this hook again.
type LayerRPCAdmissionProfilePublisher interface {
PublishAdmittedLayerProfileEvidence(
ctx context.Context,
@ -284,16 +284,16 @@ type Options struct {
// 等于 copied body；exact charge 是 typed decode 前的保守 materialization
// 上界，因此该配置不表示可并发接收 512 MiB wire body。默认 512 MiB。
RPCGlobalMaxBytes int64
// RPCResultCache*Entries bound in-flight owners and compact completed
// receipts. Exact payload bytes are not charged here: the logical-session
// RPCExecution*Entries bound in-flight owners and compact completed
// receipts. Payload bytes are not charged here: the logical-session
// outbox owns them under OutboundTrackedGlobalMaxBytes until ACK. ACK removes
// the receipt immediately; 331 seconds is only the no-ACK safety horizon.
RPCResultCacheMaxEntries int
RPCResultCacheAuthMaxEntries int
RPCResultCacheSessionMaxEntries int
// RPCResultPendingPerAuth is an additional active-owner bound, independent
RPCExecutionMaxEntries int
RPCExecutionAuthMaxEntries int
RPCExecutionSessionMaxEntries int
// RPCExecutionPendingPerAuth is an additional active-owner bound, independent
// from the retained entry limits and RPCGlobalMaxTasks. Default 2048.
RPCResultPendingPerAuth int
RPCExecutionPendingPerAuth int
// InboundFrameGlobalMaxBytes 是所有物理连接当前正在处理的 transport wire buffer
// 与最大解密 plaintext buffer 的总预算。长度前缀读取后、payload 分配前预留，默认
// 512 MiB；非正值使用默认值。
@ -332,7 +332,7 @@ type Options struct {
// generated Layer admission by configuring the canonical-only route.
legacyRPC legacyRPCHandler
// LayerRPC is the generated exact-profile production path. When configured,
// every API request must complete admission before flight/cache scheduling.
// every API request must complete admission before execution-ledger scheduling.
LayerRPC LayerRPCHandler
// Metrics 接收连接层指标。默认 NopMetrics。
Metrics Metrics
@ -389,19 +389,19 @@ func (o *Options) setDefaults() {
if o.RPCGlobalMaxBytes <= 0 {
o.RPCGlobalMaxBytes = 512 << 20
}
if o.RPCResultCacheMaxEntries == 0 {
o.RPCResultCacheMaxEntries = rpcResultCacheMaxEntries
if o.RPCExecutionMaxEntries == 0 {
o.RPCExecutionMaxEntries = rpcExecutionMaxEntries
}
if o.RPCResultCacheAuthMaxEntries == 0 {
o.RPCResultCacheAuthMaxEntries = rpcResultCacheAuthMaxEntries
if o.RPCExecutionAuthMaxEntries == 0 {
o.RPCExecutionAuthMaxEntries = rpcExecutionAuthMaxEntries
}
if o.RPCResultCacheSessionMaxEntries == 0 {
o.RPCResultCacheSessionMaxEntries = rpcResultCacheSessionMaxEntries
if o.RPCExecutionSessionMaxEntries == 0 {
o.RPCExecutionSessionMaxEntries = rpcExecutionSessionMaxEntries
}
if o.RPCResultPendingPerAuth == 0 {
o.RPCResultPendingPerAuth = rpcResultFlightMaxPendingPerAuth
if o.RPCResultPendingPerAuth > o.RPCGlobalMaxTasks {
o.RPCResultPendingPerAuth = o.RPCGlobalMaxTasks
if o.RPCExecutionPendingPerAuth == 0 {
o.RPCExecutionPendingPerAuth = rpcExecutionPendingPerAuth
if o.RPCExecutionPendingPerAuth > o.RPCGlobalMaxTasks {
o.RPCExecutionPendingPerAuth = o.RPCGlobalMaxTasks
}
}
if o.InboundFrameGlobalMaxBytes <= 0 {
@ -436,19 +436,19 @@ func (o *Options) setDefaults() {
}
}
func validateRPCResultCacheOptions(o Options) error {
if o.RPCResultCacheMaxEntries <= 0 || o.RPCResultCacheAuthMaxEntries <= 0 || o.RPCResultCacheSessionMaxEntries <= 0 {
return fmt.Errorf("rpc_result cache entry limits must be positive")
func validateRPCExecutionOptions(o Options) error {
if o.RPCExecutionMaxEntries <= 0 || o.RPCExecutionAuthMaxEntries <= 0 || o.RPCExecutionSessionMaxEntries <= 0 {
return fmt.Errorf("rpc execution ledger entry limits must be positive")
}
if o.RPCResultCacheMaxEntries < o.RPCResultCacheAuthMaxEntries ||
o.RPCResultCacheAuthMaxEntries < o.RPCResultCacheSessionMaxEntries {
return fmt.Errorf("rpc_result cache entry hierarchy must satisfy global >= auth >= session: %d/%d/%d",
o.RPCResultCacheMaxEntries, o.RPCResultCacheAuthMaxEntries, o.RPCResultCacheSessionMaxEntries)
if o.RPCExecutionMaxEntries < o.RPCExecutionAuthMaxEntries ||
o.RPCExecutionAuthMaxEntries < o.RPCExecutionSessionMaxEntries {
return fmt.Errorf("rpc execution ledger entry hierarchy must satisfy global >= auth >= session: %d/%d/%d",
o.RPCExecutionMaxEntries, o.RPCExecutionAuthMaxEntries, o.RPCExecutionSessionMaxEntries)
}
if o.RPCResultPendingPerAuth <= 0 || o.RPCResultPendingPerAuth > o.RPCGlobalMaxTasks ||
o.RPCResultPendingPerAuth > o.RPCResultCacheAuthMaxEntries {
return fmt.Errorf("rpc_result per-auth pending limit %d must be positive and <= global pending %d and auth entries %d",
o.RPCResultPendingPerAuth, o.RPCGlobalMaxTasks, o.RPCResultCacheAuthMaxEntries)
if o.RPCExecutionPendingPerAuth <= 0 || o.RPCExecutionPendingPerAuth > o.RPCGlobalMaxTasks ||
o.RPCExecutionPendingPerAuth > o.RPCExecutionAuthMaxEntries {
return fmt.Errorf("rpc execution per-auth pending limit %d must be positive and <= global pending %d and auth entries %d",
o.RPCExecutionPendingPerAuth, o.RPCGlobalMaxTasks, o.RPCExecutionAuthMaxEntries)
}
return nil
}
@ -494,7 +494,7 @@ type Server struct {
types *tmap.Map
admission *admissionController
rpcResults *rpcResultCache
rpcResults *rpcExecutionLedger
rpcRewrap *rpcRewrapRegistry
// onFrame 是测试钩子：收到一帧时回调其字节数；生产为 nil。
@ -504,8 +504,8 @@ type Server struct {
// New 创建 Server。
func New(opts Options) *Server {
opts.setDefaults()
if err := validateRPCResultCacheOptions(opts); err != nil {
panic(fmt.Sprintf("mtprotoedge: invalid result-cache options: %v", err))
if err := validateRPCExecutionOptions(opts); err != nil {
panic(fmt.Sprintf("mtprotoedge: invalid rpc execution options: %v", err))
}
conns := opts.ActiveSessions
if conns == nil {
@ -544,22 +544,19 @@ func New(opts Options) *Server {
clock: opts.Clock,
rand: opts.Rand,
types: tmap.New(tg.TypesMap(), mt.TypesMap(), proto.TypesMap()),
rpcResults: newRPCResultCacheWithFairCapacity(opts.Clock.Now, rpcResultCacheCapacity{
rpcResults: newRPCExecutionLedger(opts.Clock.Now, rpcExecutionLedgerCapacity{
maxPending: opts.RPCGlobalMaxTasks,
maxPendingPerAuth: opts.RPCResultPendingPerAuth,
globalMaxBytes: int64(opts.RPCResultCacheMaxEntries),
globalMaxEntries: opts.RPCResultCacheMaxEntries,
authMaxBytes: int64(opts.RPCResultCacheAuthMaxEntries),
authMaxEntries: opts.RPCResultCacheAuthMaxEntries,
sessionMaxBytes: int64(opts.RPCResultCacheSessionMaxEntries),
sessionMaxEntries: opts.RPCResultCacheSessionMaxEntries,
sessions: conns,
maxPendingPerAuth: opts.RPCExecutionPendingPerAuth,
globalMaxEntries: opts.RPCExecutionMaxEntries,
authMaxEntries: opts.RPCExecutionAuthMaxEntries,
sessionMaxEntries: opts.RPCExecutionSessionMaxEntries,
replayStore: conns,
}),
rpcRewrap: newRPCRewrapRegistry(opts.RPCGlobalMaxTasks),
admission: newAdmissionController(opts.MaxConnections, opts.MaxConnectionsPerIP, opts.MaxConcurrentHandshakes),
}
conns.setLogicalSessionReleaseHook(func(key sessionKey) {
server.rpcResults.forgetCompletedSession(key.authKeyID, key.sessionID)
server.rpcResults.forgetSession(key.authKeyID, key.sessionID)
})
return server
}

2
internal/mtprotoedge/session_manager.go
View file

@ -457,7 +457,7 @@ func (m *SessionManager) ApplyOrderedRawLayerForSession(
// ExplicitLayerEvidenceForAuthKey exposes live exact-session truth to
// auth.bindTempAuthKey. Router's bounded exact registry may expire while a Conn
// remains active; bind must not replace that explicit profile with a permanent
// key's inherited default merely because the cache TTL elapsed.
// key's inherited default merely because the execution-receipt TTL elapsed.
func (m *SessionManager) ExplicitLayerEvidenceForAuthKey(rawAuthKeyID [8]byte, sessionID int64) (layer int, msgID int64, ok bool) {
if m == nil || rawAuthKeyID == ([8]byte{}) || sessionID == 0 {
return 0, 0, false