refactor(mtproto): sync redesign RPC replay ownership
This commit is contained in:
parent
7e0f9d1e62
commit
ac0566f779
28 changed files with 1304 additions and 343 deletions
|
|
@ -31,13 +31,11 @@ TELESRV_MTPROTO_RPC_TIMEOUT=30s
|
||||||
TELESRV_MTPROTO_RPC_GLOBAL_WORKERS=256
|
TELESRV_MTPROTO_RPC_GLOBAL_WORKERS=256
|
||||||
TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS=8192
|
TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS=8192
|
||||||
TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES=536870912
|
TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES=536870912
|
||||||
# In-process 331s rpc_result ownership budgets: global >= auth >= session.
|
# Metadata-only rpc_result receipt budgets: global >= auth >= session. ACK deletes immediately;
|
||||||
|
# 331s is only the no-ACK horizon. Payloads live solely in the logical-session outbound budget.
|
||||||
TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES=262144
|
TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES=262144
|
||||||
TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_BYTES=67108864
|
|
||||||
TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES=32768
|
TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES=32768
|
||||||
TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_BYTES=33554432
|
|
||||||
TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES=16384
|
TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES=16384
|
||||||
TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_BYTES=16777216
|
|
||||||
TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH=2048
|
TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH=2048
|
||||||
# Process-wide in-flight transport wire + decrypted plaintext reservation.
|
# Process-wide in-flight transport wire + decrypted plaintext reservation.
|
||||||
TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES=536870912
|
TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES=536870912
|
||||||
|
|
|
||||||
|
|
@ -1417,11 +1417,8 @@ func run(logger *zap.Logger) error {
|
||||||
RPCGlobalMaxTasks: cfg.MTProtoRPCGlobalMaxTasks,
|
RPCGlobalMaxTasks: cfg.MTProtoRPCGlobalMaxTasks,
|
||||||
RPCGlobalMaxBytes: cfg.MTProtoRPCGlobalMaxBytes,
|
RPCGlobalMaxBytes: cfg.MTProtoRPCGlobalMaxBytes,
|
||||||
RPCResultCacheMaxEntries: cfg.MTProtoRPCResultCacheMaxEntries,
|
RPCResultCacheMaxEntries: cfg.MTProtoRPCResultCacheMaxEntries,
|
||||||
RPCResultCacheMaxBytes: cfg.MTProtoRPCResultCacheMaxBytes,
|
|
||||||
RPCResultCacheAuthMaxEntries: cfg.MTProtoRPCResultCacheAuthMaxEntries,
|
RPCResultCacheAuthMaxEntries: cfg.MTProtoRPCResultCacheAuthMaxEntries,
|
||||||
RPCResultCacheAuthMaxBytes: cfg.MTProtoRPCResultCacheAuthMaxBytes,
|
|
||||||
RPCResultCacheSessionMaxEntries: cfg.MTProtoRPCResultCacheSessionMaxEntries,
|
RPCResultCacheSessionMaxEntries: cfg.MTProtoRPCResultCacheSessionMaxEntries,
|
||||||
RPCResultCacheSessionMaxBytes: cfg.MTProtoRPCResultCacheSessionMaxBytes,
|
|
||||||
RPCResultPendingPerAuth: cfg.MTProtoRPCResultPendingPerAuth,
|
RPCResultPendingPerAuth: cfg.MTProtoRPCResultPendingPerAuth,
|
||||||
InboundFrameGlobalMaxBytes: cfg.MTProtoInboundFrameGlobalMaxBytes,
|
InboundFrameGlobalMaxBytes: cfg.MTProtoInboundFrameGlobalMaxBytes,
|
||||||
OutboundQueueSize: cfg.MTProtoOutboundQueueSize,
|
OutboundQueueSize: cfg.MTProtoOutboundQueueSize,
|
||||||
|
|
|
||||||
|
|
@ -35,17 +35,14 @@ This document describes every setting loaded by `internal/config`. Defaults and
|
||||||
| `TELESRV_MTPROTO_RPC_GLOBAL_WORKERS` | int / `256` | Shared fair-scheduler worker count. |
|
| `TELESRV_MTPROTO_RPC_GLOBAL_WORKERS` | int / `256` | Shared fair-scheduler worker count. |
|
||||||
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS` | int / `8192` | Process-wide scheduled/in-flight RPC task cap. |
|
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS` | int / `8192` | Process-wide scheduled/in-flight RPC task cap. |
|
||||||
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES` | int64 charge bytes / `536870912` | Process-wide reserved/queued/in-flight RPC memory charge. Exact admission reserves a conservative typed-materialization charge from wire size and grows it atomically before a nested-gzip-expanded graph is decoded; grow failure rejects the complete candidate batch. This is not an equal amount of concurrently receivable wire bytes. |
|
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES` | int64 charge bytes / `536870912` | Process-wide reserved/queued/in-flight RPC memory charge. Exact admission reserves a conservative typed-materialization charge from wire size and grows it atomically before a nested-gzip-expanded graph is decoded; grow failure rejects the complete candidate batch. This is not an equal amount of concurrently receivable wire bytes. |
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES` | int / `262144` | Global ownership entries for pending owners, completed results, and tombstones during the in-process 331-second replay window. |
|
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES` | int / `262144` | Global cap for pending owners and compact unacknowledged-result receipts. Receipts retain request identity, execution outcome, and Layer admission metadata only—never TL bodies. `msgs_ack` removes them immediately; 331 seconds is only the no-ACK safety horizon. |
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_BYTES` | int64 bytes / `67108864` | Global retained-byte budget. Owner admission reserves one byte; Put transfers it to a body or tombstone. Must be at least `16775168`. |
|
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES` | int / `32768` | Per-raw-auth owner/receipt cap; limits satisfy `global >= auth >= session`. |
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES` | int / `32768` | Per raw-auth-key ownership entries; charged together with global and session scopes. |
|
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES` | int / `16384` | Per `raw auth key + session_id` owner/receipt cap. |
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_BYTES` | int64 bytes / `33554432` | Per raw-auth-key retained bytes. Limits must satisfy `global >= auth >= session`. |
|
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES` | int / `16384` | Per `raw auth key + session_id` ownership entries. |
|
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_BYTES` | int64 bytes / `16777216` | Per `raw auth key + session_id` retained bytes; large enough for one legal outbound body. |
|
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH` | int / `2048` | Additional active-owner cap per raw auth key; no greater than global pending tasks or auth entries. |
|
| `TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH` | int / `2048` | Additional active-owner cap per raw auth key; no greater than global pending tasks or auth entries. |
|
||||||
| `TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | Process-wide reservation for transport wire bytes, maximum decrypted plaintext, and every live outer/nested gzip expansion, acquired before the corresponding payload allocation. |
|
| `TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | Process-wide reservation for transport wire bytes, maximum decrypted plaintext, and every live outer/nested gzip expansion, acquired before the corresponding payload allocation. |
|
||||||
| `TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE` | int / `128` | Per-connection normal outbound mailbox capacity. |
|
| `TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE` | int / `128` | Per-connection normal outbound mailbox capacity. |
|
||||||
| `TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE` | int / `32` | Per-connection control-message mailbox capacity. |
|
| `TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE` | int / `32` | Per-connection control-message mailbox capacity. |
|
||||||
| `TELESRV_MTPROTO_OUTBOUND_TRACKED_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | Global budget for tracked resend-pending message bodies. |
|
| `TELESRV_MTPROTO_OUTBOUND_TRACKED_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | Sole global budget for unacknowledged logical-session bodies. Reconnects reuse the same `msg_id/seq_no/body`; ACK, destroy, or six minutes offline releases it, with no second RPC cache/spool copy. |
|
||||||
| `TELESRV_MTPROTO_OUTBOUND_WRITE_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | Global budget for concurrent encrypted wire/codec/obfuscation scratch. |
|
| `TELESRV_MTPROTO_OUTBOUND_WRITE_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | Global budget for concurrent encrypted wire/codec/obfuscation scratch. |
|
||||||
|
|
||||||
Nested gzip admission adds no environment setting. Code-enforced ceilings are
|
Nested gzip admission adds no environment setting. Code-enforced ceilings are
|
||||||
|
|
|
||||||
|
|
@ -35,17 +35,14 @@
|
||||||
| `TELESRV_MTPROTO_RPC_GLOBAL_WORKERS` | int / `256` | 共享公平调度器 worker 数。 |
|
| `TELESRV_MTPROTO_RPC_GLOBAL_WORKERS` | int / `256` | 共享公平调度器 worker 数。 |
|
||||||
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS` | int / `8192` | 进程级排队与执行中的 RPC task 上限。 |
|
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS` | int / `8192` | 进程级排队与执行中的 RPC task 上限。 |
|
||||||
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES` | int64 charge bytes / `536870912` | 进程级已预留/排队/执行中 RPC 内存 charge 预算;legacy 等于 copied body,exact 是 typed decode 前按 wire 与生成对象放大计算的保守 materialization charge。nested gzip 展开后会在 decoder 分配 typed graph 前原子增长该 charge,grow 失败原子拒绝整批候选 RPC;该值不代表可并发接收同等大小的 wire body。 |
|
| `TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES` | int64 charge bytes / `536870912` | 进程级已预留/排队/执行中 RPC 内存 charge 预算;legacy 等于 copied body,exact 是 typed decode 前按 wire 与生成对象放大计算的保守 materialization charge。nested gzip 展开后会在 decoder 分配 typed graph 前原子增长该 charge,grow 失败原子拒绝整批候选 RPC;该值不代表可并发接收同等大小的 wire body。 |
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES` | int / `262144` | 331 秒进程内重放窗口中,pending owner、completed `rpc_result` 与容量 tombstone 的全局 ownership 条目上限。owner 执行前先占 1 条,转 completed 时不重复计数。 |
|
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES` | int / `262144` | pending owner 与未 ACK 的轻量结果收据全局上限。收据只存请求身份、执行结果和 Layer admission 元数据,不存 TL body;收到 `msgs_ack` 立即删除,331 秒仅是无 ACK 时的安全上限。 |
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_BYTES` | int64 bytes / `67108864` | 上述 ownership 的全局 retained-byte 上限;owner 先占 1 byte,Put 转移为真实 body 或 1-byte identity tombstone。不得低于 `16775168`(单条合法 outbound body 上限)。 |
|
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES` | int / `32768` | 单 raw auth key 的 owner/收据条目上限;必须满足 `global >= auth >= session`。 |
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES` | int / `32768` | 单 raw auth key 的 ownership 条目上限;与全局、session 层同时计费,防一个 auth key 吃满进程缓存。必须 `global >= auth >= session`。 |
|
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES` | int / `16384` | 单 `raw auth key + session_id` 的 owner/收据条目上限。 |
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_BYTES` | int64 bytes / `33554432` | 单 raw auth key retained-byte 上限;必须不低于单条合法 outbound body,且满足 byte 层级关系。 |
|
| `TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH` | int / `2048` | 单 raw auth key 的 active pending owner 附加上限;必须不大于 `RPC_GLOBAL_MAX_TASKS` 和 auth entry 上限。 |
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES` | int / `16384` | 单 `raw auth key + session_id` ownership 条目上限;不同 session 不共享该局部额度。 |
|
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_BYTES` | int64 bytes / `16777216` | 单 `raw auth key + session_id` retained-byte 上限;默认略高于单条合法 outbound body,确保空预算时任一合法结果可完整进入。 |
|
|
||||||
| `TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH` | int / `2048` | 单 raw auth key 的 active pending owner 附加上限;必须不大于 `RPC_GLOBAL_MAX_TASKS` 和 auth entry 上限。Put/Abort 都立即归还此 active 额度。 |
|
|
||||||
| `TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | transport wire、最大解密明文以及每个 live outer/nested gzip 输出的进程级在途预算,均在对应 payload 分配前预留。 |
|
| `TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | transport wire、最大解密明文以及每个 live outer/nested gzip 输出的进程级在途预算,均在对应 payload 分配前预留。 |
|
||||||
| `TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE` | int / `128` | 单连接普通 outbound mailbox 容量。 |
|
| `TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE` | int / `128` | 单连接普通 outbound mailbox 容量。 |
|
||||||
| `TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE` | int / `32` | 单连接控制消息 mailbox 容量。 |
|
| `TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE` | int / `32` | 单连接控制消息 mailbox 容量。 |
|
||||||
| `TELESRV_MTPROTO_OUTBOUND_TRACKED_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | resend pending message body 的全局预算。 |
|
| `TELESRV_MTPROTO_OUTBOUND_TRACKED_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | 所有逻辑 session 未 ACK 出站 body 的唯一全局预算。物理连接重连复用同一份 `msg_id/seq_no/body`;ACK、destroy 或离线 6 分钟回收时释放,不再另建 RPC cache/spool 副本。 |
|
||||||
| `TELESRV_MTPROTO_OUTBOUND_WRITE_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | 并发加密 wire/codec/obfuscation scratch 的全局预算。 |
|
| `TELESRV_MTPROTO_OUTBOUND_WRITE_GLOBAL_MAX_BYTES` | int64 bytes / `536870912` | 并发加密 wire/codec/obfuscation scratch 的全局预算。 |
|
||||||
|
|
||||||
nested gzip admission 不新增环境变量。代码硬限制为:每个
|
nested gzip admission 不新增环境变量。代码硬限制为:每个
|
||||||
|
|
|
||||||
|
|
@ -60,14 +60,12 @@ type Config struct {
|
||||||
MTProtoRPCGlobalWorkers int
|
MTProtoRPCGlobalWorkers int
|
||||||
MTProtoRPCGlobalMaxTasks int
|
MTProtoRPCGlobalMaxTasks int
|
||||||
MTProtoRPCGlobalMaxBytes int64
|
MTProtoRPCGlobalMaxBytes int64
|
||||||
// Pending ownership and completed rpc_result replay state share a three-level
|
// Pending ownership and compact completed receipts share three-level
|
||||||
// global/raw-auth/session budget over the full MTProto duplicate horizon.
|
// global/raw-auth/session entry accounting. Result bodies are never cached
|
||||||
|
// here; the logical-session outbox owns unacknowledged wire bytes.
|
||||||
MTProtoRPCResultCacheMaxEntries int
|
MTProtoRPCResultCacheMaxEntries int
|
||||||
MTProtoRPCResultCacheMaxBytes int64
|
|
||||||
MTProtoRPCResultCacheAuthMaxEntries int
|
MTProtoRPCResultCacheAuthMaxEntries int
|
||||||
MTProtoRPCResultCacheAuthMaxBytes int64
|
|
||||||
MTProtoRPCResultCacheSessionMaxEntries int
|
MTProtoRPCResultCacheSessionMaxEntries int
|
||||||
MTProtoRPCResultCacheSessionMaxBytes int64
|
|
||||||
MTProtoRPCResultPendingPerAuth int
|
MTProtoRPCResultPendingPerAuth int
|
||||||
// MTProtoInboundFrameGlobalMaxBytes 是 transport wire + 最大解密 plaintext 的
|
// MTProtoInboundFrameGlobalMaxBytes 是 transport wire + 最大解密 plaintext 的
|
||||||
// 进程级在途预算;frame 长度读出后、payload 分配前预留。
|
// 进程级在途预算;frame 长度读出后、payload 分配前预留。
|
||||||
|
|
@ -571,6 +569,9 @@ func Load() (Config, error) {
|
||||||
envInt64Or := fileEnv.envInt64Or
|
envInt64Or := fileEnv.envInt64Or
|
||||||
envDurationOr := fileEnv.envDurationOr
|
envDurationOr := fileEnv.envDurationOr
|
||||||
envAllowEmptyOr := fileEnv.envAllowEmptyOr
|
envAllowEmptyOr := fileEnv.envAllowEmptyOr
|
||||||
|
if err := validateStrictMTProtoCapacityEnv(fileEnv); err != nil {
|
||||||
|
return Config{}, err
|
||||||
|
}
|
||||||
|
|
||||||
publicBaseURL, err := links.ValidateBaseURL(envOr("TELESRV_PUBLIC_BASE_URL", links.DefaultPublicBaseURL))
|
publicBaseURL, err := links.ValidateBaseURL(envOr("TELESRV_PUBLIC_BASE_URL", links.DefaultPublicBaseURL))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -629,15 +630,10 @@ func Load() (Config, error) {
|
||||||
MTProtoRPCGlobalMaxTasks: envIntOr("TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS", 8192),
|
MTProtoRPCGlobalMaxTasks: envIntOr("TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS", 8192),
|
||||||
MTProtoRPCGlobalMaxBytes: envInt64Or("TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES", 512<<20),
|
MTProtoRPCGlobalMaxBytes: envInt64Or("TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES", 512<<20),
|
||||||
MTProtoRPCResultCacheMaxEntries: envIntOr("TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", 1<<18),
|
MTProtoRPCResultCacheMaxEntries: envIntOr("TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", 1<<18),
|
||||||
MTProtoRPCResultCacheMaxBytes: envInt64Or("TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_BYTES", 64<<20),
|
|
||||||
MTProtoRPCResultCacheAuthMaxEntries: envIntOr("TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES", 1<<15),
|
MTProtoRPCResultCacheAuthMaxEntries: envIntOr("TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES", 1<<15),
|
||||||
MTProtoRPCResultCacheAuthMaxBytes: envInt64Or("TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_BYTES", 32<<20),
|
|
||||||
MTProtoRPCResultCacheSessionMaxEntries: envIntOr(
|
MTProtoRPCResultCacheSessionMaxEntries: envIntOr(
|
||||||
"TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES", 1<<14,
|
"TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES", 1<<14,
|
||||||
),
|
),
|
||||||
MTProtoRPCResultCacheSessionMaxBytes: envInt64Or(
|
|
||||||
"TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_BYTES", 16<<20,
|
|
||||||
),
|
|
||||||
MTProtoRPCResultPendingPerAuth: envIntOr("TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH", 1<<11),
|
MTProtoRPCResultPendingPerAuth: envIntOr("TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH", 1<<11),
|
||||||
MTProtoInboundFrameGlobalMaxBytes: envInt64Or("TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES", 512<<20),
|
MTProtoInboundFrameGlobalMaxBytes: envInt64Or("TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES", 512<<20),
|
||||||
MTProtoOutboundQueueSize: envIntOr("TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE", 128),
|
MTProtoOutboundQueueSize: envIntOr("TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE", 128),
|
||||||
|
|
@ -1211,8 +1207,6 @@ func validateCollectibleUsernameConfig(cfg Config) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
const mtProtoRPCResultMinBytes = int64((1 << 24) - (2 << 10))
|
|
||||||
|
|
||||||
func validateRPCResultCacheConfig(cfg Config) error {
|
func validateRPCResultCacheConfig(cfg Config) error {
|
||||||
if cfg.MTProtoRPCResultCacheMaxEntries <= 0 || cfg.MTProtoRPCResultCacheAuthMaxEntries <= 0 ||
|
if cfg.MTProtoRPCResultCacheMaxEntries <= 0 || cfg.MTProtoRPCResultCacheAuthMaxEntries <= 0 ||
|
||||||
cfg.MTProtoRPCResultCacheSessionMaxEntries <= 0 {
|
cfg.MTProtoRPCResultCacheSessionMaxEntries <= 0 {
|
||||||
|
|
@ -1223,18 +1217,6 @@ func validateRPCResultCacheConfig(cfg Config) error {
|
||||||
return fmt.Errorf("MTProto rpc_result entry hierarchy must satisfy global >= auth >= session: %d/%d/%d",
|
return fmt.Errorf("MTProto rpc_result entry hierarchy must satisfy global >= auth >= session: %d/%d/%d",
|
||||||
cfg.MTProtoRPCResultCacheMaxEntries, cfg.MTProtoRPCResultCacheAuthMaxEntries, cfg.MTProtoRPCResultCacheSessionMaxEntries)
|
cfg.MTProtoRPCResultCacheMaxEntries, cfg.MTProtoRPCResultCacheAuthMaxEntries, cfg.MTProtoRPCResultCacheSessionMaxEntries)
|
||||||
}
|
}
|
||||||
if cfg.MTProtoRPCResultCacheMaxBytes < mtProtoRPCResultMinBytes ||
|
|
||||||
cfg.MTProtoRPCResultCacheAuthMaxBytes < mtProtoRPCResultMinBytes ||
|
|
||||||
cfg.MTProtoRPCResultCacheSessionMaxBytes < mtProtoRPCResultMinBytes {
|
|
||||||
return fmt.Errorf("MTProto rpc_result byte limits must each be at least %d: %d/%d/%d",
|
|
||||||
mtProtoRPCResultMinBytes, cfg.MTProtoRPCResultCacheMaxBytes,
|
|
||||||
cfg.MTProtoRPCResultCacheAuthMaxBytes, cfg.MTProtoRPCResultCacheSessionMaxBytes)
|
|
||||||
}
|
|
||||||
if cfg.MTProtoRPCResultCacheMaxBytes < cfg.MTProtoRPCResultCacheAuthMaxBytes ||
|
|
||||||
cfg.MTProtoRPCResultCacheAuthMaxBytes < cfg.MTProtoRPCResultCacheSessionMaxBytes {
|
|
||||||
return fmt.Errorf("MTProto rpc_result byte hierarchy must satisfy global >= auth >= session: %d/%d/%d",
|
|
||||||
cfg.MTProtoRPCResultCacheMaxBytes, cfg.MTProtoRPCResultCacheAuthMaxBytes, cfg.MTProtoRPCResultCacheSessionMaxBytes)
|
|
||||||
}
|
|
||||||
if cfg.MTProtoRPCGlobalMaxTasks <= 0 || cfg.MTProtoRPCResultPendingPerAuth <= 0 ||
|
if cfg.MTProtoRPCGlobalMaxTasks <= 0 || cfg.MTProtoRPCResultPendingPerAuth <= 0 ||
|
||||||
cfg.MTProtoRPCResultPendingPerAuth > cfg.MTProtoRPCGlobalMaxTasks ||
|
cfg.MTProtoRPCResultPendingPerAuth > cfg.MTProtoRPCGlobalMaxTasks ||
|
||||||
cfg.MTProtoRPCResultPendingPerAuth > cfg.MTProtoRPCResultCacheAuthMaxEntries {
|
cfg.MTProtoRPCResultPendingPerAuth > cfg.MTProtoRPCResultCacheAuthMaxEntries {
|
||||||
|
|
@ -1525,6 +1507,43 @@ func (e envSource) envIntOr(key string, def int) int {
|
||||||
return def
|
return def
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateStrictMTProtoCapacityEnv(e envSource) error {
|
||||||
|
for _, key := range []string{
|
||||||
|
"TELESRV_MTPROTO_MAX_CONNECTIONS",
|
||||||
|
"TELESRV_MTPROTO_MAX_CONNECTIONS_PER_IP",
|
||||||
|
"TELESRV_MTPROTO_MAX_CONCURRENT_HANDSHAKES",
|
||||||
|
"TELESRV_MTPROTO_RPC_MAX_INFLIGHT",
|
||||||
|
"TELESRV_MTPROTO_RPC_QUEUE_SIZE",
|
||||||
|
"TELESRV_MTPROTO_RPC_GLOBAL_WORKERS",
|
||||||
|
"TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS",
|
||||||
|
"TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES",
|
||||||
|
"TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES",
|
||||||
|
"TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES",
|
||||||
|
"TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH",
|
||||||
|
"TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE",
|
||||||
|
"TELESRV_MTPROTO_OUTBOUND_CONTROL_QUEUE_SIZE",
|
||||||
|
} {
|
||||||
|
if raw := e.envOr(key, ""); raw != "" {
|
||||||
|
if _, err := strconv.Atoi(raw); err != nil {
|
||||||
|
return fmt.Errorf("%s must be a base-10 integer: %w", key, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range []string{
|
||||||
|
"TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES",
|
||||||
|
"TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES",
|
||||||
|
"TELESRV_MTPROTO_OUTBOUND_TRACKED_GLOBAL_MAX_BYTES",
|
||||||
|
"TELESRV_MTPROTO_OUTBOUND_WRITE_GLOBAL_MAX_BYTES",
|
||||||
|
} {
|
||||||
|
if raw := e.envOr(key, ""); raw != "" {
|
||||||
|
if _, err := strconv.ParseInt(raw, 10, 64); err != nil {
|
||||||
|
return fmt.Errorf("%s must be a base-10 int64: %w", key, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (e envSource) envInt64Or(key string, def int64) int64 {
|
func (e envSource) envInt64Or(key string, def int64) int64 {
|
||||||
if v := e.envOr(key, ""); v != "" {
|
if v := e.envOr(key, ""); v != "" {
|
||||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||||
|
|
|
||||||
|
|
@ -151,11 +151,8 @@ func TestLoadMTProtoAdmissionAndRPCBudgets(t *testing.T) {
|
||||||
t.Setenv("TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS", "444")
|
t.Setenv("TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS", "444")
|
||||||
t.Setenv("TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES", "555555")
|
t.Setenv("TELESRV_MTPROTO_RPC_GLOBAL_MAX_BYTES", "555555")
|
||||||
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", "555")
|
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", "555")
|
||||||
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_BYTES", "70000000")
|
|
||||||
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES", "444")
|
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_ENTRIES", "444")
|
||||||
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_BYTES", "40000000")
|
|
||||||
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES", "333")
|
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_ENTRIES", "333")
|
||||||
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_BYTES", "20000000")
|
|
||||||
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH", "222")
|
t.Setenv("TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH", "222")
|
||||||
t.Setenv("TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES", "777777")
|
t.Setenv("TELESRV_MTPROTO_INBOUND_FRAME_GLOBAL_MAX_BYTES", "777777")
|
||||||
t.Setenv("TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE", "88")
|
t.Setenv("TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE", "88")
|
||||||
|
|
@ -177,14 +174,14 @@ func TestLoadMTProtoAdmissionAndRPCBudgets(t *testing.T) {
|
||||||
cfg.MTProtoRPCGlobalWorkers != 33 || cfg.MTProtoRPCGlobalMaxTasks != 444 || cfg.MTProtoRPCGlobalMaxBytes != 555555 {
|
cfg.MTProtoRPCGlobalWorkers != 33 || cfg.MTProtoRPCGlobalMaxTasks != 444 || cfg.MTProtoRPCGlobalMaxBytes != 555555 {
|
||||||
t.Fatalf("rpc budget config = %d/%d/%v/%d/%d/%d", cfg.MTProtoRPCMaxInflight, cfg.MTProtoRPCQueueSize, cfg.MTProtoRPCTimeout, cfg.MTProtoRPCGlobalWorkers, cfg.MTProtoRPCGlobalMaxTasks, cfg.MTProtoRPCGlobalMaxBytes)
|
t.Fatalf("rpc budget config = %d/%d/%v/%d/%d/%d", cfg.MTProtoRPCMaxInflight, cfg.MTProtoRPCQueueSize, cfg.MTProtoRPCTimeout, cfg.MTProtoRPCGlobalWorkers, cfg.MTProtoRPCGlobalMaxTasks, cfg.MTProtoRPCGlobalMaxBytes)
|
||||||
}
|
}
|
||||||
if cfg.MTProtoRPCResultCacheMaxEntries != 555 || cfg.MTProtoRPCResultCacheMaxBytes != 70000000 ||
|
if cfg.MTProtoRPCResultCacheMaxEntries != 555 ||
|
||||||
cfg.MTProtoRPCResultCacheAuthMaxEntries != 444 || cfg.MTProtoRPCResultCacheAuthMaxBytes != 40000000 ||
|
cfg.MTProtoRPCResultCacheAuthMaxEntries != 444 ||
|
||||||
cfg.MTProtoRPCResultCacheSessionMaxEntries != 333 || cfg.MTProtoRPCResultCacheSessionMaxBytes != 20000000 ||
|
cfg.MTProtoRPCResultCacheSessionMaxEntries != 333 ||
|
||||||
cfg.MTProtoRPCResultPendingPerAuth != 222 {
|
cfg.MTProtoRPCResultPendingPerAuth != 222 {
|
||||||
t.Fatalf("rpc result cache config = global:%d/%d auth:%d/%d session:%d/%d pending/auth:%d",
|
t.Fatalf("rpc result receipt config = global:%d auth:%d session:%d pending/auth:%d",
|
||||||
cfg.MTProtoRPCResultCacheMaxEntries, cfg.MTProtoRPCResultCacheMaxBytes,
|
cfg.MTProtoRPCResultCacheMaxEntries,
|
||||||
cfg.MTProtoRPCResultCacheAuthMaxEntries, cfg.MTProtoRPCResultCacheAuthMaxBytes,
|
cfg.MTProtoRPCResultCacheAuthMaxEntries,
|
||||||
cfg.MTProtoRPCResultCacheSessionMaxEntries, cfg.MTProtoRPCResultCacheSessionMaxBytes,
|
cfg.MTProtoRPCResultCacheSessionMaxEntries,
|
||||||
cfg.MTProtoRPCResultPendingPerAuth)
|
cfg.MTProtoRPCResultPendingPerAuth)
|
||||||
}
|
}
|
||||||
if cfg.MTProtoInboundFrameGlobalMaxBytes != 777777 {
|
if cfg.MTProtoInboundFrameGlobalMaxBytes != 777777 {
|
||||||
|
|
@ -204,14 +201,14 @@ func TestLoadRPCResultFairBudgetDefaults(t *testing.T) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Load: %v", err)
|
t.Fatalf("Load: %v", err)
|
||||||
}
|
}
|
||||||
if cfg.MTProtoRPCResultCacheMaxEntries != 1<<18 || cfg.MTProtoRPCResultCacheMaxBytes != 64<<20 ||
|
if cfg.MTProtoRPCResultCacheMaxEntries != 1<<18 ||
|
||||||
cfg.MTProtoRPCResultCacheAuthMaxEntries != 1<<15 || cfg.MTProtoRPCResultCacheAuthMaxBytes != 32<<20 ||
|
cfg.MTProtoRPCResultCacheAuthMaxEntries != 1<<15 ||
|
||||||
cfg.MTProtoRPCResultCacheSessionMaxEntries != 1<<14 || cfg.MTProtoRPCResultCacheSessionMaxBytes != 16<<20 ||
|
cfg.MTProtoRPCResultCacheSessionMaxEntries != 1<<14 ||
|
||||||
cfg.MTProtoRPCResultPendingPerAuth != 1<<11 {
|
cfg.MTProtoRPCResultPendingPerAuth != 1<<11 {
|
||||||
t.Fatalf("rpc_result fair defaults = global:%d/%d auth:%d/%d session:%d/%d pending/auth:%d",
|
t.Fatalf("rpc_result receipt defaults = global:%d auth:%d session:%d pending/auth:%d",
|
||||||
cfg.MTProtoRPCResultCacheMaxEntries, cfg.MTProtoRPCResultCacheMaxBytes,
|
cfg.MTProtoRPCResultCacheMaxEntries,
|
||||||
cfg.MTProtoRPCResultCacheAuthMaxEntries, cfg.MTProtoRPCResultCacheAuthMaxBytes,
|
cfg.MTProtoRPCResultCacheAuthMaxEntries,
|
||||||
cfg.MTProtoRPCResultCacheSessionMaxEntries, cfg.MTProtoRPCResultCacheSessionMaxBytes,
|
cfg.MTProtoRPCResultCacheSessionMaxEntries,
|
||||||
cfg.MTProtoRPCResultPendingPerAuth)
|
cfg.MTProtoRPCResultPendingPerAuth)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -223,8 +220,6 @@ func TestLoadRejectsInvalidRPCResultFairBudgets(t *testing.T) {
|
||||||
value string
|
value string
|
||||||
}{
|
}{
|
||||||
{name: "entry hierarchy", key: "TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", value: "1024"},
|
{name: "entry hierarchy", key: "TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", value: "1024"},
|
||||||
{name: "byte below outbound body", key: "TELESRV_MTPROTO_RPC_RESULT_CACHE_SESSION_MAX_BYTES", value: "16700000"},
|
|
||||||
{name: "byte hierarchy", key: "TELESRV_MTPROTO_RPC_RESULT_CACHE_AUTH_MAX_BYTES", value: "70000000"},
|
|
||||||
{name: "pending hierarchy", key: "TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH", value: "9000"},
|
{name: "pending hierarchy", key: "TELESRV_MTPROTO_RPC_RESULT_PENDING_PER_AUTH", value: "9000"},
|
||||||
}
|
}
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
|
|
@ -238,6 +233,27 @@ func TestLoadRejectsInvalidRPCResultFairBudgets(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadRejectsMalformedMTProtoCapacity(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
key string
|
||||||
|
value string
|
||||||
|
}{
|
||||||
|
{name: "worker tasks malformed", key: "TELESRV_MTPROTO_RPC_GLOBAL_MAX_TASKS", value: "lots"},
|
||||||
|
{name: "receipt entries overflow", key: "TELESRV_MTPROTO_RPC_RESULT_CACHE_MAX_ENTRIES", value: "999999999999999999999999"},
|
||||||
|
{name: "tracked bytes overflow", key: "TELESRV_MTPROTO_OUTBOUND_TRACKED_GLOBAL_MAX_BYTES", value: "999999999999999999999999"},
|
||||||
|
{name: "outbound queue malformed", key: "TELESRV_MTPROTO_OUTBOUND_QUEUE_SIZE", value: "many"},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
disableDefaultConfigFile(t)
|
||||||
|
t.Setenv(test.key, test.value)
|
||||||
|
if _, err := Load(); err == nil {
|
||||||
|
t.Fatalf("Load accepted %s=%q", test.key, test.value)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadOutboxPoisonPolicy(t *testing.T) {
|
func TestLoadOutboxPoisonPolicy(t *testing.T) {
|
||||||
disableDefaultConfigFile(t)
|
disableDefaultConfigFile(t)
|
||||||
t.Setenv("TELESRV_OUTBOX_POISON_RETENTION", "2m")
|
t.Setenv("TELESRV_OUTBOX_POISON_RETENTION", "2m")
|
||||||
|
|
|
||||||
|
|
@ -108,6 +108,9 @@ type Conn struct {
|
||||||
outboundControlBudgetOnce sync.Once
|
outboundControlBudgetOnce sync.Once
|
||||||
outboundScratchPool *outboundScratchPool
|
outboundScratchPool *outboundScratchPool
|
||||||
outboundScratchOnce sync.Once
|
outboundScratchOnce sync.Once
|
||||||
|
// outboundState outlives this physical Conn generation. A replacement
|
||||||
|
// physical connection for the same auth key/session reuses it.
|
||||||
|
outboundState *outboundState
|
||||||
// lifecycle is the sole monotonic activation/retirement state machine.
|
// lifecycle is the sole monotonic activation/retirement state machine.
|
||||||
// retired never transitions back to claiming/active; one atomic state avoids
|
// retired never transitions back to claiming/active; one atomic state avoids
|
||||||
// contradictory activation and shutdown observations.
|
// contradictory activation and shutdown observations.
|
||||||
|
|
@ -150,7 +153,8 @@ type Conn struct {
|
||||||
rpcRootCtx context.Context
|
rpcRootCtx context.Context
|
||||||
rpcMaxInflight int
|
rpcMaxInflight int
|
||||||
|
|
||||||
// sentContentMessages 只由 outbound actor 访问,用于生成 MTProto seq_no。
|
// sentContentMessages is retained only for standalone construction tests.
|
||||||
|
// Server connections allocate seq_no from logical-session outboundState.
|
||||||
sentContentMessages int32
|
sentContentMessages int32
|
||||||
// outboundRand 只由 outbound actor 访问:对 cipher 随机源的缓冲预读,
|
// outboundRand 只由 outbound actor 访问:对 cipher 随机源的缓冲预读,
|
||||||
// 把每帧 padding 的 getrandom syscall 摊薄成 ~1KiB 一次。
|
// 把每帧 padding 的 getrandom syscall 摊薄成 ~1KiB 一次。
|
||||||
|
|
|
||||||
|
|
@ -716,10 +716,7 @@ func (s *Server) handleInboundRPCAdmissionError(ctx context.Context, c *Conn, ms
|
||||||
zap.String("auth_key_id", c.authKeyHex),
|
zap.String("auth_key_id", c.authKeyHex),
|
||||||
zap.Int64("session_id", c.sessionID),
|
zap.Int64("session_id", c.sessionID),
|
||||||
)
|
)
|
||||||
return s.sendResult(ctx, c, msgID, &mt.RPCError{
|
return s.sendResult(ctx, c, msgID, rpcWorkerBusyError())
|
||||||
ErrorCode: 420,
|
|
||||||
ErrorMessage: "FLOOD_WAIT_1",
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
@ -853,9 +850,9 @@ var errRPCResultRetentionHandoff = errors.New("mtproto rpc result retention hand
|
||||||
type rpcResultRetentionHandoff func(*encodedOutboundMessage, error) error
|
type rpcResultRetentionHandoff func(*encodedOutboundMessage, error) error
|
||||||
|
|
||||||
// publishRPCResult ends the inbound worker's ownership at bounded egress
|
// publishRPCResult ends the inbound worker's ownership at bounded egress
|
||||||
// admission. Physical delivery is thereafter owned either by the single
|
// admission. Physical delivery is thereafter owned by the logical-session
|
||||||
// outbound actor or, under retained-byte saturation, by a fenced completed-cache
|
// outbox. Under retained-byte saturation the Conn is fenced and the receipt
|
||||||
// entry that the replacement connection can replay without rerunning business.
|
// ledger records an unavailable tombstone so business cannot rerun.
|
||||||
func (s *Server) publishRPCResult(
|
func (s *Server) publishRPCResult(
|
||||||
c *Conn,
|
c *Conn,
|
||||||
reqMsgID int64,
|
reqMsgID int64,
|
||||||
|
|
@ -893,21 +890,19 @@ func (s *Server) publishRPCResult(
|
||||||
return priority, visible
|
return priority, visible
|
||||||
}
|
}
|
||||||
|
|
||||||
// A successful business result may never leave the encode slot as an
|
// If the sole logical-outbox body budget cannot admit a completed result,
|
||||||
// unaccounted []byte. If the primary 512MiB retained-body budget is full, make
|
// fence this physical generation and publish only an execution tombstone.
|
||||||
// overload terminal for this physical generation and publish the exact result
|
// There is deliberately no fallback payload cache/spool and no business
|
||||||
// into the independently bounded completed cache before releasing the slot.
|
// re-execution hidden behind a local capacity error.
|
||||||
retainForReplay := func(encoded *encodedOutboundMessage, admissionErr error) error {
|
retainForReplay := func(encoded *encodedOutboundMessage, admissionErr error) error {
|
||||||
if s == nil || s.rpcResults == nil || c == nil || encoded == nil || reqMsgID == 0 {
|
if s == nil || s.rpcResults == nil || c == nil || encoded == nil || reqMsgID == 0 {
|
||||||
return errors.New("rpc result completed cache is unavailable")
|
return errors.New("rpc result receipt ledger is unavailable")
|
||||||
}
|
}
|
||||||
if int64(len(encoded.body)) > s.rpcResults.completedBytes.max {
|
if s.rpcResults.sessions == nil && int64(len(encoded.body)) > s.rpcResults.completedBytes.max {
|
||||||
// Every transport-legal result fits the production completed cache by the
|
// Focused legacy-cache tests may intentionally use a byte budget smaller
|
||||||
// compile-time invariant in rpc_result_cache.go. A test/custom cache that
|
// than one legal transport result. Production receipts never own bodies.
|
||||||
// violates it cannot safely complete this flight, so fail fast while the
|
|
||||||
// body is still confined to the encode slot.
|
|
||||||
panic(fmt.Sprintf(
|
panic(fmt.Sprintf(
|
||||||
"mtprotoedge: encoded rpc result exceeds completed-cache budget: body=%d max=%d",
|
"mtprotoedge: legacy encoded rpc result exceeds inline-ledger budget: body=%d max=%d",
|
||||||
len(encoded.body), s.rpcResults.completedBytes.max,
|
len(encoded.body), s.rpcResults.completedBytes.max,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
@ -929,7 +924,7 @@ func (s *Server) publishRPCResult(
|
||||||
if visible {
|
if visible {
|
||||||
resultLogLevel = zap.InfoLevel
|
resultLogLevel = zap.InfoLevel
|
||||||
}
|
}
|
||||||
if checked := s.log.Check(resultLogLevel, "RPC result retained for replay after egress saturation"); checked != nil {
|
if checked := s.log.Check(resultLogLevel, "RPC result execution fenced after egress saturation"); checked != nil {
|
||||||
checked.Write(
|
checked.Write(
|
||||||
zap.String("method", method), zap.Int64("req_msg_id", reqMsgID),
|
zap.String("method", method), zap.Int64("req_msg_id", reqMsgID),
|
||||||
zap.Int64("delivered_req_msg_id", encoded.writtenRequestID()),
|
zap.Int64("delivered_req_msg_id", encoded.writtenRequestID()),
|
||||||
|
|
@ -1070,7 +1065,7 @@ func (s *Server) sendResult(ctx context.Context, c *Conn, reqMsgID int64, result
|
||||||
}
|
}
|
||||||
|
|
||||||
// sendCachedRPCResult preserves the delivery half of the rpc_result invariant
|
// sendCachedRPCResult preserves the delivery half of the rpc_result invariant
|
||||||
// for completed-flight replays: either the cached result reaches this physical
|
// for completed-flight replays: either the logical outbox result reaches this physical
|
||||||
// byte stream, or this logical Conn is fenced so a replacement may retry it.
|
// byte stream, or this logical Conn is fenced so a replacement may retry it.
|
||||||
func (s *Server) sendCachedRPCResult(ctx context.Context, c *Conn, encoded *encodedOutboundMessage) error {
|
func (s *Server) sendCachedRPCResult(ctx context.Context, c *Conn, encoded *encodedOutboundMessage) error {
|
||||||
return s.sendCachedRPCResultWithHook(ctx, c, encoded, nil)
|
return s.sendCachedRPCResultWithHook(ctx, c, encoded, nil)
|
||||||
|
|
@ -1251,8 +1246,9 @@ func (s *Server) encodeRPCResultReservedWithHandoffContext(
|
||||||
}
|
}
|
||||||
retained = true
|
retained = true
|
||||||
// The handoff owns the only surviving pointer. Do not return a second
|
// The handoff owns the only surviving pointer. Do not return a second
|
||||||
// producer reference after the encode slot releases; the completed cache
|
// producer reference after the encode slot releases. Production handoff
|
||||||
// may independently evict the entry under its bounded policy.
|
// either transferred the body to the logical outbox or retained only an
|
||||||
|
// unavailable receipt tombstone.
|
||||||
encoded = nil
|
encoded = nil
|
||||||
return admissionErr
|
return admissionErr
|
||||||
}
|
}
|
||||||
|
|
@ -1355,6 +1351,7 @@ func (s *Server) storeRPCResult(c *Conn, reqMsgID int64, encoded *encodedOutboun
|
||||||
if s == nil || s.rpcResults == nil || c == nil {
|
if s == nil || s.rpcResults == nil || c == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
s.conns.adoptLogicalSession(c)
|
||||||
s.rpcResults.Put(c.authKeyID, c.sessionID, reqMsgID, encoded)
|
s.rpcResults.Put(c.authKeyID, c.sessionID, reqMsgID, encoded)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -434,6 +434,10 @@ func (s *Server) prepareInboundLayerRPCBatch(ctx context.Context, c *Conn, plan
|
||||||
}
|
}
|
||||||
switch claim.state {
|
switch claim.state {
|
||||||
case rpcResultAcquireCompleted:
|
case rpcResultAcquireCompleted:
|
||||||
|
// Transfer the materialization ticket to the plan before any
|
||||||
|
// profile-restore step can fail; plan.close is the universal abort
|
||||||
|
// path and sendCached releases it after outbound-budget handoff.
|
||||||
|
item.payload = claim.encoded
|
||||||
after, prepareErr := s.prepareAdmittedLayerRPCReplay(ctx, c, item.msgID, claim.admissionSeq, item.profileEvidenceFresh(), item.admitted)
|
after, prepareErr := s.prepareAdmittedLayerRPCReplay(ctx, c, item.msgID, claim.admissionSeq, item.profileEvidenceFresh(), item.admitted)
|
||||||
if prepareErr != nil {
|
if prepareErr != nil {
|
||||||
s.rpcRewrap.release(candidate)
|
s.rpcRewrap.release(candidate)
|
||||||
|
|
@ -441,10 +445,16 @@ func (s *Server) prepareInboundLayerRPCBatch(ctx context.Context, c *Conn, plan
|
||||||
}
|
}
|
||||||
s.rpcRewrap.commit(candidate)
|
s.rpcRewrap.commit(candidate)
|
||||||
item.kind = inboundItemReplayRPC
|
item.kind = inboundItemReplayRPC
|
||||||
item.payload = claim.encoded
|
|
||||||
if claim.executionKnown && claim.executionOK {
|
if claim.executionKnown && claim.executionOK {
|
||||||
item.replayAfterSuccessfulDelivery = after
|
item.replayAfterSuccessfulDelivery = after
|
||||||
}
|
}
|
||||||
|
case rpcResultAcquireAcknowledged:
|
||||||
|
// Explicit ACK is terminal proof for this request msg_id. Keep
|
||||||
|
// exact admission metadata, retire the old rewrap candidate and
|
||||||
|
// make the duplicate ACK-only without replay side effects.
|
||||||
|
s.rpcRewrap.commit(candidate)
|
||||||
|
item.kind = inboundItemDuplicate
|
||||||
|
item.payload = nil
|
||||||
case rpcResultAcquirePending:
|
case rpcResultAcquirePending:
|
||||||
after, prepareErr := s.prepareAdmittedLayerRPCReplay(ctx, c, item.msgID, claim.admissionSeq, item.profileEvidenceFresh(), item.admitted)
|
after, prepareErr := s.prepareAdmittedLayerRPCReplay(ctx, c, item.msgID, claim.admissionSeq, item.profileEvidenceFresh(), item.admitted)
|
||||||
if prepareErr != nil {
|
if prepareErr != nil {
|
||||||
|
|
@ -523,15 +533,18 @@ func (s *Server) prepareInboundLayerRPCBatch(ctx context.Context, c *Conn, plan
|
||||||
}
|
}
|
||||||
switch claim.state {
|
switch claim.state {
|
||||||
case rpcResultAcquireCompleted:
|
case rpcResultAcquireCompleted:
|
||||||
|
item.payload = claim.encoded
|
||||||
after, prepareErr := s.prepareAdmittedLayerRPCReplay(ctx, c, item.msgID, claim.admissionSeq, item.profileEvidenceFresh(), item.admitted)
|
after, prepareErr := s.prepareAdmittedLayerRPCReplay(ctx, c, item.msgID, claim.admissionSeq, item.profileEvidenceFresh(), item.admitted)
|
||||||
if prepareErr != nil {
|
if prepareErr != nil {
|
||||||
return prepareErr
|
return prepareErr
|
||||||
}
|
}
|
||||||
item.kind = inboundItemReplayRPC
|
item.kind = inboundItemReplayRPC
|
||||||
item.payload = claim.encoded
|
|
||||||
if claim.executionKnown && claim.executionOK {
|
if claim.executionKnown && claim.executionOK {
|
||||||
item.replayAfterSuccessfulDelivery = after
|
item.replayAfterSuccessfulDelivery = after
|
||||||
}
|
}
|
||||||
|
case rpcResultAcquireAcknowledged:
|
||||||
|
item.kind = inboundItemDuplicate
|
||||||
|
item.payload = nil
|
||||||
case rpcResultAcquirePending:
|
case rpcResultAcquirePending:
|
||||||
if ownersInPlan[item.msgID] != nil {
|
if ownersInPlan[item.msgID] != nil {
|
||||||
item.kind = inboundItemDuplicate
|
item.kind = inboundItemDuplicate
|
||||||
|
|
|
||||||
|
|
@ -541,7 +541,7 @@ func TestBatchProvisionalCursorKeepsRegistryWatermarkAcrossOldReplay(t *testing.
|
||||||
t.Fatalf("old owner err=%v", err)
|
t.Fatalf("old owner err=%v", err)
|
||||||
}
|
}
|
||||||
oldClaim.owner.CompleteExecution(true)
|
oldClaim.owner.CompleteExecution(true)
|
||||||
s.rpcResults.Put(authKeyID, sessionID, 100, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 100})
|
storeLogicalRPCResultForTest(t, s, c, 100, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 100})
|
||||||
|
|
||||||
nakedBody := exactOutboundLayerRPCBody(t, tlprofile.Profile227, &tg.MessagesGetHistoryRequest{
|
nakedBody := exactOutboundLayerRPCBody(t, tlprofile.Profile227, &tg.MessagesGetHistoryRequest{
|
||||||
Peer: &tg.InputPeerSelf{}, Limit: 1,
|
Peer: &tg.InputPeerSelf{}, Limit: 1,
|
||||||
|
|
@ -562,6 +562,131 @@ func TestBatchProvisionalCursorKeepsRegistryWatermarkAcrossOldReplay(t *testing.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAcknowledgedExactRPCReleasesReplayReceipt(t *testing.T) {
|
||||||
|
handler := &replayProfileCaptureLayerRPC{admissionOnlyLayerRPC: newAdmissionOnlyLayerRPC()}
|
||||||
|
s := New(Options{DC: 2, LayerRPC: handler})
|
||||||
|
authKeyID := [8]byte{0x31, 0xa1}
|
||||||
|
const sessionID, reqMsgID = int64(3191), int64(100)
|
||||||
|
body := exactLayerRPCBody(t, &tg.InvokeWithLayerRequest{
|
||||||
|
Layer: 225, Query: &tg.HelpGetConfigRequest{},
|
||||||
|
})
|
||||||
|
admitted, _, err := s.decodeInboundLayerRPC(
|
||||||
|
LayerProfileSnapshot{Profile: tlprofile.Profile225, Origin: LayerProfileExplicit}, body,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
owner, err := s.rpcResults.AcquireLayerIdentified(
|
||||||
|
authKeyID, sessionID, reqMsgID,
|
||||||
|
admitted.Call().Profile(), admitted.Prepared().Identity(),
|
||||||
|
)
|
||||||
|
if err != nil || owner.state != rpcResultAcquireOwner {
|
||||||
|
t.Fatalf("owner = %#v, err=%v", owner, err)
|
||||||
|
}
|
||||||
|
owner.owner.CompleteExecution(true)
|
||||||
|
logical := &Conn{authKeyID: authKeyID, sessionID: sessionID}
|
||||||
|
storeLogicalRPCResultForTest(t, s, logical, reqMsgID, &encodedOutboundMessage{
|
||||||
|
body: make([]byte, 32), reqMsgID: reqMsgID,
|
||||||
|
})
|
||||||
|
acknowledgeLogicalRPCResultForTest(t, s, logical, reqMsgID)
|
||||||
|
|
||||||
|
replacement := &Conn{authKeyID: authKeyID, sessionID: sessionID, metrics: NopMetrics{}}
|
||||||
|
replacement.startInboundRPCScheduler(s.rpcScheduler, 1, 8, time.Second)
|
||||||
|
defer replacement.Close()
|
||||||
|
plan := &inboundPlan{items: []inboundItem{{kind: inboundItemRPC, msgID: reqMsgID, body: body}}}
|
||||||
|
defer plan.close()
|
||||||
|
if err := s.prepareInboundLayerRPCBatch(context.Background(), replacement, plan); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if plan.items[0].kind != inboundItemRPC || plan.items[0].payload == nil {
|
||||||
|
t.Fatalf("post-ACK request = kind:%d payload:%T", plan.items[0].kind, plan.items[0].payload)
|
||||||
|
}
|
||||||
|
if len(plan.rpcTasks) != 1 || len(plan.rpcOwners) != 1 || len(plan.rewrapAliases) != 0 || plan.rpcReservation == nil {
|
||||||
|
t.Fatalf("post-ACK request scheduling: tasks=%d owners=%d aliases=%d reservation=%v",
|
||||||
|
len(plan.rpcTasks), len(plan.rpcOwners), len(plan.rewrapAliases), plan.rpcReservation != nil)
|
||||||
|
}
|
||||||
|
if profiles, known := handler.capturedProfiles(); len(profiles) != 0 || len(known) != 0 {
|
||||||
|
t.Fatalf("ACKed duplicate ran replay side effects: profiles=%v known=%v", profiles, known)
|
||||||
|
}
|
||||||
|
claim, err := s.rpcResults.AcquireLayerIdentified(
|
||||||
|
authKeyID, sessionID, reqMsgID,
|
||||||
|
admitted.Call().Profile(), admitted.Prepared().Identity(),
|
||||||
|
)
|
||||||
|
if err != nil || claim.state != rpcResultAcquirePending || claim.admissionSeq == owner.admissionSeq {
|
||||||
|
t.Fatalf("post-ACK claim after preflight = %#v, err=%v", claim, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAcknowledgedExactInitRewrapReleasesReplayReceipt(t *testing.T) {
|
||||||
|
handler := &replayProfileCaptureLayerRPC{admissionOnlyLayerRPC: newAdmissionOnlyLayerRPC()}
|
||||||
|
s := New(Options{DC: 2, LayerRPC: handler})
|
||||||
|
authKeyID := [8]byte{0x31, 0xa2}
|
||||||
|
const sessionID, oldReqID, newReqID = int64(3192), int64(100), int64(104)
|
||||||
|
c := &Conn{authKeyID: authKeyID, sessionID: sessionID, metrics: NopMetrics{}}
|
||||||
|
if err := c.SeedInheritedLayerProfile(tlprofile.Profile227); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c.startInboundRPCScheduler(s.rpcScheduler, 1, 8, time.Second)
|
||||||
|
defer c.Close()
|
||||||
|
|
||||||
|
inner := exactOutboundLayerRPCBody(t, tlprofile.Profile227, &tg.HelpGetConfigRequest{})
|
||||||
|
oldPlan := &inboundPlan{items: []inboundItem{{kind: inboundItemRPC, msgID: oldReqID, body: inner}}}
|
||||||
|
defer oldPlan.close()
|
||||||
|
if err := s.prepareInboundLayerRPCBatch(context.Background(), c, oldPlan); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(oldPlan.rpcOwners) != 1 || s.rpcRewrap.total != 1 {
|
||||||
|
t.Fatalf("old exact candidate = owners:%d candidates:%d", len(oldPlan.rpcOwners), s.rpcRewrap.total)
|
||||||
|
}
|
||||||
|
|
||||||
|
wrapped := exactLayerRPCBody(t, &tg.InvokeWithLayerRequest{
|
||||||
|
Layer: 227,
|
||||||
|
Query: &tg.InitConnectionRequest{
|
||||||
|
APIID: 6, DeviceModel: "Pixel", SystemVersion: "SDK 36", AppVersion: "12.8.1",
|
||||||
|
SystemLangCode: "en", LangPack: "android", LangCode: "en",
|
||||||
|
Query: &tg.HelpGetConfigRequest{},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
admitted, _, err := s.decodeInboundLayerRPC(
|
||||||
|
LayerProfileSnapshot{Profile: tlprofile.Profile227, Origin: LayerProfileExplicit}, wrapped,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
newOwner, err := s.rpcResults.AcquireLayerIdentified(
|
||||||
|
authKeyID, sessionID, newReqID,
|
||||||
|
admitted.Call().Profile(), admitted.Prepared().Identity(),
|
||||||
|
)
|
||||||
|
if err != nil || newOwner.state != rpcResultAcquireOwner {
|
||||||
|
t.Fatalf("new exact receipt owner = %#v, err=%v", newOwner, err)
|
||||||
|
}
|
||||||
|
newOwner.owner.CompleteExecution(true)
|
||||||
|
storeLogicalRPCResultForTest(t, s, c, newReqID, &encodedOutboundMessage{
|
||||||
|
body: make([]byte, 32), reqMsgID: newReqID,
|
||||||
|
})
|
||||||
|
acknowledgeLogicalRPCResultForTest(t, s, c, newReqID)
|
||||||
|
// In the real delivery path a successful rewrap commits its source candidate.
|
||||||
|
// Clear the synthetic pending candidate before observing post-ACK admission.
|
||||||
|
s.rpcRewrap.clearSession(c)
|
||||||
|
|
||||||
|
newPlan := &inboundPlan{items: []inboundItem{{kind: inboundItemRPC, msgID: newReqID, body: wrapped}}}
|
||||||
|
defer newPlan.close()
|
||||||
|
if err := s.prepareInboundLayerRPCBatch(context.Background(), c, newPlan); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if newPlan.items[0].kind != inboundItemRPC || newPlan.items[0].payload == nil ||
|
||||||
|
len(newPlan.rpcTasks) != 1 || len(newPlan.rpcOwners) != 1 || len(newPlan.rewrapAliases) != 0 {
|
||||||
|
t.Fatalf("post-ACK init request scheduling: kind=%d tasks=%d owners=%d aliases=%d",
|
||||||
|
newPlan.items[0].kind, len(newPlan.rpcTasks), len(newPlan.rpcOwners), len(newPlan.rewrapAliases))
|
||||||
|
}
|
||||||
|
if s.rpcRewrap.total != 0 {
|
||||||
|
t.Fatalf("ACKed exact init rewrap left %d candidates", s.rpcRewrap.total)
|
||||||
|
}
|
||||||
|
if profiles, known := handler.capturedProfiles(); len(profiles) != 0 || len(known) != 0 {
|
||||||
|
t.Fatalf("ACKed exact init rewrap ran replay side effects: profiles=%v known=%v", profiles, known)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBatchProvisionalCursorUsesPendingNewerExplicitEvidence(t *testing.T) {
|
func TestBatchProvisionalCursorUsesPendingNewerExplicitEvidence(t *testing.T) {
|
||||||
handler := newAdmissionOnlyLayerRPC()
|
handler := newAdmissionOnlyLayerRPC()
|
||||||
s := New(Options{DC: 2, LayerRPC: handler})
|
s := New(Options{DC: 2, LayerRPC: handler})
|
||||||
|
|
@ -921,7 +1046,7 @@ func TestInvariantReplayNeverCachesInternalCanonicalProfile(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
owner.CompleteExecution(true)
|
owner.CompleteExecution(true)
|
||||||
s.rpcResults.Put(authKeyID, sessionID, 100, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 100})
|
storeLogicalRPCResultForTest(t, s, original, 100, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 100})
|
||||||
if profile, ok := s.rpcResults.ExactAdmissionProfile(authKeyID, sessionID, 100); ok || profile != 0 {
|
if profile, ok := s.rpcResults.ExactAdmissionProfile(authKeyID, sessionID, 100); ok || profile != 0 {
|
||||||
t.Fatalf("completed invariant cached profile=(%d,%v)", profile, ok)
|
t.Fatalf("completed invariant cached profile=(%d,%v)", profile, ok)
|
||||||
}
|
}
|
||||||
|
|
@ -1121,7 +1246,7 @@ func TestOldCompletedLayerRequestCannotRollBackCorrectedSession(t *testing.T) {
|
||||||
t.Fatal("old request did not acquire owner")
|
t.Fatal("old request did not acquire owner")
|
||||||
}
|
}
|
||||||
oldOwner.CompleteExecution(true)
|
oldOwner.CompleteExecution(true)
|
||||||
s.rpcResults.Put(authKeyID, sessionID, 100, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 100})
|
storeLogicalRPCResultForTest(t, s, c, 100, &encodedOutboundMessage{body: []byte{1}, reqMsgID: 100})
|
||||||
|
|
||||||
correctPlan := &inboundPlan{items: []inboundItem{{
|
correctPlan := &inboundPlan{items: []inboundItem{{
|
||||||
kind: inboundItemRPC, msgID: 104,
|
kind: inboundItemRPC, msgID: 104,
|
||||||
|
|
@ -1187,7 +1312,7 @@ func TestLogicalSessionLayerWatermarkSurvivesResultExpiryAndOldContainer(t *test
|
||||||
if oldOwner == nil || !oldOwner.CompleteExecution(true) {
|
if oldOwner == nil || !oldOwner.CompleteExecution(true) {
|
||||||
t.Fatal("old Layer 225 request did not establish a completed owner")
|
t.Fatal("old Layer 225 request did not establish a completed owner")
|
||||||
}
|
}
|
||||||
s.rpcResults.Put(authKeyID, sessionID, oldMsgID, &encodedOutboundMessage{body: []byte{1}, reqMsgID: oldMsgID})
|
storeLogicalRPCResultForTest(t, s, original, oldMsgID, &encodedOutboundMessage{body: []byte{1}, reqMsgID: oldMsgID})
|
||||||
|
|
||||||
correctPlan := &inboundPlan{items: []inboundItem{{
|
correctPlan := &inboundPlan{items: []inboundItem{{
|
||||||
kind: inboundItemRPC, msgID: correctedMsgID,
|
kind: inboundItemRPC, msgID: correctedMsgID,
|
||||||
|
|
|
||||||
|
|
@ -732,8 +732,8 @@ func preflightInboundItem(msgID int64, seqNo int32, typeID uint32, content bool,
|
||||||
|
|
||||||
// prepareInboundRPCBatch performs the whole container's count/byte admission
|
// prepareInboundRPCBatch performs the whole container's count/byte admission
|
||||||
// before copying or scheduling any API RPC. Capacity exhaustion is converted
|
// before copying or scheduling any API RPC. Capacity exhaustion is converted
|
||||||
// into one consistent terminal FLOOD_WAIT result per uncached RPC; no business
|
// into one consistent local 500 WORKER_BUSY_TOO_LONG_RETRY result per uncached
|
||||||
// handler from the batch is allowed to start in that case.
|
// RPC; no business handler from the batch is allowed to start in that case.
|
||||||
func (s *Server) prepareInboundRPCBatch(ctx context.Context, c *Conn, plan *inboundPlan) error {
|
func (s *Server) prepareInboundRPCBatch(ctx context.Context, c *Conn, plan *inboundPlan) error {
|
||||||
if s.layerRPC != nil {
|
if s.layerRPC != nil {
|
||||||
return s.prepareInboundLayerRPCBatch(ctx, c, plan)
|
return s.prepareInboundLayerRPCBatch(ctx, c, plan)
|
||||||
|
|
@ -784,6 +784,13 @@ func (s *Server) prepareInboundRPCBatch(ctx context.Context, c *Conn, plan *inbo
|
||||||
s.rpcRewrap.commit(candidate)
|
s.rpcRewrap.commit(candidate)
|
||||||
item.kind = inboundItemReplayRPC
|
item.kind = inboundItemReplayRPC
|
||||||
item.payload = claim.encoded
|
item.payload = claim.encoded
|
||||||
|
case rpcResultAcquireAcknowledged:
|
||||||
|
// The client already ACKed the correlated rpc_result. Retire the
|
||||||
|
// stale rewrap candidate and keep this duplicate ACK-only; neither
|
||||||
|
// the old body nor the business handler may run again.
|
||||||
|
s.rpcRewrap.commit(candidate)
|
||||||
|
item.kind = inboundItemDuplicate
|
||||||
|
item.payload = nil
|
||||||
case rpcResultAcquirePending:
|
case rpcResultAcquirePending:
|
||||||
s.rpcRewrap.commit(candidate)
|
s.rpcRewrap.commit(candidate)
|
||||||
item.kind = inboundItemRewrappedRPC
|
item.kind = inboundItemRewrappedRPC
|
||||||
|
|
@ -847,6 +854,9 @@ func (s *Server) prepareInboundRPCBatch(ctx context.Context, c *Conn, plan *inbo
|
||||||
)
|
)
|
||||||
item.kind = inboundItemReplayRPC
|
item.kind = inboundItemReplayRPC
|
||||||
item.payload = claim.encoded
|
item.payload = claim.encoded
|
||||||
|
case rpcResultAcquireAcknowledged:
|
||||||
|
item.kind = inboundItemDuplicate
|
||||||
|
item.payload = nil
|
||||||
case rpcResultAcquirePending:
|
case rpcResultAcquirePending:
|
||||||
// A malformed/replayed container may repeat the same msg_id after this
|
// A malformed/replayed container may repeat the same msg_id after this
|
||||||
// very plan installed its owner. More generally, any request already in
|
// very plan installed its owner. More generally, any request already in
|
||||||
|
|
@ -1026,6 +1036,10 @@ func (s *Server) executeInboundPlan(ctx context.Context, cs *connState, c *Conn,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// The key cannot reconnect to ACK or replay any old answer. Release every
|
||||||
|
// logical outbox and receipt only after the terminal OK is physically on
|
||||||
|
// the wire; retaining them for the offline TTL would be pure leakage.
|
||||||
|
s.conns.ForgetLogicalSessionsForRawAuthKey(c.authKeyID)
|
||||||
c.beginTerminalShutdown()
|
c.beginTerminalShutdown()
|
||||||
c.closeTransport()
|
c.closeTransport()
|
||||||
return nil
|
return nil
|
||||||
|
|
@ -1037,10 +1051,7 @@ func (s *Server) executeInboundPlan(ctx context.Context, cs *connState, c *Conn,
|
||||||
if owner, _ := item.payload.(*rpcResultOwnerLease); owner != nil {
|
if owner, _ := item.payload.(*rpcResultOwnerLease); owner != nil {
|
||||||
owner.CompleteExecution(false)
|
owner.CompleteExecution(false)
|
||||||
}
|
}
|
||||||
if err := s.sendResult(ctx, c, item.msgID, &mt.RPCError{
|
if err := s.sendResult(ctx, c, item.msgID, rpcWorkerBusyError()); err != nil {
|
||||||
ErrorCode: 420,
|
|
||||||
ErrorMessage: "FLOOD_WAIT_1",
|
|
||||||
}); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
case inboundItemRPCAdmissionError:
|
case inboundItemRPCAdmissionError:
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,6 @@ package mtprotoedge
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
@ -93,22 +92,6 @@ type countingLayerRPCAdmission struct {
|
||||||
decodeCalls atomic.Int32
|
decodeCalls atomic.Int32
|
||||||
}
|
}
|
||||||
|
|
||||||
type failingReplayLayerRPC struct {
|
|
||||||
LayerRPCHandler
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *failingReplayLayerRPC) PrepareAdmittedReplay(
|
|
||||||
context.Context,
|
|
||||||
[8]byte,
|
|
||||||
int64,
|
|
||||||
int64,
|
|
||||||
uint64,
|
|
||||||
tlprofile.Admission,
|
|
||||||
) (func() error, error) {
|
|
||||||
return nil, h.err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *countingLayerRPCAdmission) AdmitLayer(profile tlprofile.Profile, b *bin.Buffer, limits tlprofile.Limits) (tlprofile.Admission, error) {
|
func (h *countingLayerRPCAdmission) AdmitLayer(profile tlprofile.Profile, b *bin.Buffer, limits tlprofile.Limits) (tlprofile.Admission, error) {
|
||||||
h.decodeCalls.Add(1)
|
h.decodeCalls.Add(1)
|
||||||
return h.LayerRPCHandler.AdmitLayer(profile, b, limits)
|
return h.LayerRPCHandler.AdmitLayer(profile, b, limits)
|
||||||
|
|
@ -509,7 +492,7 @@ func TestLayerRPCAdmissionCompletedReplayReleasesWholeProvisionalBatch(t *testin
|
||||||
if !claim.owner.CompleteExecution(true) {
|
if !claim.owner.CompleteExecution(true) {
|
||||||
t.Fatal("complete replay business outcome failed")
|
t.Fatal("complete replay business outcome failed")
|
||||||
}
|
}
|
||||||
s.rpcResults.Put(c.authKeyID, c.sessionID, 100, &encodedOutboundMessage{body: []byte{1, 2, 3, 4}})
|
storeLogicalRPCResultForTest(t, s, c, 100, &encodedOutboundMessage{body: []byte{1, 2, 3, 4}})
|
||||||
|
|
||||||
plan := &inboundPlan{items: []inboundItem{{kind: inboundItemRPC, msgID: 100, body: body}}}
|
plan := &inboundPlan{items: []inboundItem{{kind: inboundItemRPC, msgID: 100, body: body}}}
|
||||||
defer plan.close()
|
defer plan.close()
|
||||||
|
|
@ -530,54 +513,6 @@ func TestLayerRPCAdmissionCompletedReplayReleasesWholeProvisionalBatch(t *testin
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLayerRPCAdmissionReplayPreparationErrorIsNotSilentlyDelivered(t *testing.T) {
|
|
||||||
router := rpc.New(rpc.Config{DC: 2}, rpc.Deps{}, zaptest.NewLogger(t), clock.System)
|
|
||||||
prepareErr := errors.New("invalid replay wrapper metadata")
|
|
||||||
s := New(Options{DC: 2, LayerRPC: &failingReplayLayerRPC{
|
|
||||||
LayerRPCHandler: router,
|
|
||||||
err: prepareErr,
|
|
||||||
}})
|
|
||||||
c := &Conn{authKeyID: [8]byte{8, 9}, sessionID: 89, metrics: NopMetrics{}}
|
|
||||||
c.startInboundRPCScheduler(s.rpcScheduler, 1, 2, time.Second)
|
|
||||||
if err := c.FreezeLayerProfile(tlprofile.Profile225); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body := exactOutboundLayerRPCBody(t, tlprofile.Profile225, &tg.HelpGetConfigRequest{})
|
|
||||||
identityBuffer := &bin.Buffer{Buf: append([]byte(nil), body...)}
|
|
||||||
request, err := router.AdmitLayer(tlprofile.Profile225, identityBuffer, tlprofile.Limits{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
claim, err := s.rpcResults.AcquireIdentified(c.authKeyID, c.sessionID, 100, request.Prepared().Identity())
|
|
||||||
if err != nil || claim.owner == nil {
|
|
||||||
t.Fatalf("completed replay owner = %v, %v", claim.owner, err)
|
|
||||||
}
|
|
||||||
if !claim.owner.CompleteExecution(true) {
|
|
||||||
t.Fatal("complete replay business outcome failed")
|
|
||||||
}
|
|
||||||
s.rpcResults.Put(c.authKeyID, c.sessionID, 100, &encodedOutboundMessage{body: []byte{1, 2, 3, 4}})
|
|
||||||
|
|
||||||
plan := &inboundPlan{items: []inboundItem{{kind: inboundItemRPC, msgID: 100, body: body}}}
|
|
||||||
if err := s.prepareInboundLayerRPCBatch(context.Background(), c, plan); !errors.Is(err, prepareErr) {
|
|
||||||
plan.close()
|
|
||||||
t.Fatalf("replay preparation error = %v, want %v", err, prepareErr)
|
|
||||||
}
|
|
||||||
if plan.items[0].kind == inboundItemReplayRPC {
|
|
||||||
plan.close()
|
|
||||||
t.Fatal("invalid replay metadata was converted into a deliverable cached result")
|
|
||||||
}
|
|
||||||
plan.close()
|
|
||||||
if got := c.inflightRPCBytes.Load(); got != 0 || c.rpcReserved != 0 {
|
|
||||||
t.Fatalf("failed replay preparation leaked connection budget bytes:%d tasks:%d", got, c.rpcReserved)
|
|
||||||
}
|
|
||||||
s.rpcScheduler.budgetMu.Lock()
|
|
||||||
globalTasks, globalBytes := s.rpcScheduler.tasks, s.rpcScheduler.bytes
|
|
||||||
s.rpcScheduler.budgetMu.Unlock()
|
|
||||||
if globalTasks != 0 || globalBytes != 0 {
|
|
||||||
t.Fatalf("failed replay preparation leaked global budget %d/%d", globalTasks, globalBytes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLayerRPCAdmissionTransferredBatchClosesWithoutLeak(t *testing.T) {
|
func TestLayerRPCAdmissionTransferredBatchClosesWithoutLeak(t *testing.T) {
|
||||||
router := rpc.New(rpc.Config{DC: 2}, rpc.Deps{}, zaptest.NewLogger(t), clock.System)
|
router := rpc.New(rpc.Config{DC: 2}, rpc.Deps{}, zaptest.NewLogger(t), clock.System)
|
||||||
s := New(Options{DC: 2, LayerRPC: router})
|
s := New(Options{DC: 2, LayerRPC: router})
|
||||||
|
|
|
||||||
|
|
@ -138,6 +138,9 @@ func TestProjectionFailureCachesInternalWithoutRepeatingBusiness(t *testing.T) {
|
||||||
if rpcErr.ErrorCode != 500 || rpcErr.ErrorMessage != "INTERNAL" {
|
if rpcErr.ErrorCode != 500 || rpcErr.ErrorMessage != "INTERNAL" {
|
||||||
t.Fatalf("projection terminal = %+v", rpcErr)
|
t.Fatalf("projection terminal = %+v", rpcErr)
|
||||||
}
|
}
|
||||||
|
if _, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID); !ok {
|
||||||
|
t.Fatal("projection receipt disappeared before duplicate acquire")
|
||||||
|
}
|
||||||
// A same-msg replay is served from the completed exact identity; there is no
|
// A same-msg replay is served from the completed exact identity; there is no
|
||||||
// second DispatchAdmitted call even though projection failed after business
|
// second DispatchAdmitted call even though projection failed after business
|
||||||
// success.
|
// success.
|
||||||
|
|
@ -145,9 +148,14 @@ func TestProjectionFailureCachesInternalWithoutRepeatingBusiness(t *testing.T) {
|
||||||
c.authKeyID, c.sessionID, reqMsgID,
|
c.authKeyID, c.sessionID, reqMsgID,
|
||||||
tlprofile.Profile227, request.Prepared().Identity(),
|
tlprofile.Profile227, request.Prepared().Identity(),
|
||||||
)
|
)
|
||||||
if err != nil || replay.state != rpcResultAcquireCompleted || replay.encoded != completed.encoded {
|
if err != nil || replay.state != rpcResultAcquireCompleted {
|
||||||
t.Fatalf("projection replay = state:%d err:%v", replay.state, err)
|
t.Fatalf("projection replay = state:%d err:%v", replay.state, err)
|
||||||
}
|
}
|
||||||
|
if replay.encoded == nil || replay.encoded.replayMsgID != completed.encoded.replayMsgID ||
|
||||||
|
replay.encoded.replaySeqNo != completed.encoded.replaySeqNo ||
|
||||||
|
!sameBacking(replay.encoded.body, completed.encoded.body) {
|
||||||
|
t.Fatal("projection replay did not reference the original logical-outbox frame")
|
||||||
|
}
|
||||||
if got := handler.calls.Load(); got != 1 {
|
if got := handler.calls.Load(); got != 1 {
|
||||||
t.Fatalf("replay repeated business calls=%d", got)
|
t.Fatalf("replay repeated business calls=%d", got)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
186
internal/mtprotoedge/logical_session.go
Normal file
186
internal/mtprotoedge/logical_session.go
Normal file
|
|
@ -0,0 +1,186 @@
|
||||||
|
package mtprotoedge
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// logicalSession is the process-local MTProto session that survives physical
|
||||||
|
// transport replacement. The outbound state is the sole owner of every
|
||||||
|
// content-related server message until msgs_ack, explicit session destruction,
|
||||||
|
// or the bounded offline-retention window expires.
|
||||||
|
type logicalSession struct {
|
||||||
|
key sessionKey
|
||||||
|
outbound *outboundState
|
||||||
|
offlineAt time.Time
|
||||||
|
businessAuthKeyID [8]byte
|
||||||
|
businessAuthResolved bool
|
||||||
|
}
|
||||||
|
|
||||||
|
const logicalSessionOfflineTTL = 6 * time.Minute
|
||||||
|
|
||||||
|
func (m *SessionManager) attachLogicalSession(c *Conn, budget *outboundTrackedBudget) {
|
||||||
|
if m == nil || c == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
key := connSessionKey(c)
|
||||||
|
m.mu.Lock()
|
||||||
|
logical := m.logicalSessions[key]
|
||||||
|
if logical == nil {
|
||||||
|
logical = &logicalSession{
|
||||||
|
key: key,
|
||||||
|
outbound: newOutboundState(budget),
|
||||||
|
}
|
||||||
|
m.logicalSessions[key] = logical
|
||||||
|
}
|
||||||
|
logical.outbound.persistent.Store(true)
|
||||||
|
if businessAuthKeyID, resolved := c.BusinessAuthKeyID(); resolved {
|
||||||
|
logical.businessAuthKeyID = businessAuthKeyID
|
||||||
|
logical.businessAuthResolved = true
|
||||||
|
}
|
||||||
|
logical.offlineAt = time.Time{}
|
||||||
|
c.outboundState = logical.outbound
|
||||||
|
m.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// adoptLogicalSession is a construction-test/embedded-Conn bridge. Production
|
||||||
|
// Conns are attached before their actor starts; direct Conn builders may already
|
||||||
|
// own an actor-local state when a Server terminal callback publishes the receipt.
|
||||||
|
func (m *SessionManager) adoptLogicalSession(c *Conn) {
|
||||||
|
if m == nil || c == nil || c.outboundState == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
key := connSessionKey(c)
|
||||||
|
m.mu.Lock()
|
||||||
|
logical := m.logicalSessions[key]
|
||||||
|
if logical == nil {
|
||||||
|
logical = &logicalSession{key: key, outbound: c.outboundState}
|
||||||
|
m.logicalSessions[key] = logical
|
||||||
|
}
|
||||||
|
logical.outbound.persistent.Store(true)
|
||||||
|
if businessAuthKeyID, resolved := c.BusinessAuthKeyID(); resolved {
|
||||||
|
logical.businessAuthKeyID = businessAuthKeyID
|
||||||
|
logical.businessAuthResolved = true
|
||||||
|
}
|
||||||
|
logical.offlineAt = time.Time{}
|
||||||
|
// The actor's state pointer is immutable after startOutbound. Production
|
||||||
|
// attaches before start; this adoption bridge only marks that already-owned
|
||||||
|
// state persistent and must never write the Conn field concurrently.
|
||||||
|
m.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *SessionManager) markLogicalSessionOfflineLocked(key sessionKey, now time.Time) {
|
||||||
|
logical := m.logicalSessions[key]
|
||||||
|
if logical == nil || m.bySession[key] != nil || m.claims[key] != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if logical.offlineAt.IsZero() {
|
||||||
|
logical.offlineAt = now
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *SessionManager) destroyLogicalSessionLocked(key sessionKey) *outboundState {
|
||||||
|
logical := m.logicalSessions[key]
|
||||||
|
if logical == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
delete(m.logicalSessions, key)
|
||||||
|
return logical.outbound
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *SessionManager) bindLogicalSessionAuthKeyLocked(key sessionKey, businessAuthKeyID [8]byte) {
|
||||||
|
if logical := m.logicalSessions[key]; logical != nil {
|
||||||
|
logical.businessAuthKeyID = businessAuthKeyID
|
||||||
|
logical.businessAuthResolved = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *SessionManager) releaseLogicalSession(key sessionKey, state *outboundState) {
|
||||||
|
if state != nil {
|
||||||
|
state.releaseAll()
|
||||||
|
}
|
||||||
|
m.mu.RLock()
|
||||||
|
hook := m.logicalSessionReleased
|
||||||
|
m.mu.RUnlock()
|
||||||
|
if hook != nil {
|
||||||
|
hook(key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ForgetLogicalSessionsForRawAuthKey is the terminal auth-key destruction path.
|
||||||
|
// Once destroy_auth_key_ok is on the wire the key can never reconnect to ACK or
|
||||||
|
// replay an old answer, so retaining any session payload or receipt is useless.
|
||||||
|
func (m *SessionManager) ForgetLogicalSessionsForRawAuthKey(authKeyID [8]byte) {
|
||||||
|
if m == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var release []*logicalSession
|
||||||
|
m.mu.Lock()
|
||||||
|
for key, logical := range m.logicalSessions {
|
||||||
|
if key.authKeyID != authKeyID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
delete(m.logicalSessions, key)
|
||||||
|
if logical != nil {
|
||||||
|
release = append(release, logical)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
for _, logical := range release {
|
||||||
|
m.releaseLogicalSession(logical.key, logical.outbound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *SessionManager) sweepLogicalSessions(now time.Time) {
|
||||||
|
if m == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var release []*logicalSession
|
||||||
|
m.mu.Lock()
|
||||||
|
for key, logical := range m.logicalSessions {
|
||||||
|
if logical == nil || logical.offlineAt.IsZero() || now.Sub(logical.offlineAt) < logicalSessionOfflineTTL {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m.bySession[key] != nil || m.claims[key] != nil {
|
||||||
|
logical.offlineAt = time.Time{}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
delete(m.logicalSessions, key)
|
||||||
|
release = append(release, logical)
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
for _, logical := range release {
|
||||||
|
m.releaseLogicalSession(logical.key, logical.outbound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *SessionManager) releaseAllLogicalSessions() {
|
||||||
|
if m == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var release []*logicalSession
|
||||||
|
m.mu.Lock()
|
||||||
|
for key, logical := range m.logicalSessions {
|
||||||
|
delete(m.logicalSessions, key)
|
||||||
|
if logical != nil {
|
||||||
|
release = append(release, logical)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m.mu.Unlock()
|
||||||
|
for _, logical := range release {
|
||||||
|
m.releaseLogicalSession(logical.key, logical.outbound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rpcResult returns the exact unacknowledged wire result owned by the logical
|
||||||
|
// session. The receipt ledger calls this only after validating request identity.
|
||||||
|
func (m *SessionManager) rpcResult(authKeyID [8]byte, sessionID, reqMsgID int64) (*encodedOutboundMessage, bool) {
|
||||||
|
if m == nil || reqMsgID == 0 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
key := sessionKey{authKeyID: authKeyID, sessionID: sessionID}
|
||||||
|
m.mu.RLock()
|
||||||
|
logical := m.logicalSessions[key]
|
||||||
|
m.mu.RUnlock()
|
||||||
|
if logical == nil || logical.outbound == nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return logical.outbound.rpcResult(reqMsgID)
|
||||||
|
}
|
||||||
281
internal/mtprotoedge/logical_session_test.go
Normal file
281
internal/mtprotoedge/logical_session_test.go
Normal file
|
|
@ -0,0 +1,281 @@
|
||||||
|
package mtprotoedge
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/binary"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/iamxvbaba/td/proto"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
// storeLogicalRPCResultForTest publishes the same production invariant as the
|
||||||
|
// outbound actor: the exact rpc_result frame enters the logical-session outbox
|
||||||
|
// before its metadata receipt becomes visible. Tests that call the receipt
|
||||||
|
// ledger directly must not invent the old impossible state where a replay row
|
||||||
|
// exists without an unacknowledged server frame.
|
||||||
|
func storeLogicalRPCResultForTest(
|
||||||
|
t *testing.T,
|
||||||
|
s *Server,
|
||||||
|
c *Conn,
|
||||||
|
reqMsgID int64,
|
||||||
|
encoded *encodedOutboundMessage,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
if s == nil || c == nil || encoded == nil || len(encoded.body) == 0 {
|
||||||
|
t.Fatal("invalid logical rpc_result fixture")
|
||||||
|
}
|
||||||
|
if c.outboundState == nil {
|
||||||
|
s.conns.attachLogicalSession(c, s.outboundTrackedBudget)
|
||||||
|
} else {
|
||||||
|
s.conns.adoptLogicalSession(c)
|
||||||
|
}
|
||||||
|
physicalReqMsgID := encoded.writtenRequestID()
|
||||||
|
if physicalReqMsgID == 0 {
|
||||||
|
physicalReqMsgID = reqMsgID
|
||||||
|
}
|
||||||
|
frameBody := encoded.body
|
||||||
|
if physicalReqMsgID != reqMsgID && encoded.typeID == proto.ResultTypeID {
|
||||||
|
physical, err := cloneRPCResultForRequest(encoded, physicalReqMsgID, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("retarget logical rpc_result fixture: %v", err)
|
||||||
|
}
|
||||||
|
frameBody = physical.body
|
||||||
|
}
|
||||||
|
state := c.outboundState
|
||||||
|
state.mu.Lock()
|
||||||
|
if state.budget == nil || !state.budget.reserve(len(frameBody)) {
|
||||||
|
state.mu.Unlock()
|
||||||
|
t.Fatal("reserve logical rpc_result fixture")
|
||||||
|
}
|
||||||
|
candidate := reqMsgID*4 + 1
|
||||||
|
if candidate <= 0 {
|
||||||
|
candidate = 1
|
||||||
|
}
|
||||||
|
frame := &outboundFrame{
|
||||||
|
msgID: state.reserveMsgID(candidate), seqNo: state.peekSeqNo(true),
|
||||||
|
typeID: proto.ResultTypeID, body: frameBody, reqMsgID: physicalReqMsgID,
|
||||||
|
priority: encoded.priority, delivery: encoded.delivery,
|
||||||
|
compressed: encoded.compressed, uncompressedBytes: encoded.uncompressedBytes,
|
||||||
|
layer: encoded.layer, layerInvariant: encoded.layerInvariant,
|
||||||
|
reservedBytes: len(frameBody), reservationBudget: state.budget,
|
||||||
|
}
|
||||||
|
if err := state.admitReserved(frame); err != nil {
|
||||||
|
frame.releaseReservation(state.budget)
|
||||||
|
state.mu.Unlock()
|
||||||
|
t.Fatalf("admit logical rpc_result fixture: %v", err)
|
||||||
|
}
|
||||||
|
encoded.replayMsgID = frame.msgID
|
||||||
|
encoded.replaySeqNo = frame.seqNo
|
||||||
|
state.mu.Unlock()
|
||||||
|
s.rpcResults.Put(c.authKeyID, c.sessionID, reqMsgID, encoded)
|
||||||
|
}
|
||||||
|
|
||||||
|
func acknowledgeLogicalRPCResultForTest(t *testing.T, s *Server, c *Conn, reqMsgID int64) {
|
||||||
|
t.Helper()
|
||||||
|
if s == nil || c == nil || c.outboundState == nil {
|
||||||
|
t.Fatal("missing logical rpc_result fixture")
|
||||||
|
}
|
||||||
|
state := c.outboundState
|
||||||
|
state.mu.Lock()
|
||||||
|
msgID := state.byRequest[reqMsgID]
|
||||||
|
requestIDs := state.ack([]int64{msgID})
|
||||||
|
state.mu.Unlock()
|
||||||
|
if len(requestIDs) != 1 || requestIDs[0] != reqMsgID {
|
||||||
|
t.Fatalf("logical ACK resolved request ids %v", requestIDs)
|
||||||
|
}
|
||||||
|
if !s.rpcResults.Acknowledge(c.authKeyID, c.sessionID, reqMsgID) {
|
||||||
|
t.Fatal("logical ACK did not release result receipt")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogicalSessionOwnsExactResultAcrossPhysicalReconnect(t *testing.T) {
|
||||||
|
manager := NewSessionManager(zap.NewNop())
|
||||||
|
budget := newOutboundTrackedBudget(1024)
|
||||||
|
authKeyID := [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
|
||||||
|
first := &Conn{authKeyID: authKeyID, sessionID: 42}
|
||||||
|
manager.attachLogicalSession(first, budget)
|
||||||
|
|
||||||
|
body := append([]byte{0xf3, 0x5c, 0x6d, 0xf3}, make([]byte, 28)...)
|
||||||
|
binary.LittleEndian.PutUint64(body[4:12], uint64(77))
|
||||||
|
if !budget.reserve(len(body)) {
|
||||||
|
t.Fatal("reserve result body")
|
||||||
|
}
|
||||||
|
frame := &outboundFrame{
|
||||||
|
msgID: 101, seqNo: 1, typeID: proto.ResultTypeID, body: body,
|
||||||
|
reqMsgID: 77, reservedBytes: len(body), reservationBudget: budget,
|
||||||
|
}
|
||||||
|
first.outboundState.mu.Lock()
|
||||||
|
if err := first.outboundState.admitReserved(frame); err != nil {
|
||||||
|
first.outboundState.mu.Unlock()
|
||||||
|
t.Fatalf("admit result: %v", err)
|
||||||
|
}
|
||||||
|
first.outboundState.mu.Unlock()
|
||||||
|
|
||||||
|
second := &Conn{authKeyID: authKeyID, sessionID: 42}
|
||||||
|
manager.attachLogicalSession(second, budget)
|
||||||
|
if first.outboundState != second.outboundState {
|
||||||
|
t.Fatal("physical reconnect did not reuse logical outbound state")
|
||||||
|
}
|
||||||
|
replay, ok := manager.rpcResult(authKeyID, 42, 77)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("logical result not found after reconnect")
|
||||||
|
}
|
||||||
|
if replay.replayMsgID != frame.msgID || replay.replaySeqNo != frame.seqNo || !bytes.Equal(replay.body, body) {
|
||||||
|
t.Fatalf("replay identity/body = msg:%d seq:%d bytes:%d", replay.replayMsgID, replay.replaySeqNo, len(replay.body))
|
||||||
|
}
|
||||||
|
if &replay.body[0] != &frame.body[0] {
|
||||||
|
t.Fatal("replay created a second payload owner")
|
||||||
|
}
|
||||||
|
attempt, err := cloneRPCResultForRequest(replay, replay.reqMsgID, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("clone same-request replay descriptor: %v", err)
|
||||||
|
}
|
||||||
|
if attempt.replayMsgID != frame.msgID || attempt.replaySeqNo != frame.seqNo ||
|
||||||
|
!sameBacking(attempt.body, frame.body) {
|
||||||
|
t.Fatal("queued duplicate lost stable logical frame identity")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogicalSessionACKReleasesPayloadAndReceipt(t *testing.T) {
|
||||||
|
manager := NewSessionManager(zap.NewNop())
|
||||||
|
budget := newOutboundTrackedBudget(1024)
|
||||||
|
authKeyID := [8]byte{8, 7, 6, 5, 4, 3, 2, 1}
|
||||||
|
c := &Conn{authKeyID: authKeyID, sessionID: 43}
|
||||||
|
manager.attachLogicalSession(c, budget)
|
||||||
|
|
||||||
|
body := make([]byte, 64)
|
||||||
|
if !budget.reserve(len(body)) {
|
||||||
|
t.Fatal("reserve result body")
|
||||||
|
}
|
||||||
|
frame := &outboundFrame{
|
||||||
|
msgID: 201, seqNo: 1, typeID: proto.ResultTypeID, body: body,
|
||||||
|
reqMsgID: 88, reservedBytes: len(body), reservationBudget: budget,
|
||||||
|
}
|
||||||
|
c.outboundState.mu.Lock()
|
||||||
|
if err := c.outboundState.admitReserved(frame); err != nil {
|
||||||
|
c.outboundState.mu.Unlock()
|
||||||
|
t.Fatalf("admit result: %v", err)
|
||||||
|
}
|
||||||
|
c.outboundState.mu.Unlock()
|
||||||
|
|
||||||
|
ledger := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
|
||||||
|
maxPending: 16, maxPendingPerAuth: 16,
|
||||||
|
globalMaxEntries: 16, globalMaxBytes: 16,
|
||||||
|
authMaxEntries: 16, authMaxBytes: 16,
|
||||||
|
sessionMaxEntries: 16, sessionMaxBytes: 16,
|
||||||
|
sessions: manager,
|
||||||
|
})
|
||||||
|
claim, err := ledger.Acquire(authKeyID, 43, 88)
|
||||||
|
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||||
|
t.Fatalf("acquire owner = %#v, %v", claim, err)
|
||||||
|
}
|
||||||
|
claim.owner.CompleteExecution(true)
|
||||||
|
claim.owner.HandOff()
|
||||||
|
ledger.Put(authKeyID, 43, 88, &encodedOutboundMessage{body: body, typeID: proto.ResultTypeID, reqMsgID: 88})
|
||||||
|
|
||||||
|
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: 43, reqMsgID: 88}
|
||||||
|
shard := ledger.shard(key)
|
||||||
|
shard.mu.Lock()
|
||||||
|
entry := shard.byKey[key].Value.(*rpcResultCacheEntry)
|
||||||
|
if entry.encoded != nil || entry.size != 1 {
|
||||||
|
shard.mu.Unlock()
|
||||||
|
t.Fatalf("completed ledger retained payload: encoded=%p size=%d", entry.encoded, entry.size)
|
||||||
|
}
|
||||||
|
shard.mu.Unlock()
|
||||||
|
|
||||||
|
c.outboundState.mu.Lock()
|
||||||
|
acked := c.outboundState.ack([]int64{201})
|
||||||
|
c.outboundState.mu.Unlock()
|
||||||
|
if len(acked) != 1 || acked[0] != 88 {
|
||||||
|
t.Fatalf("acked request ids = %v", acked)
|
||||||
|
}
|
||||||
|
if !ledger.Acknowledge(authKeyID, 43, 88) {
|
||||||
|
t.Fatal("ledger did not observe ACK")
|
||||||
|
}
|
||||||
|
if got := budget.snapshot(); got != 0 {
|
||||||
|
t.Fatalf("tracked payload bytes after ACK = %d", got)
|
||||||
|
}
|
||||||
|
shard.mu.Lock()
|
||||||
|
_, retained := shard.byKey[key]
|
||||||
|
shard.mu.Unlock()
|
||||||
|
if retained {
|
||||||
|
t.Fatal("ACK retained a completed receipt")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogicalSessionCapacityNeverEvictsUnackedFrame(t *testing.T) {
|
||||||
|
budget := newOutboundTrackedBudget(1024)
|
||||||
|
state := newOutboundStateWithLimits(budget, 1, 16)
|
||||||
|
firstBody := make([]byte, 8)
|
||||||
|
if !budget.reserve(len(firstBody)) {
|
||||||
|
t.Fatal("reserve first body")
|
||||||
|
}
|
||||||
|
first := &outboundFrame{
|
||||||
|
msgID: 1, seqNo: 1, typeID: proto.ResultTypeID, body: firstBody,
|
||||||
|
reqMsgID: 11, reservedBytes: len(firstBody), reservationBudget: budget,
|
||||||
|
}
|
||||||
|
if err := state.admitReserved(first); err != nil {
|
||||||
|
t.Fatalf("admit first: %v", err)
|
||||||
|
}
|
||||||
|
second := &outboundFrame{msgID: 2, seqNo: 3, typeID: proto.ResultTypeID, body: make([]byte, 8), reqMsgID: 12}
|
||||||
|
if err := state.admitReserved(second); err == nil {
|
||||||
|
t.Fatal("capacity admitted a second unacknowledged frame")
|
||||||
|
}
|
||||||
|
if state.pending[first.msgID] != first || state.byRequest[first.reqMsgID] != first.msgID {
|
||||||
|
t.Fatal("capacity failure evicted or rewired the existing frame")
|
||||||
|
}
|
||||||
|
state.releaseAll()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogicalSessionDestroyReleasesPayloadAndCompletedReceipt(t *testing.T) {
|
||||||
|
s := New(Options{})
|
||||||
|
authKeyID := [8]byte{9, 1}
|
||||||
|
c := &Conn{authKeyID: authKeyID, sessionID: 91}
|
||||||
|
claim, err := s.rpcResults.Acquire(authKeyID, 91, 901)
|
||||||
|
if err != nil || claim.owner == nil {
|
||||||
|
t.Fatalf("acquire owner: %#v err=%v", claim, err)
|
||||||
|
}
|
||||||
|
claim.owner.CompleteExecution(true)
|
||||||
|
storeLogicalRPCResultForTest(t, s, c, 901, &encodedOutboundMessage{
|
||||||
|
body: make([]byte, 32), typeID: proto.ResultTypeID, reqMsgID: 901,
|
||||||
|
})
|
||||||
|
if _, ok := s.rpcResults.Get(authKeyID, 91, 901); !ok {
|
||||||
|
t.Fatal("logical result fixture was not published")
|
||||||
|
}
|
||||||
|
if removed := s.conns.DestroySessionForAuthKey(authKeyID, 91); removed {
|
||||||
|
t.Fatal("construction-only logical session unexpectedly reported active")
|
||||||
|
}
|
||||||
|
if _, ok := s.rpcResults.Get(authKeyID, 91, 901); ok {
|
||||||
|
t.Fatal("destroy retained completed receipt")
|
||||||
|
}
|
||||||
|
if got := s.outboundTrackedBudget.snapshot(); got != 0 {
|
||||||
|
t.Fatalf("destroy retained %d payload bytes", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBusinessAuthRevocationReleasesOfflineTempLogicalSession(t *testing.T) {
|
||||||
|
s := New(Options{})
|
||||||
|
rawAuthKeyID := [8]byte{9, 2}
|
||||||
|
businessAuthKeyID := [8]byte{9, 3}
|
||||||
|
c := &Conn{authKeyID: rawAuthKeyID, sessionID: 92}
|
||||||
|
c.SetBusinessAuthKeyID(businessAuthKeyID)
|
||||||
|
claim, err := s.rpcResults.Acquire(rawAuthKeyID, 92, 902)
|
||||||
|
if err != nil || claim.owner == nil {
|
||||||
|
t.Fatalf("acquire owner: %#v err=%v", claim, err)
|
||||||
|
}
|
||||||
|
claim.owner.CompleteExecution(true)
|
||||||
|
storeLogicalRPCResultForTest(t, s, c, 902, &encodedOutboundMessage{
|
||||||
|
body: make([]byte, 48), typeID: proto.ResultTypeID, reqMsgID: 902,
|
||||||
|
})
|
||||||
|
if closed := s.conns.CloseSessionsForBusinessAuthKey(businessAuthKeyID); closed != 0 {
|
||||||
|
t.Fatalf("offline logical revocation closed %d physical conns", closed)
|
||||||
|
}
|
||||||
|
if _, ok := s.rpcResults.Get(rawAuthKeyID, 92, 902); ok {
|
||||||
|
t.Fatal("business auth revocation retained temp-key receipt")
|
||||||
|
}
|
||||||
|
if got := s.outboundTrackedBudget.snapshot(); got != 0 {
|
||||||
|
t.Fatalf("business auth revocation retained %d payload bytes", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -50,8 +50,8 @@ const (
|
||||||
maxTrackedServerMsgIDs = 4096
|
maxTrackedServerMsgIDs = 4096
|
||||||
maxTrackedAckedMsgIDs = 1024
|
maxTrackedAckedMsgIDs = 1024
|
||||||
// maxTrackedServerBytes 是 pending(已发送待 ack、用于 resend)总 body 字节上限。
|
// maxTrackedServerBytes 是 pending(已发送待 ack、用于 resend)总 body 字节上限。
|
||||||
// 与 maxTrackedServerMsgIDs 并列:客户端从不 ack 时,大响应体按字节滚动丢弃,
|
// 与 maxTrackedServerMsgIDs 并列;到达任一上限后拒绝新可靠 frame,绝不滚动
|
||||||
// 防 pending 被「4096 条 × 大 body」撑爆。
|
// 丢弃尚未 ACK 的旧 frame。
|
||||||
maxTrackedServerBytes = 64 << 20 // 64 MiB
|
maxTrackedServerBytes = 64 << 20 // 64 MiB
|
||||||
// Encrypted transport adds auth-key/msg-key, plaintext headers, randomized
|
// Encrypted transport adds auth-key/msg-key, plaintext headers, randomized
|
||||||
// padding and codec framing. Reject before creating the two encryption buffers.
|
// padding and codec framing. Reject before creating the two encryption buffers.
|
||||||
|
|
@ -214,6 +214,12 @@ type encodedOutboundMessage struct {
|
||||||
layer *outboundLayerBinding
|
layer *outboundLayerBinding
|
||||||
compressed bool
|
compressed bool
|
||||||
uncompressedBytes int
|
uncompressedBytes int
|
||||||
|
// replayMsgID/replaySeqNo identify an existing logical-session frame. They
|
||||||
|
// are populated only by the receipt ledger's outbox lookup, never by a newly
|
||||||
|
// encoded result. A replay writes that exact frame instead of allocating a
|
||||||
|
// second payload owner or a new MTProto message identity.
|
||||||
|
replayMsgID int64
|
||||||
|
replaySeqNo int32
|
||||||
}
|
}
|
||||||
|
|
||||||
type rpcResultDeliveryState uint32
|
type rpcResultDeliveryState uint32
|
||||||
|
|
@ -263,7 +269,7 @@ type rpcResultDeliveryCoordinator struct {
|
||||||
// deferredToReplay is sticky while a successful initConnection retarget owns
|
// deferredToReplay is sticky while a successful initConnection retarget owns
|
||||||
// the logical hook. The ordinary source terminal may mark physical delivery,
|
// the logical hook. The ordinary source terminal may mark physical delivery,
|
||||||
// but only the alias restore barrier may claim the hook. On terminal alias
|
// but only the alias restore barrier may claim the hook. On terminal alias
|
||||||
// failure the flag is released so a later completed-cache replay can retry.
|
// failure the flag is released so a later logical-outbox replay can retry.
|
||||||
deferredToReplay bool
|
deferredToReplay bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -722,18 +728,25 @@ func cloneRPCResultForRequest(encoded *encodedOutboundMessage, reqMsgID int64, s
|
||||||
if shareDelivery {
|
if shareDelivery {
|
||||||
delivery = encoded.delivery
|
delivery = encoded.delivery
|
||||||
}
|
}
|
||||||
|
var replayMsgID int64
|
||||||
|
var replaySeqNo int32
|
||||||
|
if reqMsgID == encoded.reqMsgID {
|
||||||
|
replayMsgID = encoded.replayMsgID
|
||||||
|
replaySeqNo = encoded.replaySeqNo
|
||||||
|
}
|
||||||
return &encodedOutboundMessage{
|
return &encodedOutboundMessage{
|
||||||
body: body, typeID: encoded.typeID, reqMsgID: reqMsgID,
|
body: body, typeID: encoded.typeID, reqMsgID: reqMsgID,
|
||||||
priority: encoded.priority, delivery: delivery, compressed: encoded.compressed,
|
priority: encoded.priority, delivery: delivery, compressed: encoded.compressed,
|
||||||
layer: encoded.layer, layerInvariant: encoded.layerInvariant,
|
layer: encoded.layer, layerInvariant: encoded.layerInvariant,
|
||||||
uncompressedBytes: encoded.uncompressedBytes,
|
uncompressedBytes: encoded.uncompressedBytes,
|
||||||
|
replayMsgID: replayMsgID, replaySeqNo: replaySeqNo,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// cloneRPCResultForRequestReserved charges the target connection's retained-body
|
// cloneRPCResultForRequestReserved charges the target connection's retained-body
|
||||||
// budget before a retarget copy can exist. Even the same-req_id zero-copy case
|
// budget before a retarget copy can exist. Even the same-req_id zero-copy case
|
||||||
// needs a reservation: the replay/rewrap owner may outlive cache eviction while
|
// needs a reservation: the replay/rewrap attempt may outlive ACK/removal of its
|
||||||
// it is queued, so its immutable body must remain independently pinned.
|
// source frame while queued, so its immutable body must remain independently pinned.
|
||||||
func (c *Conn) cloneRPCResultForRequestReserved(
|
func (c *Conn) cloneRPCResultForRequestReserved(
|
||||||
encoded *encodedOutboundMessage,
|
encoded *encodedOutboundMessage,
|
||||||
reqMsgID int64,
|
reqMsgID int64,
|
||||||
|
|
@ -776,14 +789,20 @@ type outboundFrame struct {
|
||||||
// but their bytes must remain on the independent control budget for the full lifetime.
|
// but their bytes must remain on the independent control budget for the full lifetime.
|
||||||
reservationBudget *outboundTrackedBudget
|
reservationBudget *outboundTrackedBudget
|
||||||
reqMsgID int64
|
reqMsgID int64
|
||||||
|
priority outboundPriority
|
||||||
|
delivery *rpcResultDelivery
|
||||||
|
compressed bool
|
||||||
|
uncompressedBytes int
|
||||||
// layer is retained only for proactive session-bound frames so a later
|
// layer is retained only for proactive session-bound frames so a later
|
||||||
// msg_resend_req cannot replay bytes from an obsolete profile epoch.
|
// msg_resend_req cannot replay bytes from an obsolete profile epoch.
|
||||||
layer *outboundLayerBinding
|
layer *outboundLayerBinding
|
||||||
sentAt time.Time
|
layerInvariant bool
|
||||||
sends int
|
sentAt time.Time
|
||||||
|
sends int
|
||||||
}
|
}
|
||||||
|
|
||||||
type outboundState struct {
|
type outboundState struct {
|
||||||
|
mu sync.Mutex
|
||||||
pending map[int64]*outboundFrame
|
pending map[int64]*outboundFrame
|
||||||
order []int64
|
order []int64
|
||||||
byRequest map[int64]int64
|
byRequest map[int64]int64
|
||||||
|
|
@ -793,6 +812,16 @@ type outboundState struct {
|
||||||
maxMessages int
|
maxMessages int
|
||||||
maxBytes int
|
maxBytes int
|
||||||
budget *outboundTrackedBudget
|
budget *outboundTrackedBudget
|
||||||
|
// sentContentMessages is logical-session state, not physical-connection
|
||||||
|
// state. Reserving a new content frame advances it before the first write so
|
||||||
|
// a failed write can be replayed with the same seq_no after reconnect.
|
||||||
|
sentContentMessages int32
|
||||||
|
lastMsgID int64
|
||||||
|
// persistent becomes true once this state is owned by a logical session.
|
||||||
|
// Directly constructed Conns may start with actor-local ownership and be
|
||||||
|
// adopted only from the terminal callback after a failed first write; the
|
||||||
|
// actor must not release that outbox while the logical session retains it.
|
||||||
|
persistent atomic.Bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// outboundTrackedBudget 是 body/control/write 三类预算共用的原子 byte-budget primitive。
|
// outboundTrackedBudget 是 body/control/write 三类预算共用的原子 byte-budget primitive。
|
||||||
|
|
@ -1001,6 +1030,12 @@ func (c *Conn) startOutbound() {
|
||||||
c.outboundBulk = make(chan outboundOp, bulkSize)
|
c.outboundBulk = make(chan outboundOp, bulkSize)
|
||||||
c.outboundStop = make(chan struct{})
|
c.outboundStop = make(chan struct{})
|
||||||
c.outboundDone = make(chan struct{})
|
c.outboundDone = make(chan struct{})
|
||||||
|
// Publish the actor state before starting its goroutine. The pointer remains
|
||||||
|
// immutable for the physical Conn lifetime; logical-session adoption may only
|
||||||
|
// mark the existing state persistent.
|
||||||
|
if c.outboundState == nil {
|
||||||
|
c.outboundState = newOutboundState(c.outboundTrackedBudget)
|
||||||
|
}
|
||||||
go c.outboundLoop()
|
go c.outboundLoop()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1669,12 +1704,14 @@ func (c *Conn) endOutboundEnqueue() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Conn) outboundLoop() {
|
func (c *Conn) outboundLoop() {
|
||||||
state := newOutboundState(c.outboundTrackedBudget)
|
state := c.outboundState
|
||||||
ordinarySinceBulk := 0
|
ordinarySinceBulk := 0
|
||||||
defer func() {
|
defer func() {
|
||||||
// pending frames belong exclusively to this actor. Releasing after drain ensures no
|
// A Server Conn leaves pending frames with the logical session. Standalone
|
||||||
// resend path can race the final budget return and no Conn body survives actor exit.
|
// construction/tests retain the old actor-local ownership boundary.
|
||||||
state.releaseAll()
|
if !state.persistent.Load() {
|
||||||
|
state.releaseAll()
|
||||||
|
}
|
||||||
close(c.outboundDone)
|
close(c.outboundDone)
|
||||||
}()
|
}()
|
||||||
for {
|
for {
|
||||||
|
|
@ -1790,29 +1827,39 @@ func (c *Conn) handleOutboundOp(state *outboundState, op outboundOp) {
|
||||||
if op.kind != outboundSend {
|
if op.kind != outboundSend {
|
||||||
defer op.releaseReservation(state.budget)
|
defer op.releaseReservation(state.budget)
|
||||||
}
|
}
|
||||||
|
// Physical generations may overlap briefly during activation fencing. The
|
||||||
|
// logical-session mutex preserves one writer/seq/outbox state machine across
|
||||||
|
// both actors without transferring payload ownership. Terminal callbacks run
|
||||||
|
// only after unlock: publication may look the frame up in this same outbox.
|
||||||
|
state.mu.Lock()
|
||||||
|
var (
|
||||||
|
result outboundResult
|
||||||
|
acked []int64
|
||||||
|
)
|
||||||
switch op.kind {
|
switch op.kind {
|
||||||
case outboundSend:
|
case outboundSend:
|
||||||
c.handleOutboundSend(state, op)
|
result.err = c.handleOutboundSend(state, op)
|
||||||
case outboundAck:
|
case outboundAck:
|
||||||
for _, reqMsgID := range state.ack(op.ids) {
|
acked = state.ack(op.ids)
|
||||||
if c.rpcResultAcked != nil {
|
|
||||||
c.rpcResultAcked(c, reqMsgID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case outboundQueryState:
|
case outboundQueryState:
|
||||||
op.finish(outboundResult{info: state.stateInfo(op.ids)})
|
result.info = state.stateInfo(op.ids)
|
||||||
case outboundResend:
|
case outboundResend:
|
||||||
info, err := c.handleOutboundResend(state, op.ctx, op.ids)
|
result.info, result.err = c.handleOutboundResend(state, op.ctx, op.ids)
|
||||||
op.finish(outboundResult{info: info, err: err})
|
|
||||||
case outboundResendByRequest:
|
case outboundResendByRequest:
|
||||||
resent, err := c.handleOutboundResendByRequest(state, op.ctx, op.reqMsgID)
|
result.resent, result.err = c.handleOutboundResendByRequest(state, op.ctx, op.reqMsgID)
|
||||||
op.finish(outboundResult{resent: resent, err: err})
|
|
||||||
default:
|
default:
|
||||||
op.finish(outboundResult{err: fmt.Errorf("unknown outbound op %d", op.kind)})
|
result.err = fmt.Errorf("unknown outbound op %d", op.kind)
|
||||||
}
|
}
|
||||||
|
state.mu.Unlock()
|
||||||
|
for _, reqMsgID := range acked {
|
||||||
|
if c.rpcResultAcked != nil {
|
||||||
|
c.rpcResultAcked(c, reqMsgID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
op.finish(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Conn) handleOutboundSend(state *outboundState, op outboundOp) {
|
func (c *Conn) handleOutboundSend(state *outboundState, op outboundOp) error {
|
||||||
var binding *outboundLayerBinding
|
var binding *outboundLayerBinding
|
||||||
if op.encoded != nil {
|
if op.encoded != nil {
|
||||||
binding = op.encoded.layer
|
binding = op.encoded.layer
|
||||||
|
|
@ -1861,7 +1908,7 @@ func (c *Conn) handleOutboundSend(state *outboundState, op outboundOp) {
|
||||||
}
|
}
|
||||||
var frame *outboundFrame
|
var frame *outboundFrame
|
||||||
if err == nil {
|
if err == nil {
|
||||||
frame, err = c.buildFrame(op.ctx, op.msgType, op.msg, op.encoded)
|
frame, err = c.buildFrameWithState(op.ctx, op.msgType, op.msg, op.encoded, state)
|
||||||
}
|
}
|
||||||
// A profile-bound preparation can allocate a different body. Reserve the
|
// A profile-bound preparation can allocate a different body. Reserve the
|
||||||
// replacement before dropping the original prepared-body reservation. The
|
// replacement before dropping the original prepared-body reservation. The
|
||||||
|
|
@ -1875,11 +1922,30 @@ func (c *Conn) handleOutboundSend(state *outboundState, op outboundOp) {
|
||||||
reserved = len(frame.body)
|
reserved = len(frame.body)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err == nil && frame != nil && frameNeedsAck(frame.typeID) {
|
needsAck := err == nil && frame != nil && frameNeedsAck(frame.typeID)
|
||||||
// The queue reservation is transferred to pending after write. A frame larger
|
replaying := false
|
||||||
// than the per-Conn resend ceiling is rejected before any bytes hit the wire.
|
if needsAck && frame.replayMsgID() != 0 {
|
||||||
|
if existing := state.pending[frame.msgID]; existing != nil {
|
||||||
|
frame = existing
|
||||||
|
replaying = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if needsAck && !replaying {
|
||||||
|
// Transfer the producer reservation into the logical outbox before the
|
||||||
|
// first physical write. A write failure therefore remains replayable on a
|
||||||
|
// replacement connection with the same msg_id/seq_no.
|
||||||
if len(frame.body) > maxTrackedServerBytes {
|
if len(frame.body) > maxTrackedServerBytes {
|
||||||
err = ErrOutboundTrackedBudget
|
err = ErrOutboundTrackedBudget
|
||||||
|
} else {
|
||||||
|
frame.reservedBytes = reserved
|
||||||
|
frame.reservationBudget = reservationBudget
|
||||||
|
if admitErr := state.admitReserved(frame); admitErr != nil {
|
||||||
|
frame.reservedBytes = 0
|
||||||
|
frame.reservationBudget = nil
|
||||||
|
err = admitErr
|
||||||
|
} else {
|
||||||
|
reserved = 0
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if errors.Is(err, ErrOutboundTrackedBudget) {
|
if errors.Is(err, ErrOutboundTrackedBudget) {
|
||||||
|
|
@ -1894,18 +1960,6 @@ func (c *Conn) handleOutboundSend(state *outboundState, op outboundOp) {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = c.writeFrame(op.ctx, frame)
|
err = c.writeFrame(op.ctx, frame)
|
||||||
}
|
}
|
||||||
if err == nil && frame != nil && frameNeedsAck(frame.typeID) {
|
|
||||||
// 写成功后才提交 content seq_no 递增(peekSeqNo 已按当前计数算好本帧 seq_no)。
|
|
||||||
c.commitContentSeqNo()
|
|
||||||
frame.reservedBytes = reserved
|
|
||||||
frame.reservationBudget = reservationBudget
|
|
||||||
reserved = 0
|
|
||||||
if dropped := state.addReserved(frame); dropped > 0 {
|
|
||||||
for i := 0; i < dropped; i++ {
|
|
||||||
c.metrics.OutboundDropped("tracked_queue_overflow")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
queueWait := time.Since(op.enqueuedAt)
|
queueWait := time.Since(op.enqueuedAt)
|
||||||
bytes := 0
|
bytes := 0
|
||||||
typeID := uint32(0)
|
typeID := uint32(0)
|
||||||
|
|
@ -1914,7 +1968,7 @@ func (c *Conn) handleOutboundSend(state *outboundState, op outboundOp) {
|
||||||
typeID = frame.typeID
|
typeID = frame.typeID
|
||||||
}
|
}
|
||||||
c.metrics.OutboundSend(typeID, queueWait, bytes, err)
|
c.metrics.OutboundSend(typeID, queueWait, bytes, err)
|
||||||
op.finish(outboundResult{err: err})
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Conn) handleOutboundResend(state *outboundState, ctx context.Context, ids []int64) ([]byte, error) {
|
func (c *Conn) handleOutboundResend(state *outboundState, ctx context.Context, ids []int64) ([]byte, error) {
|
||||||
|
|
@ -2141,6 +2195,16 @@ func (c *Conn) ensureOutboundControlTrackedBudget() *outboundTrackedBudget {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Conn) buildFrame(ctx context.Context, t proto.MessageType, msg bin.Encoder, encoded *encodedOutboundMessage) (*outboundFrame, error) {
|
func (c *Conn) buildFrame(ctx context.Context, t proto.MessageType, msg bin.Encoder, encoded *encodedOutboundMessage) (*outboundFrame, error) {
|
||||||
|
return c.buildFrameWithState(ctx, t, msg, encoded, c.outboundState)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Conn) buildFrameWithState(
|
||||||
|
ctx context.Context,
|
||||||
|
t proto.MessageType,
|
||||||
|
msg bin.Encoder,
|
||||||
|
encoded *encodedOutboundMessage,
|
||||||
|
state *outboundState,
|
||||||
|
) (*outboundFrame, error) {
|
||||||
if encoded == nil {
|
if encoded == nil {
|
||||||
var err error
|
var err error
|
||||||
encoded, err = encodeOutboundMessage(msg)
|
encoded, err = encodeOutboundMessage(msg)
|
||||||
|
|
@ -2155,14 +2219,29 @@ func (c *Conn) buildFrame(ctx context.Context, t proto.MessageType, msg bin.Enco
|
||||||
return nil, ErrOutboundLayerBindingRequired
|
return nil, ErrOutboundLayerBindingRequired
|
||||||
}
|
}
|
||||||
content := frameNeedsAck(encoded.typeID)
|
content := frameNeedsAck(encoded.typeID)
|
||||||
msgID := c.msgID.New(t)
|
msgID := encoded.replayMsgID
|
||||||
|
seqNo := encoded.replaySeqNo
|
||||||
|
if msgID == 0 {
|
||||||
|
msgID = c.msgID.New(t)
|
||||||
|
if state != nil {
|
||||||
|
msgID = state.reserveMsgID(msgID)
|
||||||
|
seqNo = state.peekSeqNo(content)
|
||||||
|
} else {
|
||||||
|
seqNo = c.peekSeqNo(content)
|
||||||
|
}
|
||||||
|
}
|
||||||
return &outboundFrame{
|
return &outboundFrame{
|
||||||
msgID: msgID,
|
msgID: msgID,
|
||||||
seqNo: c.peekSeqNo(content),
|
seqNo: seqNo,
|
||||||
typeID: encoded.typeID,
|
typeID: encoded.typeID,
|
||||||
body: encoded.body,
|
body: encoded.body,
|
||||||
reqMsgID: encoded.reqMsgID,
|
reqMsgID: encoded.reqMsgID,
|
||||||
layer: encoded.layer,
|
layer: encoded.layer,
|
||||||
|
layerInvariant: encoded.layerInvariant,
|
||||||
|
priority: encoded.priority,
|
||||||
|
delivery: encoded.delivery,
|
||||||
|
compressed: encoded.compressed,
|
||||||
|
uncompressedBytes: encoded.uncompressedBytes,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2545,7 +2624,18 @@ func outboundRequestMsgID(msg bin.Encoder) int64 {
|
||||||
|
|
||||||
// addReserved 接管调用方已经取得的全局 body 预算。pending 的每个元素恰好对应一份
|
// addReserved 接管调用方已经取得的全局 body 预算。pending 的每个元素恰好对应一份
|
||||||
// reservation;后续只有 removePending/releaseAll 能归还。
|
// reservation;后续只有 removePending/releaseAll 能归还。
|
||||||
func (s *outboundState) addReserved(frame *outboundFrame) int {
|
func (s *outboundState) admitReserved(frame *outboundFrame) error {
|
||||||
|
if _, exists := s.pending[frame.msgID]; exists {
|
||||||
|
return fmt.Errorf("mtprotoedge: duplicate outbound msg_id inserted into resend tracking")
|
||||||
|
}
|
||||||
|
if len(s.pending) >= s.maxMessages || s.totalBytes > s.maxBytes-len(frame.body) {
|
||||||
|
return ErrOutboundTrackedBudget
|
||||||
|
}
|
||||||
|
s.insertReserved(frame)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *outboundState) insertReserved(frame *outboundFrame) {
|
||||||
if _, exists := s.pending[frame.msgID]; exists {
|
if _, exists := s.pending[frame.msgID]; exists {
|
||||||
panic("mtprotoedge: duplicate outbound msg_id inserted into resend tracking")
|
panic("mtprotoedge: duplicate outbound msg_id inserted into resend tracking")
|
||||||
}
|
}
|
||||||
|
|
@ -2561,6 +2651,15 @@ func (s *outboundState) addReserved(frame *outboundFrame) int {
|
||||||
}
|
}
|
||||||
s.byRequest[frame.reqMsgID] = frame.msgID
|
s.byRequest[frame.reqMsgID] = frame.msgID
|
||||||
}
|
}
|
||||||
|
if frameNeedsAck(frame.typeID) {
|
||||||
|
s.sentContentMessages++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// addReserved is retained as a focused-test helper. Production uses
|
||||||
|
// admitReserved and never evicts an unacknowledged frame.
|
||||||
|
func (s *outboundState) addReserved(frame *outboundFrame) int {
|
||||||
|
s.insertReserved(frame)
|
||||||
return s.shrinkPending()
|
return s.shrinkPending()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2619,6 +2718,8 @@ func (s *outboundState) markAcked(id int64) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// shrinkPending remains only for focused legacy state tests. Production
|
||||||
|
// admission never calls it: unacknowledged frames must not be silently evicted.
|
||||||
func (s *outboundState) shrinkPending() int {
|
func (s *outboundState) shrinkPending() int {
|
||||||
dropped := 0
|
dropped := 0
|
||||||
for (len(s.pending) > s.maxMessages || s.totalBytes > s.maxBytes) && len(s.order) > 0 {
|
for (len(s.pending) > s.maxMessages || s.totalBytes > s.maxBytes) && len(s.order) > 0 {
|
||||||
|
|
@ -2652,6 +2753,8 @@ func (s *outboundState) removePending(id int64) bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *outboundState) releaseAll() {
|
func (s *outboundState) releaseAll() {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
for _, frame := range s.pending {
|
for _, frame := range s.pending {
|
||||||
frame.body = nil
|
frame.body = nil
|
||||||
frame.releaseReservation(s.budget)
|
frame.releaseReservation(s.budget)
|
||||||
|
|
@ -2662,6 +2765,58 @@ func (s *outboundState) releaseAll() {
|
||||||
s.totalBytes = 0
|
s.totalBytes = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *outboundState) peekSeqNo(content bool) int32 {
|
||||||
|
seqNo := s.sentContentMessages * 2
|
||||||
|
if content {
|
||||||
|
seqNo++
|
||||||
|
}
|
||||||
|
return seqNo
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *outboundState) reserveMsgID(candidate int64) int64 {
|
||||||
|
if candidate <= s.lastMsgID {
|
||||||
|
candidate = s.lastMsgID + 4
|
||||||
|
}
|
||||||
|
for s.pending[candidate] != nil {
|
||||||
|
candidate += 4
|
||||||
|
}
|
||||||
|
s.lastMsgID = candidate
|
||||||
|
return candidate
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *outboundState) rpcResult(reqMsgID int64) (*encodedOutboundMessage, bool) {
|
||||||
|
if s == nil || reqMsgID == 0 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
msgID := s.byRequest[reqMsgID]
|
||||||
|
frame := s.pending[msgID]
|
||||||
|
if frame == nil || frame.typeID != proto.ResultTypeID || len(frame.body) == 0 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return &encodedOutboundMessage{
|
||||||
|
body: frame.body,
|
||||||
|
typeID: frame.typeID,
|
||||||
|
reqMsgID: frame.reqMsgID,
|
||||||
|
priority: frame.priority,
|
||||||
|
delivery: frame.delivery,
|
||||||
|
layer: frame.layer,
|
||||||
|
layerInvariant: frame.layerInvariant,
|
||||||
|
compressed: frame.compressed,
|
||||||
|
uncompressedBytes: frame.uncompressedBytes,
|
||||||
|
replayMsgID: frame.msgID,
|
||||||
|
replaySeqNo: frame.seqNo,
|
||||||
|
}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *outboundFrame) replayMsgID() int64 {
|
||||||
|
if f == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return f.msgID
|
||||||
|
}
|
||||||
|
|
||||||
func (f *outboundFrame) releaseReservation(defaultBudget *outboundTrackedBudget) {
|
func (f *outboundFrame) releaseReservation(defaultBudget *outboundTrackedBudget) {
|
||||||
if f == nil || f.reservedBytes <= 0 {
|
if f == nil || f.reservedBytes <= 0 {
|
||||||
return
|
return
|
||||||
|
|
|
||||||
20
internal/mtprotoedge/rpc_capacity_error.go
Normal file
20
internal/mtprotoedge/rpc_capacity_error.go
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
package mtprotoedge
|
||||||
|
|
||||||
|
import "github.com/iamxvbaba/td/mt"
|
||||||
|
|
||||||
|
const (
|
||||||
|
rpcWorkerBusyErrorCode = 500
|
||||||
|
rpcWorkerBusyErrorMessage = "WORKER_BUSY_TOO_LONG_RETRY"
|
||||||
|
)
|
||||||
|
|
||||||
|
// rpcWorkerBusyError reports transient server admission pressure. A local
|
||||||
|
// worker/queue/ledger ceiling is not Telegram flood control: returning a 420
|
||||||
|
// FLOOD_WAIT would falsely blame the account and makes clients surface or cache
|
||||||
|
// a rate-limit state. Official clients classify 5xx as transient; TDLib and
|
||||||
|
// DrKlo additionally recognize WORKER_BUSY_TOO_LONG_RETRY and retry with delay.
|
||||||
|
func rpcWorkerBusyError() *mt.RPCError {
|
||||||
|
return &mt.RPCError{
|
||||||
|
ErrorCode: rpcWorkerBusyErrorCode,
|
||||||
|
ErrorMessage: rpcWorkerBusyErrorMessage,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -18,6 +18,7 @@ var (
|
||||||
ErrRPCResultSubscriberCapacity = errors.New("mtproto rpc result subscriber capacity exhausted")
|
ErrRPCResultSubscriberCapacity = errors.New("mtproto rpc result subscriber capacity exhausted")
|
||||||
ErrRPCResultFlightInvalid = errors.New("mtproto rpc result in-flight claim is invalid")
|
ErrRPCResultFlightInvalid = errors.New("mtproto rpc result in-flight claim is invalid")
|
||||||
ErrRPCResultIdentityMismatch = errors.New("mtproto rpc result request identity mismatch")
|
ErrRPCResultIdentityMismatch = errors.New("mtproto rpc result request identity mismatch")
|
||||||
|
ErrRPCResultReplayUnavailable = errors.New("mtproto rpc result replay payload is unavailable")
|
||||||
ErrRPCAdmissionSeqExhausted = errors.New("mtproto rpc admission sequence exhausted")
|
ErrRPCAdmissionSeqExhausted = errors.New("mtproto rpc admission sequence exhausted")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -61,6 +62,7 @@ type rpcResultAcquireState uint8
|
||||||
|
|
||||||
const (
|
const (
|
||||||
rpcResultAcquireCompleted rpcResultAcquireState = iota + 1
|
rpcResultAcquireCompleted rpcResultAcquireState = iota + 1
|
||||||
|
rpcResultAcquireAcknowledged
|
||||||
rpcResultAcquirePending
|
rpcResultAcquirePending
|
||||||
rpcResultAcquireOwner
|
rpcResultAcquireOwner
|
||||||
)
|
)
|
||||||
|
|
@ -70,6 +72,8 @@ const (
|
||||||
//
|
//
|
||||||
// Exactly one state-specific field is non-nil:
|
// Exactly one state-specific field is non-nil:
|
||||||
// - completed: encoded contains the immutable completed rpc_result;
|
// - completed: encoded contains the immutable completed rpc_result;
|
||||||
|
// - acknowledged: the client ACKed the completed rpc_result, so only the
|
||||||
|
// compact request receipt remains and the duplicate must be ACK-only;
|
||||||
// - pending: waiter joins the already-running owner;
|
// - pending: waiter joins the already-running owner;
|
||||||
// - owner: owner must eventually complete through rpcResultCache.Put or Abort.
|
// - owner: owner must eventually complete through rpcResultCache.Put or Abort.
|
||||||
type rpcResultAcquire struct {
|
type rpcResultAcquire struct {
|
||||||
|
|
@ -82,7 +86,7 @@ type rpcResultAcquire struct {
|
||||||
executionOK bool
|
executionOK bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// rpcResultFlight is not part of the completed cache TTL lifecycle. Its
|
// rpcResultFlight is not part of the completed receipt TTL lifecycle. Its
|
||||||
// done channel is closed exactly once while holding the owning cache shard lock;
|
// done channel is closed exactly once while holding the owning cache shard lock;
|
||||||
// channel close publishes encoded/ok to all waiters without a waiter goroutine.
|
// channel close publishes encoded/ok to all waiters without a waiter goroutine.
|
||||||
type rpcResultFlight struct {
|
type rpcResultFlight struct {
|
||||||
|
|
@ -93,6 +97,11 @@ type rpcResultFlight struct {
|
||||||
executionDone bool
|
executionDone bool
|
||||||
executionOK bool
|
executionOK bool
|
||||||
executionSubscribers []func(bool)
|
executionSubscribers []func(bool)
|
||||||
|
// acknowledged is set only from the sole outbound actor's server-msg-id to
|
||||||
|
// request-msg-id mapping. It can win the race with the asynchronous Put;
|
||||||
|
// publication then keeps only the compact receipt while still resolving
|
||||||
|
// waiters with the immutable result that was already written on the wire.
|
||||||
|
acknowledged bool
|
||||||
// subscriberSlots counts callbacks retained by this pending flight. Result
|
// subscriberSlots counts callbacks retained by this pending flight. Result
|
||||||
// and execution callbacks are charged independently; a replay alias installs
|
// and execution callbacks are charged independently; a replay alias installs
|
||||||
// both atomically so a capacity failure cannot leave half an alias behind.
|
// both atomically so a capacity failure cannot leave half an alias behind.
|
||||||
|
|
@ -465,7 +474,7 @@ func (l *rpcResultFlightLimit) snapshot() int64 {
|
||||||
|
|
||||||
// Acquire atomically returns a completed result, joins the existing in-flight
|
// Acquire atomically returns a completed result, joins the existing in-flight
|
||||||
// owner, or installs the unique owner lease. Pending entries have a separate
|
// owner, or installs the unique owner lease. Pending entries have a separate
|
||||||
// lifecycle from completed cache TTL but reserve the same global/auth/session
|
// lifecycle from completed receipt TTL but reserve the same global/auth/session
|
||||||
// ownership that Put later transfers to a completed result or tombstone.
|
// ownership that Put later transfers to a completed result or tombstone.
|
||||||
func (c *rpcResultCache) Acquire(authKeyID [8]byte, sessionID, reqMsgID int64) (rpcResultAcquire, error) {
|
func (c *rpcResultCache) Acquire(authKeyID [8]byte, sessionID, reqMsgID int64) (rpcResultAcquire, error) {
|
||||||
return c.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{})
|
return c.acquire(authKeyID, sessionID, reqMsgID, rpcResultRequestIdentity{})
|
||||||
|
|
@ -544,10 +553,35 @@ func (c *rpcResultCache) acquire(
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
return rpcResultAcquire{}, identityMismatch(entry.identity)
|
return rpcResultAcquire{}, identityMismatch(entry.identity)
|
||||||
}
|
}
|
||||||
if entry.capacity || entry.encoded == nil {
|
if entry.acknowledged {
|
||||||
|
result := rpcResultAcquire{
|
||||||
|
state: rpcResultAcquireAcknowledged, admissionSeq: entry.admissionSeq,
|
||||||
|
executionKnown: entry.executionKnown, executionOK: entry.executionOK,
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
if entry.capacity {
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
return rpcResultAcquire{}, ErrRPCResultFlightCapacity
|
return rpcResultAcquire{}, ErrRPCResultFlightCapacity
|
||||||
}
|
}
|
||||||
|
if c.sessions != nil {
|
||||||
|
result := rpcResultAcquire{
|
||||||
|
state: rpcResultAcquireCompleted, admissionSeq: entry.admissionSeq,
|
||||||
|
executionKnown: entry.executionKnown, executionOK: entry.executionOK,
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
encoded, replayable := c.sessions.rpcResult(authKeyID, sessionID, reqMsgID)
|
||||||
|
if !replayable {
|
||||||
|
return rpcResultAcquire{}, ErrRPCResultReplayUnavailable
|
||||||
|
}
|
||||||
|
result.encoded = encoded
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
if entry.encoded == nil {
|
||||||
|
s.mu.Unlock()
|
||||||
|
return rpcResultAcquire{}, ErrRPCResultReplayUnavailable
|
||||||
|
}
|
||||||
result := rpcResultAcquire{
|
result := rpcResultAcquire{
|
||||||
state: rpcResultAcquireCompleted, admissionSeq: entry.admissionSeq, encoded: entry.encoded,
|
state: rpcResultAcquireCompleted, admissionSeq: entry.admissionSeq, encoded: entry.encoded,
|
||||||
executionKnown: entry.executionKnown, executionOK: entry.executionOK,
|
executionKnown: entry.executionKnown, executionOK: entry.executionOK,
|
||||||
|
|
@ -560,6 +594,14 @@ func (c *rpcResultCache) acquire(
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
return rpcResultAcquire{}, identityMismatch(flight.identity)
|
return rpcResultAcquire{}, identityMismatch(flight.identity)
|
||||||
}
|
}
|
||||||
|
if flight.acknowledged {
|
||||||
|
result := rpcResultAcquire{
|
||||||
|
state: rpcResultAcquireAcknowledged, admissionSeq: flight.admissionSeq,
|
||||||
|
executionKnown: flight.executionDone, executionOK: flight.executionOK,
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
result := rpcResultAcquire{
|
result := rpcResultAcquire{
|
||||||
state: rpcResultAcquirePending,
|
state: rpcResultAcquirePending,
|
||||||
admissionSeq: flight.admissionSeq,
|
admissionSeq: flight.admissionSeq,
|
||||||
|
|
@ -621,7 +663,7 @@ func (c *rpcResultCache) acquire(
|
||||||
}
|
}
|
||||||
|
|
||||||
// completeRPCResultFlightLocked publishes encoded to the current owner claim.
|
// completeRPCResultFlightLocked publishes encoded to the current owner claim.
|
||||||
// The caller must hold s.mu and must publish the completed cache entry first.
|
// The caller must hold s.mu and must publish the completed receipt first.
|
||||||
func (c *rpcResultCache) completeRPCResultFlightLocked(
|
func (c *rpcResultCache) completeRPCResultFlightLocked(
|
||||||
s *rpcResultCacheShard,
|
s *rpcResultCacheShard,
|
||||||
key rpcResultCacheKey,
|
key rpcResultCacheKey,
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ package mtprotoedge
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"container/list"
|
"container/list"
|
||||||
|
"context"
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
"hash/maphash"
|
"hash/maphash"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
@ -26,12 +27,6 @@ const (
|
||||||
rpcResultCacheSessionMaxEntries = 1 << 14
|
rpcResultCacheSessionMaxEntries = 1 << 14
|
||||||
rpcResultCacheSessionMaxBytes = 16 << 20
|
rpcResultCacheSessionMaxBytes = 16 << 20
|
||||||
rpcResultFlightMaxPendingPerAuth = 1 << 11
|
rpcResultFlightMaxPendingPerAuth = 1 << 11
|
||||||
// Keep every transport-legal rpc_result cacheable. Converting the constant
|
|
||||||
// difference to uint64 intentionally fails compilation if a future transport
|
|
||||||
// limit grows beyond the completed-result budget.
|
|
||||||
_ = uint64(rpcResultCacheMaxBytes - maxOutboundBodyBytes)
|
|
||||||
_ = uint64(rpcResultCacheAuthMaxBytes - maxOutboundBodyBytes)
|
|
||||||
_ = uint64(rpcResultCacheSessionMaxBytes - maxOutboundBodyBytes)
|
|
||||||
// rpcResultCacheShards hashes the complete replay identity with a random
|
// rpcResultCacheShards hashes the complete replay identity with a random
|
||||||
// per-instance maphash seed. Including req_msg_id spreads one hot session's
|
// per-instance maphash seed. Including req_msg_id spreads one hot session's
|
||||||
// independent requests instead of forcing them through one mutex. The shard
|
// independent requests instead of forcing them through one mutex. The shard
|
||||||
|
|
@ -46,7 +41,10 @@ type rpcResultCacheKey struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
type rpcResultCacheEntry struct {
|
type rpcResultCacheEntry struct {
|
||||||
key rpcResultCacheKey
|
key rpcResultCacheKey
|
||||||
|
// encoded is used only by legacy focused constructors whose cache has no
|
||||||
|
// SessionManager. Production rows are metadata receipts and always leave it
|
||||||
|
// nil; exact bytes have one owner in the logical-session outbox.
|
||||||
encoded *encodedOutboundMessage
|
encoded *encodedOutboundMessage
|
||||||
size int
|
size int
|
||||||
expiresAt time.Time
|
expiresAt time.Time
|
||||||
|
|
@ -54,6 +52,10 @@ type rpcResultCacheEntry struct {
|
||||||
admissionSeq uint64
|
admissionSeq uint64
|
||||||
executionKnown bool
|
executionKnown bool
|
||||||
executionOK bool
|
executionOK bool
|
||||||
|
// acknowledged exists only for the ACK-before-Put race. A completed row is
|
||||||
|
// removed immediately on ACK; unlike the previous design, success receipts
|
||||||
|
// are not retained for the rest of the 331-second horizon.
|
||||||
|
acknowledged bool
|
||||||
// capacity marks a bounded replay tombstone. The original owner and its
|
// capacity marks a bounded replay tombstone. The original owner and its
|
||||||
// already-joined waiters received encoded, but the byte budget could not
|
// already-joined waiters received encoded, but the byte budget could not
|
||||||
// retain that body. Keeping the immutable identity until TTL prevents a
|
// retain that body. Keeping the immutable identity until TTL prevents a
|
||||||
|
|
@ -84,6 +86,10 @@ type rpcResultCache struct {
|
||||||
flightLimit rpcResultFlightLimit
|
flightLimit rpcResultFlightLimit
|
||||||
subscriberBudget *rpcResultSubscriberBudget
|
subscriberBudget *rpcResultSubscriberBudget
|
||||||
subscriberPerFlight int
|
subscriberPerFlight int
|
||||||
|
// sessions is the production payload source. Completed ledger rows retain
|
||||||
|
// only request identity/outcome/TTL metadata; exact bytes stay solely in the
|
||||||
|
// logical-session unacked outbox. Nil is kept as a legacy unit-test seam.
|
||||||
|
sessions *SessionManager
|
||||||
// nextAdmissionSeq is the process-wide ordering authority for auth-key
|
// nextAdmissionSeq is the process-wide ordering authority for auth-key
|
||||||
// shared Layer defaults. Exact owners allocate once; joins/replays retain the
|
// shared Layer defaults. Exact owners allocate once; joins/replays retain the
|
||||||
// owner's value from their flight/completed descriptor.
|
// owner's value from their flight/completed descriptor.
|
||||||
|
|
@ -172,6 +178,7 @@ type rpcResultCacheCapacity struct {
|
||||||
subscriberMaxAuth int
|
subscriberMaxAuth int
|
||||||
subscriberMaxSession int
|
subscriberMaxSession int
|
||||||
subscriberMaxPerFlight int
|
subscriberMaxPerFlight int
|
||||||
|
sessions *SessionManager
|
||||||
}
|
}
|
||||||
|
|
||||||
func newRPCResultCacheWithFairCapacity(now func() time.Time, capacity rpcResultCacheCapacity) *rpcResultCache {
|
func newRPCResultCacheWithFairCapacity(now func() time.Time, capacity rpcResultCacheCapacity) *rpcResultCache {
|
||||||
|
|
@ -214,7 +221,7 @@ func newRPCResultCacheWithFairCapacity(now func() time.Time, capacity rpcResultC
|
||||||
if capacity.subscriberMaxPerFlight <= 0 {
|
if capacity.subscriberMaxPerFlight <= 0 {
|
||||||
capacity.subscriberMaxPerFlight = rpcResultSubscriberMaxPerFlight
|
capacity.subscriberMaxPerFlight = rpcResultSubscriberMaxPerFlight
|
||||||
}
|
}
|
||||||
c := &rpcResultCache{hashSeed: maphash.MakeSeed()}
|
c := &rpcResultCache{hashSeed: maphash.MakeSeed(), sessions: capacity.sessions}
|
||||||
c.completedBytes.max = capacity.globalMaxBytes
|
c.completedBytes.max = capacity.globalMaxBytes
|
||||||
c.completedEntries.max = int64(capacity.globalMaxEntries)
|
c.completedEntries.max = int64(capacity.globalMaxEntries)
|
||||||
c.flightLimit.max = int64(capacity.maxPending)
|
c.flightLimit.max = int64(capacity.maxPending)
|
||||||
|
|
@ -267,21 +274,62 @@ func (c *rpcResultCache) Get(authKeyID [8]byte, sessionID, reqMsgID int64) (*enc
|
||||||
now := s.now()
|
now := s.now()
|
||||||
|
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
|
||||||
|
|
||||||
elem, ok := s.byKey[key]
|
elem, ok := s.byKey[key]
|
||||||
if !ok {
|
if !ok {
|
||||||
|
s.mu.Unlock()
|
||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
entry := elem.Value.(*rpcResultCacheEntry)
|
entry := elem.Value.(*rpcResultCacheEntry)
|
||||||
if !entry.expiresAt.After(now) {
|
if !entry.expiresAt.After(now) {
|
||||||
s.removeElement(elem)
|
s.removeElement(elem)
|
||||||
|
s.mu.Unlock()
|
||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
if entry.capacity || entry.encoded == nil {
|
if entry.capacity || entry.acknowledged {
|
||||||
|
s.mu.Unlock()
|
||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
return entry.encoded, true
|
if c.sessions != nil {
|
||||||
|
s.mu.Unlock()
|
||||||
|
return c.sessions.rpcResult(authKeyID, sessionID, reqMsgID)
|
||||||
|
}
|
||||||
|
encoded := entry.encoded
|
||||||
|
s.mu.Unlock()
|
||||||
|
return encoded, encoded != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Acknowledge removes a completed rpc_result receipt immediately.
|
||||||
|
// The caller must have resolved reqMsgID from a client msgs_ack through the
|
||||||
|
// sole outbound actor's tracked server-message mapping; an untrusted request
|
||||||
|
// msg_id must never call this method directly.
|
||||||
|
//
|
||||||
|
// ACK can race the asynchronous terminal Put. A pending flight records the
|
||||||
|
// event so Put resolves current waiters but does not retain a completed row.
|
||||||
|
// This follows the logical-session model: once the client has acknowledged the
|
||||||
|
// exact server message, neither its payload nor its duplicate receipt is useful.
|
||||||
|
func (c *rpcResultCache) Acknowledge(authKeyID [8]byte, sessionID, reqMsgID int64) bool {
|
||||||
|
if c == nil || reqMsgID == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||||
|
s := c.shard(key)
|
||||||
|
now := s.now()
|
||||||
|
s.mu.Lock()
|
||||||
|
s.expireLocked(now)
|
||||||
|
|
||||||
|
if elem := s.byKey[key]; elem != nil {
|
||||||
|
s.removeElement(elem)
|
||||||
|
s.mu.Unlock()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if flight := s.pending[key]; flight != nil {
|
||||||
|
flight.acknowledged = true
|
||||||
|
s.mu.Unlock()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// ObserveDependency returns a waiter for an admitted in-flight dependency, a
|
// ObserveDependency returns a waiter for an admitted in-flight dependency, a
|
||||||
|
|
@ -337,7 +385,15 @@ func (c *rpcResultCache) Put(authKeyID [8]byte, sessionID, reqMsgID int64, encod
|
||||||
func (c *rpcResultCache) putOnce(authKeyID [8]byte, sessionID, reqMsgID int64, encoded *encodedOutboundMessage) bool {
|
func (c *rpcResultCache) putOnce(authKeyID [8]byte, sessionID, reqMsgID int64, encoded *encodedOutboundMessage) bool {
|
||||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: reqMsgID}
|
||||||
s := c.shard(key)
|
s := c.shard(key)
|
||||||
|
logicalOutbox := c.sessions != nil
|
||||||
|
logicalReplayable := false
|
||||||
|
if logicalOutbox {
|
||||||
|
_, logicalReplayable = c.sessions.rpcResult(authKeyID, sessionID, reqMsgID)
|
||||||
|
}
|
||||||
accountedSize := len(encoded.body)
|
accountedSize := len(encoded.body)
|
||||||
|
if logicalOutbox {
|
||||||
|
accountedSize = 1
|
||||||
|
}
|
||||||
if accountedSize < 1 {
|
if accountedSize < 1 {
|
||||||
// Every owner reserves one byte at admission. Keeping zero-length results
|
// Every owner reserves one byte at admission. Keeping zero-length results
|
||||||
// at the same minimum makes entry and byte capacity linearizable.
|
// at the same minimum makes entry and byte capacity linearizable.
|
||||||
|
|
@ -361,17 +417,17 @@ func (c *rpcResultCache) putOnce(authKeyID [8]byte, sessionID, reqMsgID int64, e
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
identity, admissionSeq, executionKnown, executionOK := rpcResultFlightMetadataLocked(s, key)
|
identity, admissionSeq, executionKnown, executionOK, acknowledged := rpcResultFlightMetadataLocked(s, key)
|
||||||
var oldEntry *rpcResultCacheEntry
|
var oldEntry *rpcResultCacheEntry
|
||||||
if old != nil {
|
if old != nil {
|
||||||
oldEntry = old.Value.(*rpcResultCacheEntry)
|
oldEntry = old.Value.(*rpcResultCacheEntry)
|
||||||
if flight == nil {
|
if flight == nil {
|
||||||
// A defensive duplicate terminal publication must never downgrade
|
// The first terminal publication is immutable, whether it retained an
|
||||||
// completed dependency/identity metadata after its flight disappeared.
|
// legacy inline body, receipt or capacity tombstone. A
|
||||||
identity = oldEntry.identity
|
// stale callback must not replace exact bytes, extend TTL, resurrect an
|
||||||
admissionSeq = oldEntry.admissionSeq
|
// ACKed body or valid receipt.
|
||||||
executionKnown = oldEntry.executionKnown
|
s.mu.Unlock()
|
||||||
executionOK = oldEntry.executionOK
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -396,7 +452,24 @@ func (c *rpcResultCache) putOnce(authKeyID [8]byte, sessionID, reqMsgID int64, e
|
||||||
retainedSize := accountedSize
|
retainedSize := accountedSize
|
||||||
retained := encoded
|
retained := encoded
|
||||||
capacity := false
|
capacity := false
|
||||||
if !reservation.resizeBytes(accountedSize) {
|
if acknowledged {
|
||||||
|
const receiptSize = 1
|
||||||
|
if !reservation.resizeBytes(receiptSize) {
|
||||||
|
s.mu.Unlock()
|
||||||
|
panic("mtprotoedge: acknowledged rpc result owner lost its receipt reservation")
|
||||||
|
}
|
||||||
|
retainedSize = receiptSize
|
||||||
|
retained = nil
|
||||||
|
} else if logicalOutbox {
|
||||||
|
const receiptSize = 1
|
||||||
|
if !reservation.resizeBytes(receiptSize) {
|
||||||
|
s.mu.Unlock()
|
||||||
|
panic("mtprotoedge: logical rpc result owner lost its receipt reservation")
|
||||||
|
}
|
||||||
|
retainedSize = receiptSize
|
||||||
|
retained = nil
|
||||||
|
capacity = !logicalReplayable
|
||||||
|
} else if !reservation.resizeBytes(accountedSize) {
|
||||||
if flight == nil {
|
if flight == nil {
|
||||||
// A direct replacement cannot discard the prior replay body. Leave it
|
// A direct replacement cannot discard the prior replay body. Leave it
|
||||||
// untouched and let Put perform one cross-shard expiry reap before its
|
// untouched and let Put perform one cross-shard expiry reap before its
|
||||||
|
|
@ -436,6 +509,7 @@ func (c *rpcResultCache) putOnce(authKeyID [8]byte, sessionID, reqMsgID int64, e
|
||||||
admissionSeq: admissionSeq,
|
admissionSeq: admissionSeq,
|
||||||
executionKnown: executionKnown,
|
executionKnown: executionKnown,
|
||||||
executionOK: executionOK,
|
executionOK: executionOK,
|
||||||
|
acknowledged: acknowledged,
|
||||||
capacity: capacity,
|
capacity: capacity,
|
||||||
reservation: reservation,
|
reservation: reservation,
|
||||||
}
|
}
|
||||||
|
|
@ -446,6 +520,11 @@ func (c *rpcResultCache) putOnce(authKeyID [8]byte, sessionID, reqMsgID int64, e
|
||||||
// Resolve the independent in-flight entry only after either the completed
|
// Resolve the independent in-flight entry only after either the completed
|
||||||
// result or its replay tombstone is published under the same shard lock.
|
// result or its replay tombstone is published under the same shard lock.
|
||||||
subscribers, executionSubscribers, executionOK := c.completeRPCResultFlightLocked(s, key, encoded)
|
subscribers, executionSubscribers, executionOK := c.completeRPCResultFlightLocked(s, key, encoded)
|
||||||
|
if acknowledged {
|
||||||
|
// ACK won before terminal publication. Current subscribers still receive
|
||||||
|
// encoded below, but no post-ACK receipt survives this critical section.
|
||||||
|
s.removeElement(elem)
|
||||||
|
}
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
for _, subscriber := range subscribers {
|
for _, subscriber := range subscribers {
|
||||||
subscriber(encoded, true)
|
subscriber(encoded, true)
|
||||||
|
|
@ -461,11 +540,12 @@ func rpcResultFlightMetadataLocked(s *rpcResultCacheShard, key rpcResultCacheKey
|
||||||
uint64,
|
uint64,
|
||||||
bool,
|
bool,
|
||||||
bool,
|
bool,
|
||||||
|
bool,
|
||||||
) {
|
) {
|
||||||
if flight := s.pending[key]; flight != nil {
|
if flight := s.pending[key]; flight != nil {
|
||||||
return flight.identity, flight.admissionSeq, flight.executionDone, flight.executionOK
|
return flight.identity, flight.admissionSeq, flight.executionDone, flight.executionOK, flight.acknowledged
|
||||||
}
|
}
|
||||||
return rpcResultRequestIdentity{}, 0, false, false
|
return rpcResultRequestIdentity{}, 0, false, false, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// expireCompletedResults performs the cold-path cross-shard reap used only
|
// expireCompletedResults performs the cold-path cross-shard reap used only
|
||||||
|
|
@ -484,6 +564,37 @@ func (c *rpcResultCache) expireCompletedResults() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *rpcResultCache) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *rpcResultCache) CloseContext(context.Context) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetCompletedSession removes every terminal receipt for a logical session.
|
||||||
|
// SessionManager invokes it only after the session's physical producers have
|
||||||
|
// converged, so no pending owner should remain and no later terminal callback
|
||||||
|
// can recreate a receipt for the destroyed outbox.
|
||||||
|
func (c *rpcResultCache) forgetCompletedSession(authKeyID [8]byte, sessionID int64) {
|
||||||
|
if c == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for i := range c.shards {
|
||||||
|
s := &c.shards[i]
|
||||||
|
s.mu.Lock()
|
||||||
|
for elem := s.order.Front(); elem != nil; {
|
||||||
|
next := elem.Next()
|
||||||
|
entry := elem.Value.(*rpcResultCacheEntry)
|
||||||
|
if entry.key.authKeyID == authKeyID && entry.key.sessionID == sessionID {
|
||||||
|
s.removeElement(elem)
|
||||||
|
}
|
||||||
|
elem = next
|
||||||
|
}
|
||||||
|
s.mu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *rpcResultCacheShard) expireLocked(now time.Time) {
|
func (s *rpcResultCacheShard) expireLocked(now time.Time) {
|
||||||
for elem := s.order.Front(); elem != nil; {
|
for elem := s.order.Front(); elem != nil; {
|
||||||
next := elem.Next()
|
next := elem.Next()
|
||||||
|
|
|
||||||
|
|
@ -187,8 +187,8 @@ func TestRPCResultCacheFairReservationLifecycleReturnsEveryScope(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
cache.Put(auth, 1, 102, &encodedOutboundMessage{body: make([]byte, 2)})
|
cache.Put(auth, 1, 102, &encodedOutboundMessage{body: make([]byte, 2)})
|
||||||
if got := cache.completedBytes.snapshot(); got != 3 {
|
if got := cache.completedBytes.snapshot(); got != 5 {
|
||||||
t.Fatalf("replacement did not resize global bytes: %d", got)
|
t.Fatalf("duplicate publication changed immutable global bytes: %d", got)
|
||||||
}
|
}
|
||||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||||
_, _ = cache.Get(auth, 1, 102)
|
_, _ = cache.Get(auth, 1, 102)
|
||||||
|
|
@ -536,7 +536,7 @@ func TestRPCResultCacheDuplicatePutPreservesCompletedExecutionMetadata(t *testin
|
||||||
cache.Put(keyID, sessionID, reqMsgID, second)
|
cache.Put(keyID, sessionID, reqMsgID, second)
|
||||||
|
|
||||||
replay, err := cache.Acquire(keyID, sessionID, reqMsgID)
|
replay, err := cache.Acquire(keyID, sessionID, reqMsgID)
|
||||||
if err != nil || replay.state != rpcResultAcquireCompleted || replay.encoded != second ||
|
if err != nil || replay.state != rpcResultAcquireCompleted || replay.encoded != first ||
|
||||||
!replay.executionKnown || !replay.executionOK {
|
!replay.executionKnown || !replay.executionOK {
|
||||||
t.Fatalf("duplicate Put metadata = %#v, err=%v", replay, err)
|
t.Fatalf("duplicate Put metadata = %#v, err=%v", replay, err)
|
||||||
}
|
}
|
||||||
|
|
@ -609,18 +609,18 @@ func TestRPCResultCacheByteBudgetReturnsOnReplaceExpiryAndCapacity(t *testing.T)
|
||||||
|
|
||||||
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 4)})
|
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 4)})
|
||||||
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 7)})
|
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 7)})
|
||||||
if got := cache.completedBytes.snapshot(); got != 7 {
|
if got := cache.completedBytes.snapshot(); got != 4 {
|
||||||
t.Fatalf("completed bytes after growing replacement = %d, want 7", got)
|
t.Fatalf("completed bytes after duplicate publication = %d, want 4", got)
|
||||||
}
|
}
|
||||||
if usage := cache.fairBudget.sessionSnapshot(keyID, 1); usage.entries != 1 || usage.bytes != 7 {
|
if usage := cache.fairBudget.sessionSnapshot(keyID, 1); usage.entries != 1 || usage.bytes != 4 {
|
||||||
t.Fatalf("replacement fair reservation after growth = %#v", usage)
|
t.Fatalf("immutable fair reservation after duplicate = %#v", usage)
|
||||||
}
|
}
|
||||||
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 2)})
|
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 2)})
|
||||||
if got := cache.completedBytes.snapshot(); got != 2 {
|
if got := cache.completedBytes.snapshot(); got != 4 {
|
||||||
t.Fatalf("completed bytes after shrinking replacement = %d, want 2", got)
|
t.Fatalf("completed bytes after second duplicate = %d, want 4", got)
|
||||||
}
|
}
|
||||||
if usage := cache.fairBudget.sessionSnapshot(keyID, 1); usage.entries != 1 || usage.bytes != 2 {
|
if usage := cache.fairBudget.sessionSnapshot(keyID, 1); usage.entries != 1 || usage.bytes != 4 {
|
||||||
t.Fatalf("replacement fair reservation after shrink = %#v", usage)
|
t.Fatalf("immutable fair reservation after second duplicate = %#v", usage)
|
||||||
}
|
}
|
||||||
|
|
||||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||||
|
|
@ -712,23 +712,19 @@ func TestRPCResultCacheByteCapacityReclaimsExpiredAcrossShards(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRPCResultCacheServerOptionsPropagateFairLimits(t *testing.T) {
|
func TestRPCResultCacheServerOptionsPropagateFairLimits(t *testing.T) {
|
||||||
sessionBytes := int64(maxOutboundBodyBytes)
|
|
||||||
s := New(Options{
|
s := New(Options{
|
||||||
RPCGlobalMaxTasks: 6,
|
RPCGlobalMaxTasks: 6,
|
||||||
RPCResultCacheMaxEntries: 12,
|
RPCResultCacheMaxEntries: 12,
|
||||||
RPCResultCacheMaxBytes: sessionBytes + 2048,
|
|
||||||
RPCResultCacheAuthMaxEntries: 8,
|
RPCResultCacheAuthMaxEntries: 8,
|
||||||
RPCResultCacheAuthMaxBytes: sessionBytes + 1024,
|
|
||||||
RPCResultCacheSessionMaxEntries: 4,
|
RPCResultCacheSessionMaxEntries: 4,
|
||||||
RPCResultCacheSessionMaxBytes: sessionBytes,
|
|
||||||
RPCResultPendingPerAuth: 3,
|
RPCResultPendingPerAuth: 3,
|
||||||
})
|
})
|
||||||
if s.rpcResults.completedEntries.max != 12 || s.rpcResults.completedBytes.max != sessionBytes+2048 {
|
if s.rpcResults.completedEntries.max != 12 || s.rpcResults.completedBytes.max != 12 {
|
||||||
t.Fatalf("global option propagation = %d/%d", s.rpcResults.completedEntries.max, s.rpcResults.completedBytes.max)
|
t.Fatalf("global option propagation = %d/%d", s.rpcResults.completedEntries.max, s.rpcResults.completedBytes.max)
|
||||||
}
|
}
|
||||||
budget := s.rpcResults.fairBudget
|
budget := s.rpcResults.fairBudget
|
||||||
if budget.authLimit.entries != 8 || budget.authLimit.bytes != sessionBytes+1024 ||
|
if budget.authLimit.entries != 8 || budget.authLimit.bytes != 8 ||
|
||||||
budget.sessionLimit.entries != 4 || budget.sessionLimit.bytes != sessionBytes || budget.pendingPerAuth != 3 {
|
budget.sessionLimit.entries != 4 || budget.sessionLimit.bytes != 4 || budget.pendingPerAuth != 3 {
|
||||||
t.Fatalf("fair option propagation = auth:%#v session:%#v pending:%d",
|
t.Fatalf("fair option propagation = auth:%#v session:%#v pending:%d",
|
||||||
budget.authLimit, budget.sessionLimit, budget.pendingPerAuth)
|
budget.authLimit, budget.sessionLimit, budget.pendingPerAuth)
|
||||||
}
|
}
|
||||||
|
|
@ -738,11 +734,8 @@ func TestRPCResultCacheServerOptionsFailFast(t *testing.T) {
|
||||||
base := Options{
|
base := Options{
|
||||||
RPCGlobalMaxTasks: 6,
|
RPCGlobalMaxTasks: 6,
|
||||||
RPCResultCacheMaxEntries: 12,
|
RPCResultCacheMaxEntries: 12,
|
||||||
RPCResultCacheMaxBytes: 64 << 20,
|
|
||||||
RPCResultCacheAuthMaxEntries: 8,
|
RPCResultCacheAuthMaxEntries: 8,
|
||||||
RPCResultCacheAuthMaxBytes: 32 << 20,
|
|
||||||
RPCResultCacheSessionMaxEntries: 4,
|
RPCResultCacheSessionMaxEntries: 4,
|
||||||
RPCResultCacheSessionMaxBytes: 16 << 20,
|
|
||||||
RPCResultPendingPerAuth: 3,
|
RPCResultPendingPerAuth: 3,
|
||||||
}
|
}
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
|
|
@ -750,7 +743,6 @@ func TestRPCResultCacheServerOptionsFailFast(t *testing.T) {
|
||||||
mutate func(*Options)
|
mutate func(*Options)
|
||||||
}{
|
}{
|
||||||
{name: "entry hierarchy", mutate: func(o *Options) { o.RPCResultCacheAuthMaxEntries = 13 }},
|
{name: "entry hierarchy", mutate: func(o *Options) { o.RPCResultCacheAuthMaxEntries = 13 }},
|
||||||
{name: "body does not fit session", mutate: func(o *Options) { o.RPCResultCacheSessionMaxBytes = maxOutboundBodyBytes - 1 }},
|
|
||||||
{name: "pending hierarchy", mutate: func(o *Options) { o.RPCResultPendingPerAuth = 7 }},
|
{name: "pending hierarchy", mutate: func(o *Options) { o.RPCResultPendingPerAuth = 7 }},
|
||||||
}
|
}
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
|
|
|
||||||
|
|
@ -401,8 +401,8 @@ func TestRPCResultFailureAfterIntentionalTerminalDoesNotCloseTransferLease(t *te
|
||||||
|
|
||||||
// Session replacement publishes terminal before transferring the physical
|
// Session replacement publishes terminal before transferring the physical
|
||||||
// lease. A late old-generation result sees ErrConnClosed at producer admission;
|
// lease. A late old-generation result sees ErrConnClosed at producer admission;
|
||||||
// it may publish cache-only, but must not upgrade that intentional fence into
|
// it publishes a metadata-only capacity tombstone, but must not upgrade that
|
||||||
// a physical close that makes Transfer fail.
|
// intentional fence into a physical close that makes Transfer fail.
|
||||||
oldConn.beginTerminalShutdown()
|
oldConn.beginTerminalShutdown()
|
||||||
err = s.sendResult(context.Background(), oldConn, reqMsgID, exactTestRPCResult(&tg.Config{ThisDC: 2}))
|
err = s.sendResult(context.Background(), oldConn, reqMsgID, exactTestRPCResult(&tg.Config{ThisDC: 2}))
|
||||||
if !errors.Is(err, ErrOutboundTrackedBudget) {
|
if !errors.Is(err, ErrOutboundTrackedBudget) {
|
||||||
|
|
@ -423,9 +423,8 @@ func TestRPCResultFailureAfterIntentionalTerminalDoesNotCloseTransferLease(t *te
|
||||||
if tr.closed.Load() || !newConn.isPhysicalTransportCurrentOpen() {
|
if tr.closed.Load() || !newConn.isPhysicalTransportCurrentOpen() {
|
||||||
t.Fatalf("stale close after transfer: raw_closed=%v current_open=%v", tr.closed.Load(), newConn.isPhysicalTransportCurrentOpen())
|
t.Fatalf("stale close after transfer: raw_closed=%v current_open=%v", tr.closed.Load(), newConn.isPhysicalTransportCurrentOpen())
|
||||||
}
|
}
|
||||||
completed, acquireErr := s.rpcResults.Acquire(key.ID, oldConn.sessionID, reqMsgID)
|
if _, acquireErr := s.rpcResults.Acquire(key.ID, oldConn.sessionID, reqMsgID); !errors.Is(acquireErr, ErrRPCResultFlightCapacity) {
|
||||||
if acquireErr != nil || completed.state != rpcResultAcquireCompleted || completed.encoded == nil {
|
t.Fatalf("late result tombstone err=%v, want %v", acquireErr, ErrRPCResultFlightCapacity)
|
||||||
t.Fatalf("late result cache = %+v err=%v", completed, acquireErr)
|
|
||||||
}
|
}
|
||||||
newConn.ForceClose()
|
newConn.ForceClose()
|
||||||
}
|
}
|
||||||
|
|
@ -459,9 +458,8 @@ func TestRPCResultPublishesBeforePathologicalPhysicalCloseReturns(t *testing.T)
|
||||||
case <-time.After(time.Second):
|
case <-time.After(time.Second):
|
||||||
t.Fatal("pathological physical Close blocked result publication")
|
t.Fatal("pathological physical Close blocked result publication")
|
||||||
}
|
}
|
||||||
completed, acquireErr := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
|
if _, acquireErr := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID); !errors.Is(acquireErr, ErrRPCResultFlightCapacity) {
|
||||||
if acquireErr != nil || completed.state != rpcResultAcquireCompleted || completed.encoded == nil {
|
t.Fatalf("result tombstone before raw Close return err=%v, want %v", acquireErr, ErrRPCResultFlightCapacity)
|
||||||
t.Fatalf("completed result before raw Close return = %+v err=%v", completed, acquireErr)
|
|
||||||
}
|
}
|
||||||
close(tr.release)
|
close(tr.release)
|
||||||
if c.transportLease != nil {
|
if c.transportLease != nil {
|
||||||
|
|
|
||||||
|
|
@ -162,7 +162,7 @@ func TestOutboundActorRetargetRequiresSecondBodyReservation(t *testing.T) {
|
||||||
state := newOutboundState(budget)
|
state := newOutboundState(budget)
|
||||||
var terminalErr error
|
var terminalErr error
|
||||||
var terminalBytes int64
|
var terminalBytes int64
|
||||||
c.handleOutboundSend(state, outboundOp{
|
op := outboundOp{
|
||||||
kind: outboundSend,
|
kind: outboundSend,
|
||||||
ctx: context.Background(),
|
ctx: context.Background(),
|
||||||
msgType: proto.MessageServerResponse,
|
msgType: proto.MessageServerResponse,
|
||||||
|
|
@ -174,12 +174,14 @@ func TestOutboundActorRetargetRequiresSecondBodyReservation(t *testing.T) {
|
||||||
terminalErr = err
|
terminalErr = err
|
||||||
terminalBytes = budget.snapshot()
|
terminalBytes = budget.snapshot()
|
||||||
},
|
},
|
||||||
})
|
}
|
||||||
|
err := c.handleOutboundSend(state, op)
|
||||||
|
op.finish(outboundResult{err: err})
|
||||||
if !errors.Is(terminalErr, ErrOutboundTrackedBudget) {
|
if !errors.Is(terminalErr, ErrOutboundTrackedBudget) {
|
||||||
t.Fatalf("retarget terminal error = %v, want %v", terminalErr, ErrOutboundTrackedBudget)
|
t.Fatalf("retarget terminal error = %v, want %v", terminalErr, ErrOutboundTrackedBudget)
|
||||||
}
|
}
|
||||||
if terminalBytes != int64(len(encoded.body)) {
|
if terminalBytes != 0 {
|
||||||
t.Fatalf("bytes visible to terminal = %d, want original body %d retained", terminalBytes, len(encoded.body))
|
t.Fatalf("bytes visible to terminal = %d, want producer reservation released", terminalBytes)
|
||||||
}
|
}
|
||||||
if got := tr.sends.Load(); got != 0 {
|
if got := tr.sends.Load(); got != 0 {
|
||||||
t.Fatalf("retarget under one-body budget wrote %d frames, want 0", got)
|
t.Fatalf("retarget under one-body budget wrote %d frames, want 0", got)
|
||||||
|
|
@ -213,7 +215,7 @@ func TestOutboundActorRetargetTransfersOnlyReplacementToPending(t *testing.T) {
|
||||||
state := newOutboundState(budget)
|
state := newOutboundState(budget)
|
||||||
var terminalErr error
|
var terminalErr error
|
||||||
var terminalBytes int64
|
var terminalBytes int64
|
||||||
c.handleOutboundSend(state, outboundOp{
|
op := outboundOp{
|
||||||
kind: outboundSend,
|
kind: outboundSend,
|
||||||
ctx: context.Background(),
|
ctx: context.Background(),
|
||||||
msgType: proto.MessageServerResponse,
|
msgType: proto.MessageServerResponse,
|
||||||
|
|
@ -225,12 +227,14 @@ func TestOutboundActorRetargetTransfersOnlyReplacementToPending(t *testing.T) {
|
||||||
terminalErr = err
|
terminalErr = err
|
||||||
terminalBytes = budget.snapshot()
|
terminalBytes = budget.snapshot()
|
||||||
},
|
},
|
||||||
})
|
}
|
||||||
|
err := c.handleOutboundSend(state, op)
|
||||||
|
op.finish(outboundResult{err: err})
|
||||||
if terminalErr != nil {
|
if terminalErr != nil {
|
||||||
t.Fatalf("retarget terminal error: %v", terminalErr)
|
t.Fatalf("retarget terminal error: %v", terminalErr)
|
||||||
}
|
}
|
||||||
if terminalBytes != int64(2*perBody) {
|
if terminalBytes != int64(perBody) {
|
||||||
t.Fatalf("bytes visible to terminal = %d, want original+replacement %d", terminalBytes, 2*perBody)
|
t.Fatalf("bytes visible to terminal = %d, want only pending replacement %d", terminalBytes, perBody)
|
||||||
}
|
}
|
||||||
if got := budget.snapshot(); got != int64(perBody) {
|
if got := budget.snapshot(); got != int64(perBody) {
|
||||||
t.Fatalf("bytes after terminal = %d, want one pending replacement %d", got, perBody)
|
t.Fatalf("bytes after terminal = %d, want one pending replacement %d", got, perBody)
|
||||||
|
|
|
||||||
|
|
@ -510,8 +510,10 @@ const (
|
||||||
rpcRewrapObserverWorkers = 1
|
rpcRewrapObserverWorkers = 1
|
||||||
rpcRewrapObserverQueue = 64
|
rpcRewrapObserverQueue = 64
|
||||||
// Queue residence, physical delivery and ordered restore share one absolute
|
// Queue residence, physical delivery and ordered restore share one absolute
|
||||||
// deadline. An admitted alias must never retain a Conn scheduler barrier for
|
// control deadline. It prevents later stages from starting and fences logical
|
||||||
// minutes behind older slow jobs.
|
// ownership, but cannot cancel non-cooperative filesystem, transport or restore
|
||||||
|
// work. Failure publication may wait for replay preparation to leave its
|
||||||
|
// ownership transition before releasing the Conn scheduler barrier.
|
||||||
rpcRewrapDeliveryQueueTimeout = 5 * time.Second
|
rpcRewrapDeliveryQueueTimeout = 5 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -532,10 +534,10 @@ const (
|
||||||
rpcRewrapJobFailed
|
rpcRewrapJobFailed
|
||||||
)
|
)
|
||||||
|
|
||||||
// rpcRewrapDeliveryControl lets an independent deadline timer retire queued,
|
// rpcRewrapDeliveryControl lets the deadline timer independently transition
|
||||||
// running and physically committed jobs. A late worker cannot enter run after
|
// queued, running and physically committed jobs to Failed. A late worker cannot
|
||||||
// the timer wins; a committed non-cooperative restore is fenced by fail, and
|
// enter run after the timer wins. The timer cannot cancel a physical transport
|
||||||
// its eventual return cannot report failure or finish the barrier a second time.
|
// write; a late return still cannot report failure or finish the barrier twice.
|
||||||
type rpcRewrapDeliveryControl struct {
|
type rpcRewrapDeliveryControl struct {
|
||||||
state atomic.Uint32
|
state atomic.Uint32
|
||||||
timerMu sync.Mutex
|
timerMu sync.Mutex
|
||||||
|
|
@ -550,9 +552,9 @@ type rpcRewrapPhysicalOutcome struct {
|
||||||
// waitRPCRewrapPhysicalTerminal deliberately keeps one of the four bounded
|
// waitRPCRewrapPhysicalTerminal deliberately keeps one of the four bounded
|
||||||
// workers attached to an in-progress actor write even after the watchdog fences
|
// workers attached to an in-progress actor write even after the watchdog fences
|
||||||
// the Conn. A broken transport may therefore strand at most four workers, while
|
// the Conn. A broken transport may therefore strand at most four workers, while
|
||||||
// queued jobs still time out independently. If that transport later reports
|
// queued jobs still transition to Failed at their control deadlines. If that
|
||||||
// success, the worker cannot lose the logical hook merely because timeout won
|
// transport later reports success, the worker cannot lose the logical hook
|
||||||
// before its goroutine resumed.
|
// merely because timeout won before its goroutine resumed.
|
||||||
func waitRPCRewrapPhysicalTerminal(
|
func waitRPCRewrapPhysicalTerminal(
|
||||||
c *Conn,
|
c *Conn,
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
|
|
@ -700,12 +702,6 @@ func runRPCRewrapDeliveryJob(j rpcRewrapDeliveryJob) {
|
||||||
if !control.transition(rpcRewrapJobPending, rpcRewrapJobRunning) {
|
if !control.transition(rpcRewrapJobPending, rpcRewrapJobRunning) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if !j.deadline.IsZero() && !time.Now().Before(j.deadline) {
|
|
||||||
if control.fail() {
|
|
||||||
j.reportFailure(context.DeadlineExceeded)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer func() {
|
defer func() {
|
||||||
if recovered := recover(); recovered != nil {
|
if recovered := recover(); recovered != nil {
|
||||||
if control.fail() {
|
if control.fail() {
|
||||||
|
|
@ -716,6 +712,12 @@ func runRPCRewrapDeliveryJob(j rpcRewrapDeliveryJob) {
|
||||||
}
|
}
|
||||||
control.complete()
|
control.complete()
|
||||||
}()
|
}()
|
||||||
|
if !j.deadline.IsZero() && !time.Now().Before(j.deadline) {
|
||||||
|
if control.fail() {
|
||||||
|
j.reportFailure(context.DeadlineExceeded)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
j.run(control, j.deadline)
|
j.run(control, j.deadline)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -747,9 +749,10 @@ func scheduleRPCRewrapJob(
|
||||||
delay = 0
|
delay = 0
|
||||||
}
|
}
|
||||||
timer := time.AfterFunc(delay, func() {
|
timer := time.AfterFunc(delay, func() {
|
||||||
if job.control.timeout() {
|
if !job.control.timeout() {
|
||||||
job.reportFailure(context.DeadlineExceeded)
|
return
|
||||||
}
|
}
|
||||||
|
job.reportFailure(context.DeadlineExceeded)
|
||||||
})
|
})
|
||||||
job.control.installTimer(timer)
|
job.control.installTimer(timer)
|
||||||
select {
|
select {
|
||||||
|
|
@ -766,6 +769,22 @@ func scheduleRPCRewrapJob(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Server) attachRPCRewrapReplayPreparation(
|
||||||
|
job *rpcRewrapDeliveryJob,
|
||||||
|
c *Conn,
|
||||||
|
reqMsgID int64,
|
||||||
|
method string,
|
||||||
|
encoded *encodedOutboundMessage,
|
||||||
|
) {
|
||||||
|
if job == nil || s == nil || c == nil || encoded == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Freeze scheduling metadata before the watchdog can publish the compact
|
||||||
|
// receipt. Exact wire bytes are owned only by the logical-session outbox, so
|
||||||
|
// there is no pre-send spool preparation or I/O gate.
|
||||||
|
encoded.priority = rpcResultPriority(method, encoded)
|
||||||
|
}
|
||||||
|
|
||||||
func scheduleRPCRewrapDeliveryJob(job rpcRewrapDeliveryJob) bool {
|
func scheduleRPCRewrapDeliveryJob(job rpcRewrapDeliveryJob) bool {
|
||||||
return scheduleRPCRewrapJob(job, &rpcRewrapDeliveryOnce, &rpcRewrapDeliveryJobs,
|
return scheduleRPCRewrapJob(job, &rpcRewrapDeliveryOnce, &rpcRewrapDeliveryJobs,
|
||||||
rpcRewrapDeliveryWorkers, rpcRewrapDeliveryQueue)
|
rpcRewrapDeliveryWorkers, rpcRewrapDeliveryQueue)
|
||||||
|
|
@ -953,6 +972,7 @@ func (a *rpcRewrapAlias) activate(s *Server) error {
|
||||||
s.log.Debug("Retargeted RPC restore failed", zap.Error(restoreErr))
|
s.log.Debug("Retargeted RPC restore failed", zap.Error(restoreErr))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
s.attachRPCRewrapReplayPreparation(&job, a.conn, a.newReqID, a.method, clone)
|
||||||
job.fail = func(err error) {
|
job.fail = func(err error) {
|
||||||
// Never enter deliveredFinalizeOnce from the timer goroutine: the worker
|
// Never enter deliveredFinalizeOnce from the timer goroutine: the worker
|
||||||
// may already own a non-cooperative restore. Fence and release its Conn
|
// may already own a non-cooperative restore. Fence and release its Conn
|
||||||
|
|
@ -978,6 +998,7 @@ func (a *rpcRewrapAlias) activate(s *Server) error {
|
||||||
job := s.rpcRewrapRestoreJob(a, "pending init rewrap result", func(control *rpcRewrapDeliveryControl, deadline time.Time) {
|
job := s.rpcRewrapRestoreJob(a, "pending init rewrap result", func(control *rpcRewrapDeliveryControl, deadline time.Time) {
|
||||||
s.publishRewrappedRPCResult(a.conn, a.newReqID, a.method, a.newOwner, clone, a, control, deadline)
|
s.publishRewrappedRPCResult(a.conn, a.newReqID, a.method, a.newOwner, clone, a, control, deadline)
|
||||||
})
|
})
|
||||||
|
s.attachRPCRewrapReplayPreparation(&job, a.conn, a.newReqID, a.method, clone)
|
||||||
// Once this alias consumed the source candidate, expiration or panic of
|
// Once this alias consumed the source candidate, expiration or panic of
|
||||||
// the admitted worker job must still publish the immutable result under
|
// the admitted worker job must still publish the immutable result under
|
||||||
// the new msg_id. Otherwise the alias owner would remain pending forever
|
// the new msg_id. Otherwise the alias owner would remain pending forever
|
||||||
|
|
@ -1067,14 +1088,15 @@ func (s *Server) publishRewrappedRPCResult(
|
||||||
alias.releaseBodyReservation()
|
alias.releaseBodyReservation()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
priority := rpcResultPriority(method, encoded)
|
|
||||||
encoded.priority = priority
|
|
||||||
encoded.markQueued()
|
|
||||||
if deadline.IsZero() {
|
if deadline.IsZero() {
|
||||||
deadline = time.Now().Add(rpcRewrapDeliveryQueueTimeout)
|
deadline = time.Now().Add(rpcRewrapDeliveryQueueTimeout)
|
||||||
}
|
}
|
||||||
ctx, cancel := context.WithDeadline(context.Background(), deadline)
|
ctx, cancel := context.WithDeadline(context.Background(), deadline)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
// attachRPCRewrapReplayPreparation froze cache-copied scheduling metadata
|
||||||
|
// before the watchdog started. The delivery path must not mutate plain fields
|
||||||
|
// that a concurrent timeout publication can copy into the replay ledger.
|
||||||
|
encoded.markQueued()
|
||||||
// Rewrap delivery is synchronous on this small bounded worker pool. This
|
// Rewrap delivery is synchronous on this small bounded worker pool. This
|
||||||
// makes the queue deadline cover the physical write and lets the pending
|
// makes the queue deadline cover the physical write and lets the pending
|
||||||
// logical hook join the same per-Conn ordered restore, without touching the
|
// logical hook join the same per-Conn ordered restore, without touching the
|
||||||
|
|
@ -1093,10 +1115,10 @@ func (s *Server) publishRewrappedRPCResult(
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// Physical success outranks an already-fired watchdog. The timeout path may
|
// Physical success outranks an already-fired watchdog. The timeout path may
|
||||||
// have fenced and cached a replayable clone, but it cannot revoke bytes; the
|
// have fenced and published a replayable receipt, but it cannot revoke bytes; the
|
||||||
// shared once/coordinator below still completes logical state exactly once.
|
// shared once/coordinator below still completes logical state exactly once.
|
||||||
// Run replacement metadata then the original logical hook before publishing
|
// Run replacement metadata then the original logical hook before publishing
|
||||||
// the alias cache entry. Whole-finalization once also covers a watchdog racing
|
// the alias receipt. Whole-finalization once also covers a watchdog racing
|
||||||
// a late physical terminal, so completed metadata cannot be overwritten.
|
// a late physical terminal, so completed metadata cannot be overwritten.
|
||||||
restoreParent := ctx
|
restoreParent := ctx
|
||||||
if !outcome.owned {
|
if !outcome.owned {
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
package mtprotoedge
|
package mtprotoedge
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
"errors"
|
"errors"
|
||||||
|
|
@ -445,7 +446,7 @@ func TestInitRewrapAfterWritingReplaysWithoutBusinessExecution(t *testing.T) {
|
||||||
if !oldOwner.HandOff() {
|
if !oldOwner.HandOff() {
|
||||||
t.Fatal("old owner handoff failed")
|
t.Fatal("old owner handoff failed")
|
||||||
}
|
}
|
||||||
s.rpcResults.Put(c.authKeyID, c.sessionID, oldReqID, encoded)
|
storeLogicalRPCResultForTest(t, s, c, oldReqID, encoded)
|
||||||
|
|
||||||
deadline := time.Now().Add(2 * time.Second)
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
var replayed *encodedOutboundMessage
|
var replayed *encodedOutboundMessage
|
||||||
|
|
@ -534,7 +535,7 @@ func TestInitRewrapAliasesExecutionAndRetargetsQueuedResult(t *testing.T) {
|
||||||
t.Fatal("old owner handoff failed")
|
t.Fatal("old owner handoff failed")
|
||||||
}
|
}
|
||||||
encoded.markDelivered()
|
encoded.markDelivered()
|
||||||
s.rpcResults.Put(c.authKeyID, c.sessionID, oldReqID, encoded)
|
storeLogicalRPCResultForTest(t, s, c, oldReqID, encoded)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
aliased *encodedOutboundMessage
|
aliased *encodedOutboundMessage
|
||||||
|
|
@ -772,7 +773,7 @@ func TestRetargetedRPCRestoreIsOrderedAndIndependentOfGlobalHookExecutor(t *test
|
||||||
t.Fatalf("retargeted physical req_msg_id = %d, want %d", got, newReqID)
|
t.Fatalf("retargeted physical req_msg_id = %d, want %d", got, newReqID)
|
||||||
}
|
}
|
||||||
encoded.markDelivered()
|
encoded.markDelivered()
|
||||||
s.rpcResults.Put(c.authKeyID, c.sessionID, oldReqID, encoded)
|
storeLogicalRPCResultForTest(t, s, c, oldReqID, encoded)
|
||||||
|
|
||||||
deadline := time.Now().Add(time.Second)
|
deadline := time.Now().Add(time.Second)
|
||||||
for order.Load() != 2 && time.Now().Before(deadline) {
|
for order.Load() != 2 && time.Now().Before(deadline) {
|
||||||
|
|
@ -1108,7 +1109,8 @@ func TestRPCRewrapSubscriberPanicCannotLoseClaimedLogicalHook(t *testing.T) {
|
||||||
if pending != 0 {
|
if pending != 0 {
|
||||||
t.Fatalf("restore barriers after subscriber panic = %d, want 0", pending)
|
t.Fatalf("restore barriers after subscriber panic = %d, want 0", pending)
|
||||||
}
|
}
|
||||||
if cached, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID); !ok || cached != encoded {
|
if cached, ok := s.rpcResults.Get(c.authKeyID, c.sessionID, reqMsgID); !ok ||
|
||||||
t.Fatalf("completed result after subscriber panic = (%p, %v), want (%p, true)", cached, ok, encoded)
|
cached.delivery != encoded.delivery || !bytes.Equal(cached.body, encoded.body) || cached.replayMsgID == 0 {
|
||||||
|
t.Fatalf("logical outbox result after subscriber panic = (%p, %v), source=%p", cached, ok, encoded)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -118,7 +118,7 @@ func TestLayerRPCGetConfigUsesExactAdmittedProfile(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestInboundRPCQueueFullReturnsFloodWait(t *testing.T) {
|
func TestInboundRPCQueueFullReturnsWorkerBusy(t *testing.T) {
|
||||||
const dc = 2
|
const dc = 2
|
||||||
handler := &blockingRPC{
|
handler := &blockingRPC{
|
||||||
started: make(chan struct{}, 1),
|
started: make(chan struct{}, 1),
|
||||||
|
|
@ -152,8 +152,8 @@ func TestInboundRPCQueueFullReturnsFloodWait(t *testing.T) {
|
||||||
if err := rpcErr.Decode(&bin.Buffer{Buf: result.Result}); err != nil {
|
if err := rpcErr.Decode(&bin.Buffer{Buf: result.Result}); err != nil {
|
||||||
t.Fatalf("decode rpc_error: %v", err)
|
t.Fatalf("decode rpc_error: %v", err)
|
||||||
}
|
}
|
||||||
if rpcErr.ErrorCode != 420 || rpcErr.ErrorMessage != "FLOOD_WAIT_1" {
|
if rpcErr.ErrorCode != rpcWorkerBusyErrorCode || rpcErr.ErrorMessage != rpcWorkerBusyErrorMessage {
|
||||||
t.Fatalf("rpc_error = %d %q, want 420 FLOOD_WAIT_1", rpcErr.ErrorCode, rpcErr.ErrorMessage)
|
t.Fatalf("rpc_error = %d %q, want %d %s", rpcErr.ErrorCode, rpcErr.ErrorMessage, rpcWorkerBusyErrorCode, rpcWorkerBusyErrorMessage)
|
||||||
}
|
}
|
||||||
close(handler.release)
|
close(handler.release)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -284,16 +284,13 @@ type Options struct {
|
||||||
// 等于 copied body;exact charge 是 typed decode 前的保守 materialization
|
// 等于 copied body;exact charge 是 typed decode 前的保守 materialization
|
||||||
// 上界,因此该配置不表示可并发接收 512 MiB wire body。默认 512 MiB。
|
// 上界,因此该配置不表示可并发接收 512 MiB wire body。默认 512 MiB。
|
||||||
RPCGlobalMaxBytes int64
|
RPCGlobalMaxBytes int64
|
||||||
// RPCResultCache* limits bound pending ownership and completed rpc_result
|
// RPCResultCache*Entries bound in-flight owners and compact completed
|
||||||
// replay state across the full 331-second duplicate horizon. Every owner is
|
// receipts. Exact payload bytes are not charged here: the logical-session
|
||||||
// charged simultaneously at global, raw-auth and session scopes. Defaults:
|
// outbox owns them under OutboundTrackedGlobalMaxBytes until ACK. ACK removes
|
||||||
// global 262144/64 MiB, auth 32768/32 MiB, session 16384/16 MiB.
|
// the receipt immediately; 331 seconds is only the no-ACK safety horizon.
|
||||||
RPCResultCacheMaxEntries int
|
RPCResultCacheMaxEntries int
|
||||||
RPCResultCacheMaxBytes int64
|
|
||||||
RPCResultCacheAuthMaxEntries int
|
RPCResultCacheAuthMaxEntries int
|
||||||
RPCResultCacheAuthMaxBytes int64
|
|
||||||
RPCResultCacheSessionMaxEntries int
|
RPCResultCacheSessionMaxEntries int
|
||||||
RPCResultCacheSessionMaxBytes int64
|
|
||||||
// RPCResultPendingPerAuth is an additional active-owner bound, independent
|
// RPCResultPendingPerAuth is an additional active-owner bound, independent
|
||||||
// from the retained entry limits and RPCGlobalMaxTasks. Default 2048.
|
// from the retained entry limits and RPCGlobalMaxTasks. Default 2048.
|
||||||
RPCResultPendingPerAuth int
|
RPCResultPendingPerAuth int
|
||||||
|
|
@ -395,21 +392,12 @@ func (o *Options) setDefaults() {
|
||||||
if o.RPCResultCacheMaxEntries == 0 {
|
if o.RPCResultCacheMaxEntries == 0 {
|
||||||
o.RPCResultCacheMaxEntries = rpcResultCacheMaxEntries
|
o.RPCResultCacheMaxEntries = rpcResultCacheMaxEntries
|
||||||
}
|
}
|
||||||
if o.RPCResultCacheMaxBytes == 0 {
|
|
||||||
o.RPCResultCacheMaxBytes = rpcResultCacheMaxBytes
|
|
||||||
}
|
|
||||||
if o.RPCResultCacheAuthMaxEntries == 0 {
|
if o.RPCResultCacheAuthMaxEntries == 0 {
|
||||||
o.RPCResultCacheAuthMaxEntries = rpcResultCacheAuthMaxEntries
|
o.RPCResultCacheAuthMaxEntries = rpcResultCacheAuthMaxEntries
|
||||||
}
|
}
|
||||||
if o.RPCResultCacheAuthMaxBytes == 0 {
|
|
||||||
o.RPCResultCacheAuthMaxBytes = rpcResultCacheAuthMaxBytes
|
|
||||||
}
|
|
||||||
if o.RPCResultCacheSessionMaxEntries == 0 {
|
if o.RPCResultCacheSessionMaxEntries == 0 {
|
||||||
o.RPCResultCacheSessionMaxEntries = rpcResultCacheSessionMaxEntries
|
o.RPCResultCacheSessionMaxEntries = rpcResultCacheSessionMaxEntries
|
||||||
}
|
}
|
||||||
if o.RPCResultCacheSessionMaxBytes == 0 {
|
|
||||||
o.RPCResultCacheSessionMaxBytes = rpcResultCacheSessionMaxBytes
|
|
||||||
}
|
|
||||||
if o.RPCResultPendingPerAuth == 0 {
|
if o.RPCResultPendingPerAuth == 0 {
|
||||||
o.RPCResultPendingPerAuth = rpcResultFlightMaxPendingPerAuth
|
o.RPCResultPendingPerAuth = rpcResultFlightMaxPendingPerAuth
|
||||||
if o.RPCResultPendingPerAuth > o.RPCGlobalMaxTasks {
|
if o.RPCResultPendingPerAuth > o.RPCGlobalMaxTasks {
|
||||||
|
|
@ -457,17 +445,6 @@ func validateRPCResultCacheOptions(o Options) error {
|
||||||
return fmt.Errorf("rpc_result cache entry hierarchy must satisfy global >= auth >= session: %d/%d/%d",
|
return fmt.Errorf("rpc_result cache entry hierarchy must satisfy global >= auth >= session: %d/%d/%d",
|
||||||
o.RPCResultCacheMaxEntries, o.RPCResultCacheAuthMaxEntries, o.RPCResultCacheSessionMaxEntries)
|
o.RPCResultCacheMaxEntries, o.RPCResultCacheAuthMaxEntries, o.RPCResultCacheSessionMaxEntries)
|
||||||
}
|
}
|
||||||
if o.RPCResultCacheMaxBytes < int64(maxOutboundBodyBytes) ||
|
|
||||||
o.RPCResultCacheAuthMaxBytes < int64(maxOutboundBodyBytes) ||
|
|
||||||
o.RPCResultCacheSessionMaxBytes < int64(maxOutboundBodyBytes) {
|
|
||||||
return fmt.Errorf("rpc_result cache byte limits must each be at least max outbound body %d: %d/%d/%d",
|
|
||||||
maxOutboundBodyBytes, o.RPCResultCacheMaxBytes, o.RPCResultCacheAuthMaxBytes, o.RPCResultCacheSessionMaxBytes)
|
|
||||||
}
|
|
||||||
if o.RPCResultCacheMaxBytes < o.RPCResultCacheAuthMaxBytes ||
|
|
||||||
o.RPCResultCacheAuthMaxBytes < o.RPCResultCacheSessionMaxBytes {
|
|
||||||
return fmt.Errorf("rpc_result cache byte hierarchy must satisfy global >= auth >= session: %d/%d/%d",
|
|
||||||
o.RPCResultCacheMaxBytes, o.RPCResultCacheAuthMaxBytes, o.RPCResultCacheSessionMaxBytes)
|
|
||||||
}
|
|
||||||
if o.RPCResultPendingPerAuth <= 0 || o.RPCResultPendingPerAuth > o.RPCGlobalMaxTasks ||
|
if o.RPCResultPendingPerAuth <= 0 || o.RPCResultPendingPerAuth > o.RPCGlobalMaxTasks ||
|
||||||
o.RPCResultPendingPerAuth > o.RPCResultCacheAuthMaxEntries {
|
o.RPCResultPendingPerAuth > o.RPCResultCacheAuthMaxEntries {
|
||||||
return fmt.Errorf("rpc_result per-auth pending limit %d must be positive and <= global pending %d and auth entries %d",
|
return fmt.Errorf("rpc_result per-auth pending limit %d must be positive and <= global pending %d and auth entries %d",
|
||||||
|
|
@ -534,7 +511,7 @@ func New(opts Options) *Server {
|
||||||
if conns == nil {
|
if conns == nil {
|
||||||
conns = NewSessionManager(opts.Logger.Named("sessions"))
|
conns = NewSessionManager(opts.Logger.Named("sessions"))
|
||||||
}
|
}
|
||||||
return &Server{
|
server := &Server{
|
||||||
log: opts.Logger,
|
log: opts.Logger,
|
||||||
codec: opts.Codec,
|
codec: opts.Codec,
|
||||||
obfuscated: opts.ObfuscatedTCP,
|
obfuscated: opts.ObfuscatedTCP,
|
||||||
|
|
@ -570,16 +547,21 @@ func New(opts Options) *Server {
|
||||||
rpcResults: newRPCResultCacheWithFairCapacity(opts.Clock.Now, rpcResultCacheCapacity{
|
rpcResults: newRPCResultCacheWithFairCapacity(opts.Clock.Now, rpcResultCacheCapacity{
|
||||||
maxPending: opts.RPCGlobalMaxTasks,
|
maxPending: opts.RPCGlobalMaxTasks,
|
||||||
maxPendingPerAuth: opts.RPCResultPendingPerAuth,
|
maxPendingPerAuth: opts.RPCResultPendingPerAuth,
|
||||||
globalMaxBytes: opts.RPCResultCacheMaxBytes,
|
globalMaxBytes: int64(opts.RPCResultCacheMaxEntries),
|
||||||
globalMaxEntries: opts.RPCResultCacheMaxEntries,
|
globalMaxEntries: opts.RPCResultCacheMaxEntries,
|
||||||
authMaxBytes: opts.RPCResultCacheAuthMaxBytes,
|
authMaxBytes: int64(opts.RPCResultCacheAuthMaxEntries),
|
||||||
authMaxEntries: opts.RPCResultCacheAuthMaxEntries,
|
authMaxEntries: opts.RPCResultCacheAuthMaxEntries,
|
||||||
sessionMaxBytes: opts.RPCResultCacheSessionMaxBytes,
|
sessionMaxBytes: int64(opts.RPCResultCacheSessionMaxEntries),
|
||||||
sessionMaxEntries: opts.RPCResultCacheSessionMaxEntries,
|
sessionMaxEntries: opts.RPCResultCacheSessionMaxEntries,
|
||||||
|
sessions: conns,
|
||||||
}),
|
}),
|
||||||
rpcRewrap: newRPCRewrapRegistry(opts.RPCGlobalMaxTasks),
|
rpcRewrap: newRPCRewrapRegistry(opts.RPCGlobalMaxTasks),
|
||||||
admission: newAdmissionController(opts.MaxConnections, opts.MaxConnectionsPerIP, opts.MaxConcurrentHandshakes),
|
admission: newAdmissionController(opts.MaxConnections, opts.MaxConnectionsPerIP, opts.MaxConcurrentHandshakes),
|
||||||
}
|
}
|
||||||
|
conns.setLogicalSessionReleaseHook(func(key sessionKey) {
|
||||||
|
server.rpcResults.forgetCompletedSession(key.authKeyID, key.sessionID)
|
||||||
|
})
|
||||||
|
return server
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListenAndServe binds the public MTProto socket and immediately enters Serve.
|
// ListenAndServe binds the public MTProto socket and immediately enters Serve.
|
||||||
|
|
@ -641,8 +623,16 @@ func (s *Server) buildConn(tc transport.Conn, lease *physicalTransportLease, key
|
||||||
outboundTrackedBudget: s.outboundTrackedBudget,
|
outboundTrackedBudget: s.outboundTrackedBudget,
|
||||||
outboundControlTrackedBudget: s.outboundControlBudget,
|
outboundControlTrackedBudget: s.outboundControlBudget,
|
||||||
outboundScratchPool: s.outboundScratchPool,
|
outboundScratchPool: s.outboundScratchPool,
|
||||||
rpcResultAcked: s.rpcRewrap.acknowledge,
|
rpcResultAcked: func(conn *Conn, reqMsgID int64) {
|
||||||
|
// The sole outbound actor invokes this only after resolving a client
|
||||||
|
// msgs_ack server msg_id through its tracked resend frame. The actor has
|
||||||
|
// already removed the sole outbox frame; now delete its receipt and
|
||||||
|
// retire any init-rewrap bookkeeping.
|
||||||
|
s.rpcResults.Acknowledge(conn.authKeyID, conn.sessionID, reqMsgID)
|
||||||
|
s.rpcRewrap.acknowledge(conn, reqMsgID)
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
s.conns.attachLogicalSession(c, s.outboundTrackedBudget)
|
||||||
c.startOutbound()
|
c.startOutbound()
|
||||||
c.startInboundRPCScheduler(s.rpcScheduler, s.rpcInflight, s.rpcQueueSize, s.rpcTimeout)
|
c.startInboundRPCScheduler(s.rpcScheduler, s.rpcInflight, s.rpcQueueSize, s.rpcTimeout)
|
||||||
return c
|
return c
|
||||||
|
|
@ -655,6 +645,7 @@ func (s *Server) Serve(ctx context.Context, ln net.Listener) error {
|
||||||
// serveTCP/serveMixed 返回前会等待连接 goroutine 收敛,各 Conn 已先排空/取消任务;
|
// serveTCP/serveMixed 返回前会等待连接 goroutine 收敛,各 Conn 已先排空/取消任务;
|
||||||
// 最后再停止全局池,避免关闭过程中留下无人消费但仍占预算的队列。
|
// 最后再停止全局池,避免关闭过程中留下无人消费但仍占预算的队列。
|
||||||
s.rpcScheduler.start()
|
s.rpcScheduler.start()
|
||||||
|
defer s.conns.releaseAllLogicalSessions()
|
||||||
defer s.rpcScheduler.stop(rpcCloseWaitTimeout)
|
defer s.rpcScheduler.stop(rpcCloseWaitTimeout)
|
||||||
// 只在最外层 listener 包一次,确保 same-port mux 的 sniff/HTTP upgrade 也计入
|
// 只在最外层 listener 包一次,确保 same-port mux 的 sniff/HTTP upgrade 也计入
|
||||||
// raw admission,而不是等连接已经分流后才计数。
|
// raw admission,而不是等连接已经分流后才计数。
|
||||||
|
|
|
||||||
|
|
@ -156,7 +156,7 @@ func TestContainerRPCAdmissionFailureIsAtomic(t *testing.T) {
|
||||||
if err := rpcErr.Decode(&bin.Buffer{Buf: result.Result}); err != nil {
|
if err := rpcErr.Decode(&bin.Buffer{Buf: result.Result}); err != nil {
|
||||||
t.Fatalf("decode capacity rpc_error: %v", err)
|
t.Fatalf("decode capacity rpc_error: %v", err)
|
||||||
}
|
}
|
||||||
if rpcErr.ErrorCode != 420 || rpcErr.ErrorMessage != "FLOOD_WAIT_1" {
|
if rpcErr.ErrorCode != rpcWorkerBusyErrorCode || rpcErr.ErrorMessage != rpcWorkerBusyErrorMessage {
|
||||||
t.Fatalf("capacity rpc_error = %+v", rpcErr)
|
t.Fatalf("capacity rpc_error = %+v", rpcErr)
|
||||||
}
|
}
|
||||||
delete(requestIDs, result.RequestMessageID)
|
delete(requestIDs, result.RequestMessageID)
|
||||||
|
|
|
||||||
|
|
@ -171,6 +171,10 @@ func notifySessionDestroyed(observer SessionLifecycleObserver, authKeyID [8]byte
|
||||||
type SessionManager struct {
|
type SessionManager struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
bySession map[sessionKey]*Conn
|
bySession map[sessionKey]*Conn
|
||||||
|
// logicalSessions owns MTProto resend state independently of physical Conn
|
||||||
|
// generations. It is bounded by the Server-wide tracked-body budget and a
|
||||||
|
// short offline retention window; ACK and destroy release bodies immediately.
|
||||||
|
logicalSessions map[sessionKey]*logicalSession
|
||||||
// claims owns the provisional -> active gap. A claimant is intentionally
|
// claims owns the provisional -> active gap. A claimant is intentionally
|
||||||
// absent from every push/online index until its required session control frame
|
// absent from every push/online index until its required session control frame
|
||||||
// is on the wire and PublishActivation validates the same owner.
|
// is on the wire and PublishActivation validates the same owner.
|
||||||
|
|
@ -188,6 +192,7 @@ type SessionManager struct {
|
||||||
pending map[sessionKey][]queuedPush // updates-ready 前暂存的主动推送
|
pending map[sessionKey][]queuedPush // updates-ready 前暂存的主动推送
|
||||||
flushing map[sessionKey]bool // 置位时暂存正在排空的 session;排空完成前推送继续进 pending 保序
|
flushing map[sessionKey]bool // 置位时暂存正在排空的 session;排空完成前推送继续进 pending 保序
|
||||||
pendingBudget *outboundTrackedBudget // 未就绪 session 暂存 encoded body 的进程级上限
|
pendingBudget *outboundTrackedBudget // 未就绪 session 暂存 encoded body 的进程级上限
|
||||||
|
logicalSessionReleased func(sessionKey)
|
||||||
|
|
||||||
lifecycle SessionLifecycleObserver
|
lifecycle SessionLifecycleObserver
|
||||||
log *zap.Logger
|
log *zap.Logger
|
||||||
|
|
@ -200,6 +205,7 @@ func NewSessionManager(log *zap.Logger) *SessionManager {
|
||||||
}
|
}
|
||||||
return &SessionManager{
|
return &SessionManager{
|
||||||
bySession: make(map[sessionKey]*Conn),
|
bySession: make(map[sessionKey]*Conn),
|
||||||
|
logicalSessions: make(map[sessionKey]*logicalSession),
|
||||||
claims: make(map[sessionKey]*Conn),
|
claims: make(map[sessionKey]*Conn),
|
||||||
claimsByAuth: make(map[[8]byte]map[int64]*Conn),
|
claimsByAuth: make(map[[8]byte]map[int64]*Conn),
|
||||||
byAuthKey: make(map[[8]byte]map[int64]*Conn),
|
byAuthKey: make(map[[8]byte]map[int64]*Conn),
|
||||||
|
|
@ -225,6 +231,12 @@ func (m *SessionManager) SetLifecycleObserver(observer SessionLifecycleObserver)
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *SessionManager) setLogicalSessionReleaseHook(hook func(sessionKey)) {
|
||||||
|
m.mu.Lock()
|
||||||
|
m.logicalSessionReleased = hook
|
||||||
|
m.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
// SeedInheritedLayerForRawAuthKey supplies an auth-key-wide default to every
|
// SeedInheritedLayerForRawAuthKey supplies an auth-key-wide default to every
|
||||||
// currently unknown active/provisional connection for rawAuthKeyID. Existing
|
// currently unknown active/provisional connection for rawAuthKeyID. Existing
|
||||||
// inherited or explicit state is left untouched; only ordered invokeWithLayer
|
// inherited or explicit state is left untouched; only ordered invokeWithLayer
|
||||||
|
|
@ -634,6 +646,7 @@ func (m *SessionManager) AbortActivation(c *Conn) {
|
||||||
m.deletePendingLocked(key)
|
m.deletePendingLocked(key)
|
||||||
delete(m.flushing, key)
|
delete(m.flushing, key)
|
||||||
}
|
}
|
||||||
|
m.markLogicalSessionOfflineLocked(key, time.Now())
|
||||||
owned = true
|
owned = true
|
||||||
}
|
}
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
@ -677,6 +690,7 @@ func (m *SessionManager) Unregister(c *Conn) {
|
||||||
zap.Int("online", len(m.bySession)),
|
zap.Int("online", len(m.bySession)),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
m.markLogicalSessionOfflineLocked(key, time.Now())
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
if observer != nil {
|
if observer != nil {
|
||||||
observer.SessionOffline(c.authKeyID, c.sessionID, offlineUser, lastForUser)
|
observer.SessionOffline(c.authKeyID, c.sessionID, offlineUser, lastForUser)
|
||||||
|
|
@ -692,6 +706,7 @@ func (m *SessionManager) DestroySessionForAuthKey(authKeyID [8]byte, sessionID i
|
||||||
if !ok {
|
if !ok {
|
||||||
if claim := m.claims[key]; claim != nil {
|
if claim := m.claims[key]; claim != nil {
|
||||||
m.retireClaimLocked(key, claim, true)
|
m.retireClaimLocked(key, claim, true)
|
||||||
|
outbound := m.destroyLogicalSessionLocked(key)
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
if !forceCloseConnBatch([]*Conn{claim}, forceCloseBatchTimeout) {
|
if !forceCloseConnBatch([]*Conn{claim}, forceCloseBatchTimeout) {
|
||||||
m.log.Warn("Claimed session close exceeded shared deadline",
|
m.log.Warn("Claimed session close exceeded shared deadline",
|
||||||
|
|
@ -699,15 +714,23 @@ func (m *SessionManager) DestroySessionForAuthKey(authKeyID [8]byte, sessionID i
|
||||||
zap.Int64("session_id", sessionID),
|
zap.Int64("session_id", sessionID),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
if outbound != nil {
|
||||||
|
m.releaseLogicalSession(key, outbound)
|
||||||
|
}
|
||||||
notifySessionDestroyed(observer, authKeyID, sessionID)
|
notifySessionDestroyed(observer, authKeyID, sessionID)
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
m.deletePendingLocked(key)
|
m.deletePendingLocked(key)
|
||||||
|
outbound := m.destroyLogicalSessionLocked(key)
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
if outbound != nil {
|
||||||
|
m.releaseLogicalSession(key, outbound)
|
||||||
|
}
|
||||||
notifySessionDestroyed(observer, authKeyID, sessionID)
|
notifySessionDestroyed(observer, authKeyID, sessionID)
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
offlineUser := m.retireConnLocked(c, true)
|
offlineUser := m.retireConnLocked(c, true)
|
||||||
|
outbound := m.destroyLogicalSessionLocked(key)
|
||||||
lastForUser := offlineUser != 0 && len(m.byUser[offlineUser]) == 0
|
lastForUser := offlineUser != 0 && len(m.byUser[offlineUser]) == 0
|
||||||
m.log.Debug("Session destroyed",
|
m.log.Debug("Session destroyed",
|
||||||
zap.String("auth_key_id", sessionKeyLog(authKeyID)),
|
zap.String("auth_key_id", sessionKeyLog(authKeyID)),
|
||||||
|
|
@ -721,6 +744,9 @@ func (m *SessionManager) DestroySessionForAuthKey(authKeyID [8]byte, sessionID i
|
||||||
zap.Int64("session_id", sessionID),
|
zap.Int64("session_id", sessionID),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
if outbound != nil {
|
||||||
|
m.releaseLogicalSession(key, outbound)
|
||||||
|
}
|
||||||
if observer != nil && offlineUser != 0 {
|
if observer != nil && offlineUser != 0 {
|
||||||
observer.SessionOffline(authKeyID, sessionID, offlineUser, lastForUser)
|
observer.SessionOffline(authKeyID, sessionID, offlineUser, lastForUser)
|
||||||
}
|
}
|
||||||
|
|
@ -815,6 +841,7 @@ func (m *SessionManager) bindAuthKeyLocked(c *Conn, key sessionKey, authKeyID [8
|
||||||
removeBusinessAuthKeyIndex(m.byBusinessAuthKey, oldAuthKeyID, key)
|
removeBusinessAuthKeyIndex(m.byBusinessAuthKey, oldAuthKeyID, key)
|
||||||
}
|
}
|
||||||
c.SetBusinessAuthKeyID(authKeyID)
|
c.SetBusinessAuthKeyID(authKeyID)
|
||||||
|
m.bindLogicalSessionAuthKeyLocked(key, authKeyID)
|
||||||
addBusinessAuthKeyIndex(m.byBusinessAuthKey, authKeyID, key, c)
|
addBusinessAuthKeyIndex(m.byBusinessAuthKey, authKeyID, key, c)
|
||||||
if changed {
|
if changed {
|
||||||
if oldUserID != 0 {
|
if oldUserID != 0 {
|
||||||
|
|
@ -865,6 +892,7 @@ func (m *SessionManager) CloseSessionsForBusinessAuthKey(authKeyID [8]byte) int
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
var conns []*Conn
|
var conns []*Conn
|
||||||
var events []offlineEvent
|
var events []offlineEvent
|
||||||
|
var logicalRelease []*logicalSession
|
||||||
for key, c := range m.businessAuthKeyCandidatesLocked(authKeyID) {
|
for key, c := range m.businessAuthKeyCandidatesLocked(authKeyID) {
|
||||||
if !connUsesBusinessAuthKey(c, authKeyID) {
|
if !connUsesBusinessAuthKey(c, authKeyID) {
|
||||||
continue
|
continue
|
||||||
|
|
@ -880,6 +908,14 @@ func (m *SessionManager) CloseSessionsForBusinessAuthKey(authKeyID [8]byte) int
|
||||||
m.retireClaimLocked(key, c, true)
|
m.retireClaimLocked(key, c, true)
|
||||||
conns = append(conns, c)
|
conns = append(conns, c)
|
||||||
}
|
}
|
||||||
|
for key, logical := range m.logicalSessions {
|
||||||
|
if logical == nil || (key.authKeyID != authKeyID &&
|
||||||
|
(!logical.businessAuthResolved || logical.businessAuthKeyID != authKeyID)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
delete(m.logicalSessions, key)
|
||||||
|
logicalRelease = append(logicalRelease, logical)
|
||||||
|
}
|
||||||
observer := m.lifecycle
|
observer := m.lifecycle
|
||||||
if len(conns) > 0 {
|
if len(conns) > 0 {
|
||||||
m.log.Debug("Force close sessions for revoked auth key",
|
m.log.Debug("Force close sessions for revoked auth key",
|
||||||
|
|
@ -894,6 +930,9 @@ func (m *SessionManager) CloseSessionsForBusinessAuthKey(authKeyID [8]byte) int
|
||||||
zap.Int("sessions", len(conns)),
|
zap.Int("sessions", len(conns)),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
for _, logical := range logicalRelease {
|
||||||
|
m.releaseLogicalSession(logical.key, logical.outbound)
|
||||||
|
}
|
||||||
if observer != nil {
|
if observer != nil {
|
||||||
for _, e := range events {
|
for _, e := range events {
|
||||||
observer.SessionOffline(e.key.authKeyID, e.key.sessionID, e.userID, e.last)
|
observer.SessionOffline(e.key.authKeyID, e.key.sessionID, e.userID, e.last)
|
||||||
|
|
@ -2567,6 +2606,7 @@ func (m *SessionManager) RunPendingSweeper(ctx context.Context, interval time.Du
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
}
|
}
|
||||||
m.sweepStalePending()
|
m.sweepStalePending()
|
||||||
|
m.sweepLogicalSessions(time.Now())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue