feat(loadtest): sync add real 500-session capacity harness
This commit is contained in:
parent
ac0566f779
commit
141f2f20c4
39 changed files with 4157 additions and 42 deletions
|
|
@ -76,9 +76,13 @@ func TestTelegramClientEndToEnd(t *testing.T) {
|
|||
if cfg.ThisDC != dc {
|
||||
t.Errorf("config.ThisDC = %d, want %d", cfg.ThisDC, dc)
|
||||
}
|
||||
// 不下发 DCOptions:客户端使用自己的 DCList / 写死 static 地址。
|
||||
if len(cfg.DCOptions) != 0 {
|
||||
t.Errorf("config.DCOptions = %+v, want empty", cfg.DCOptions)
|
||||
if len(cfg.DCOptions) != 1 {
|
||||
t.Errorf("config.DCOptions = %+v, want one reconnect route", cfg.DCOptions)
|
||||
} else {
|
||||
option := cfg.DCOptions[0]
|
||||
if option.ID != dc || option.IPAddress != tcpAddr.IP.String() || option.Port != tcpAddr.Port {
|
||||
t.Errorf("config.DCOptions[0] = %+v, want dc=%d at %s", option, dc, tcpAddr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ package mtprotoedge
|
|||
|
||||
import "time"
|
||||
|
||||
// Metrics 接收连接层运行指标。实现可对接 Prometheus 等监控系统;
|
||||
// 默认 NopMetrics(零开销)。第一阶段仅预留钩子,正式指标后续接入。
|
||||
// Metrics 接收连接层运行指标。生产入口接入有界 Prometheus exporter;
|
||||
// 其它 embedder 可继续使用 NopMetrics(零开销)。
|
||||
type Metrics interface {
|
||||
// ConnOpened 在接受一个连接时调用。
|
||||
ConnOpened()
|
||||
|
|
@ -39,6 +39,14 @@ type RPCResultMetrics interface {
|
|||
RPCResultDelivered(method string, egressLatency time.Duration, wireBytes int, err error)
|
||||
}
|
||||
|
||||
// LogicalOutboxMetrics observes the sole owner of unacknowledged server frames.
|
||||
// It is intentionally optional: embedders can keep the small Metrics surface,
|
||||
// while production capacity tests can distinguish physical delivery from the
|
||||
// later client ACK that actually releases retained bytes.
|
||||
type LogicalOutboxMetrics interface {
|
||||
LogicalOutboxAcknowledged(bytes int, retainedFor time.Duration, rpcResult bool)
|
||||
}
|
||||
|
||||
// ConnectionIntakeMetrics is an optional extension for the pre-session
|
||||
// connection pipeline. stage is one of raw_accept, mux_sniff, mux_delivery,
|
||||
// transport_dispatch, transport_promote, or first_frame; outcome is a bounded
|
||||
|
|
|
|||
|
|
@ -1834,13 +1834,13 @@ func (c *Conn) handleOutboundOp(state *outboundState, op outboundOp) {
|
|||
state.mu.Lock()
|
||||
var (
|
||||
result outboundResult
|
||||
acked []int64
|
||||
acked []outboundAcknowledgement
|
||||
)
|
||||
switch op.kind {
|
||||
case outboundSend:
|
||||
result.err = c.handleOutboundSend(state, op)
|
||||
case outboundAck:
|
||||
acked = state.ack(op.ids)
|
||||
acked = state.ackWithDetails(op.ids)
|
||||
case outboundQueryState:
|
||||
result.info = state.stateInfo(op.ids)
|
||||
case outboundResend:
|
||||
|
|
@ -1851,9 +1851,16 @@ func (c *Conn) handleOutboundOp(state *outboundState, op outboundOp) {
|
|||
result.err = fmt.Errorf("unknown outbound op %d", op.kind)
|
||||
}
|
||||
state.mu.Unlock()
|
||||
for _, reqMsgID := range acked {
|
||||
if c.rpcResultAcked != nil {
|
||||
c.rpcResultAcked(c, reqMsgID)
|
||||
for _, ack := range acked {
|
||||
if metrics, ok := c.metrics.(LogicalOutboxMetrics); ok {
|
||||
retainedFor := time.Duration(0)
|
||||
if !ack.sentAt.IsZero() {
|
||||
retainedFor = time.Since(ack.sentAt)
|
||||
}
|
||||
metrics.LogicalOutboxAcknowledged(ack.bytes, retainedFor, ack.reqMsgID != 0)
|
||||
}
|
||||
if ack.reqMsgID != 0 && c.rpcResultAcked != nil {
|
||||
c.rpcResultAcked(c, ack.reqMsgID)
|
||||
}
|
||||
}
|
||||
op.finish(result)
|
||||
|
|
@ -2663,25 +2670,45 @@ func (s *outboundState) addReserved(frame *outboundFrame) int {
|
|||
return s.shrinkPending()
|
||||
}
|
||||
|
||||
type outboundAcknowledgement struct {
|
||||
reqMsgID int64
|
||||
bytes int
|
||||
sentAt time.Time
|
||||
}
|
||||
|
||||
func (s *outboundState) ack(ids []int64) []int64 {
|
||||
var requestIDs []int64
|
||||
details := s.ackWithDetails(ids)
|
||||
requestIDs := make([]int64, 0, len(details))
|
||||
for _, detail := range details {
|
||||
if detail.reqMsgID != 0 {
|
||||
requestIDs = append(requestIDs, detail.reqMsgID)
|
||||
}
|
||||
}
|
||||
return requestIDs
|
||||
}
|
||||
|
||||
func (s *outboundState) ackWithDetails(ids []int64) []outboundAcknowledgement {
|
||||
var acknowledged []outboundAcknowledgement
|
||||
for _, id := range ids {
|
||||
frame, ok := s.pending[id]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if frame.reqMsgID != 0 {
|
||||
requestIDs = append(requestIDs, frame.reqMsgID)
|
||||
detail := outboundAcknowledgement{
|
||||
reqMsgID: frame.reqMsgID,
|
||||
bytes: len(frame.body),
|
||||
sentAt: frame.sentAt,
|
||||
}
|
||||
if !s.removePending(id) {
|
||||
continue
|
||||
}
|
||||
s.markAcked(id)
|
||||
acknowledged = append(acknowledged, detail)
|
||||
}
|
||||
if len(s.order) > s.maxMessages*2 {
|
||||
s.compactOrder()
|
||||
}
|
||||
return requestIDs
|
||||
return acknowledged
|
||||
}
|
||||
|
||||
func (s *outboundState) stateInfo(ids []int64) []byte {
|
||||
|
|
|
|||
|
|
@ -29,6 +29,21 @@ type failAfterTransport struct {
|
|||
last []byte
|
||||
}
|
||||
|
||||
type acknowledgementCaptureMetrics struct {
|
||||
NopMetrics
|
||||
count atomic.Int64
|
||||
bytes atomic.Int64
|
||||
retainedNS atomic.Int64
|
||||
rpcResult atomic.Bool
|
||||
}
|
||||
|
||||
func (m *acknowledgementCaptureMetrics) LogicalOutboxAcknowledged(bytes int, retainedFor time.Duration, rpcResult bool) {
|
||||
m.count.Add(1)
|
||||
m.bytes.Add(int64(bytes))
|
||||
m.retainedNS.Store(int64(retainedFor))
|
||||
m.rpcResult.Store(rpcResult)
|
||||
}
|
||||
|
||||
func TestRPCResultReplayAttemptHooksArePhysicalConnectionLocal(t *testing.T) {
|
||||
const reqMsgID = int64(771)
|
||||
base := &encodedOutboundMessage{
|
||||
|
|
@ -814,6 +829,8 @@ func TestOutboundTrackedBudgetAckAndCloseReturnExactly(t *testing.T) {
|
|||
budget := newOutboundTrackedBudget(64)
|
||||
tr := &failAfterTransport{}
|
||||
c := newOutboundTestConn(t, tr, budget)
|
||||
metrics := &acknowledgementCaptureMetrics{}
|
||||
c.metrics = metrics
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
body := exactTestUpdatesEncoded(t, c, make([]byte, 12))
|
||||
|
|
@ -827,6 +844,9 @@ func TestOutboundTrackedBudgetAckAndCloseReturnExactly(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("decrypt frame: %v", err)
|
||||
}
|
||||
// Windows wall-clock resolution can otherwise make an immediate ACK look
|
||||
// like zero retention even though sentAt was populated after the write.
|
||||
time.Sleep(time.Millisecond)
|
||||
c.AckServerMessages([]int64{data.MessageID})
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for budget.snapshot() != 0 && time.Now().Before(deadline) {
|
||||
|
|
@ -835,6 +855,18 @@ func TestOutboundTrackedBudgetAckAndCloseReturnExactly(t *testing.T) {
|
|||
if got := budget.snapshot(); got != 0 {
|
||||
t.Fatalf("tracked bytes after ack = %d, want 0", got)
|
||||
}
|
||||
if got := metrics.count.Load(); got != 1 {
|
||||
t.Fatalf("logical ACK metric count = %d, want 1", got)
|
||||
}
|
||||
if got := metrics.bytes.Load(); got != 12 {
|
||||
t.Fatalf("logical ACK metric bytes = %d, want 12", got)
|
||||
}
|
||||
if metrics.retainedNS.Load() <= 0 {
|
||||
t.Fatal("logical ACK metric did not record positive retention")
|
||||
}
|
||||
if metrics.rpcResult.Load() {
|
||||
t.Fatal("ordinary update ACK was classified as rpc_result")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("close", func(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -62,9 +62,12 @@ func TestRPCGetConfig(t *testing.T) {
|
|||
if cfg.ThisDC != dc {
|
||||
t.Fatalf("config.ThisDC = %d, want %d", cfg.ThisDC, dc)
|
||||
}
|
||||
// 不下发 DCOptions:客户端使用写死的 static DC 地址(空列表令其保留本地地址)。
|
||||
if len(cfg.DCOptions) != 0 {
|
||||
t.Fatalf("config.DCOptions = %+v, want empty (client uses pinned static address)", cfg.DCOptions)
|
||||
if len(cfg.DCOptions) != 1 {
|
||||
t.Fatalf("config.DCOptions = %+v, want one reconnect route", cfg.DCOptions)
|
||||
}
|
||||
option := cfg.DCOptions[0]
|
||||
if option.ID != dc || option.IPAddress != advIP || option.Port != advPort {
|
||||
t.Fatalf("config.DCOptions[0] = %+v, want dc=%d at %s:%d", option, dc, advIP, advPort)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
186
internal/mtprotoedge/runtime_metrics.go
Normal file
186
internal/mtprotoedge/runtime_metrics.go
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
package mtprotoedge
|
||||
|
||||
// RuntimeSnapshot is a point-in-time, identity-free view of the MTProto edge.
|
||||
// It deliberately exposes only bounded aggregate values so callers can publish
|
||||
// it through a metrics endpoint without leaking auth keys, sessions or remote
|
||||
// addresses. Values from independently locked components can differ by one
|
||||
// concurrent transition; every individual budget/count remains internally
|
||||
// consistent.
|
||||
type RuntimeSnapshot struct {
|
||||
RawConnections int64
|
||||
RawConnectionLimit int64
|
||||
Handshakes int64
|
||||
HandshakeLimit int64
|
||||
ActiveSessions int64
|
||||
ProvisionalSessions int64
|
||||
LogicalSessions int64
|
||||
OfflineLogicalSessions int64
|
||||
LogicalOutboxFrames int64
|
||||
LogicalOutboxBytes int64
|
||||
PendingPushBytes int64
|
||||
InboundRPCTasks int64
|
||||
InboundRPCBytes int64
|
||||
InboundRPCReadyConnections int64
|
||||
InboundRPCMaxTasks int64
|
||||
InboundRPCMaxBytes int64
|
||||
InboundFrameBytes int64
|
||||
InboundFrameMaxBytes int64
|
||||
OutboundTrackedBytes int64
|
||||
OutboundTrackedMaxBytes int64
|
||||
OutboundControlBytes int64
|
||||
OutboundControlMaxBytes int64
|
||||
OutboundWriteBytes int64
|
||||
OutboundWriteMaxBytes int64
|
||||
RPCResultOwners int64
|
||||
RPCResultReceipts int64
|
||||
RPCResultReceiptBytes int64
|
||||
RPCResultSubscribers int64
|
||||
}
|
||||
|
||||
type sessionManagerRuntimeSnapshot struct {
|
||||
active int64
|
||||
provisional int64
|
||||
logical int64
|
||||
offlineLogical int64
|
||||
frames int64
|
||||
bytes int64
|
||||
pendingBytes int64
|
||||
}
|
||||
|
||||
func (m *SessionManager) runtimeSnapshot() sessionManagerRuntimeSnapshot {
|
||||
if m == nil {
|
||||
return sessionManagerRuntimeSnapshot{}
|
||||
}
|
||||
|
||||
// Never hold SessionManager.mu while taking an outbound-state mutex. The
|
||||
// physical actor can publish/retire a Conn next to an outbox transition, and
|
||||
// metrics must not add a new cross-component lock order.
|
||||
m.mu.RLock()
|
||||
states := make([]*outboundState, 0, len(m.logicalSessions))
|
||||
result := sessionManagerRuntimeSnapshot{
|
||||
active: int64(len(m.bySession)),
|
||||
provisional: int64(len(m.claims)),
|
||||
logical: int64(len(m.logicalSessions)),
|
||||
}
|
||||
if m.pendingBudget != nil {
|
||||
result.pendingBytes = m.pendingBudget.snapshot()
|
||||
}
|
||||
for _, logical := range m.logicalSessions {
|
||||
if logical == nil {
|
||||
continue
|
||||
}
|
||||
if !logical.offlineAt.IsZero() {
|
||||
result.offlineLogical++
|
||||
}
|
||||
if logical.outbound != nil {
|
||||
states = append(states, logical.outbound)
|
||||
}
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
|
||||
for _, state := range states {
|
||||
state.mu.Lock()
|
||||
result.frames += int64(len(state.pending))
|
||||
result.bytes += int64(state.totalBytes)
|
||||
state.mu.Unlock()
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type admissionRuntimeSnapshot struct {
|
||||
connections int64
|
||||
connectionLimit int64
|
||||
handshakes int64
|
||||
handshakeLimit int64
|
||||
}
|
||||
|
||||
func (a *admissionController) runtimeSnapshot() admissionRuntimeSnapshot {
|
||||
if a == nil {
|
||||
return admissionRuntimeSnapshot{}
|
||||
}
|
||||
a.mu.Lock()
|
||||
result := admissionRuntimeSnapshot{
|
||||
connections: int64(a.connections),
|
||||
connectionLimit: int64(a.maxConnections),
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if a.handshakes != nil {
|
||||
result.handshakes = int64(len(a.handshakes))
|
||||
result.handshakeLimit = int64(cap(a.handshakes))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type inboundRPCRuntimeSnapshot struct {
|
||||
tasks int64
|
||||
bytes int64
|
||||
ready int64
|
||||
}
|
||||
|
||||
func (s *inboundRPCScheduler) runtimeSnapshot() inboundRPCRuntimeSnapshot {
|
||||
if s == nil {
|
||||
return inboundRPCRuntimeSnapshot{}
|
||||
}
|
||||
s.budgetMu.Lock()
|
||||
result := inboundRPCRuntimeSnapshot{tasks: int64(s.tasks), bytes: s.bytes}
|
||||
s.budgetMu.Unlock()
|
||||
s.readyMu.Lock()
|
||||
result.ready = int64(s.ready.Len())
|
||||
s.readyMu.Unlock()
|
||||
return result
|
||||
}
|
||||
|
||||
// RuntimeSnapshot returns aggregate MTProto ownership and capacity state.
|
||||
func (s *Server) RuntimeSnapshot() RuntimeSnapshot {
|
||||
if s == nil {
|
||||
return RuntimeSnapshot{}
|
||||
}
|
||||
sessions := s.conns.runtimeSnapshot()
|
||||
admission := s.admission.runtimeSnapshot()
|
||||
inbound := s.rpcScheduler.runtimeSnapshot()
|
||||
result := RuntimeSnapshot{
|
||||
RawConnections: admission.connections,
|
||||
RawConnectionLimit: admission.connectionLimit,
|
||||
Handshakes: admission.handshakes,
|
||||
HandshakeLimit: admission.handshakeLimit,
|
||||
ActiveSessions: sessions.active,
|
||||
ProvisionalSessions: sessions.provisional,
|
||||
LogicalSessions: sessions.logical,
|
||||
OfflineLogicalSessions: sessions.offlineLogical,
|
||||
LogicalOutboxFrames: sessions.frames,
|
||||
LogicalOutboxBytes: sessions.bytes,
|
||||
PendingPushBytes: sessions.pendingBytes,
|
||||
InboundRPCTasks: inbound.tasks,
|
||||
InboundRPCBytes: inbound.bytes,
|
||||
InboundRPCReadyConnections: inbound.ready,
|
||||
}
|
||||
if s.rpcScheduler != nil {
|
||||
result.InboundRPCMaxTasks = int64(s.rpcScheduler.maxTasks)
|
||||
result.InboundRPCMaxBytes = s.rpcScheduler.maxBytes
|
||||
}
|
||||
if s.frameBudget != nil {
|
||||
result.InboundFrameBytes = s.frameBudget.usedBytes()
|
||||
result.InboundFrameMaxBytes = s.frameBudget.max
|
||||
}
|
||||
if s.outboundTrackedBudget != nil {
|
||||
result.OutboundTrackedBytes = s.outboundTrackedBudget.snapshot()
|
||||
result.OutboundTrackedMaxBytes = s.outboundTrackedBudget.maxBytes
|
||||
}
|
||||
if s.outboundControlBudget != nil {
|
||||
result.OutboundControlBytes = s.outboundControlBudget.snapshot()
|
||||
result.OutboundControlMaxBytes = s.outboundControlBudget.maxBytes
|
||||
}
|
||||
if s.outboundScratchPool != nil && s.outboundScratchPool.budget != nil {
|
||||
result.OutboundWriteBytes = s.outboundScratchPool.snapshot()
|
||||
result.OutboundWriteMaxBytes = s.outboundScratchPool.budget.maxBytes
|
||||
}
|
||||
if s.rpcResults != nil {
|
||||
result.RPCResultOwners = s.rpcResults.flightLimit.snapshot()
|
||||
result.RPCResultReceipts = s.rpcResults.completedEntries.snapshot()
|
||||
result.RPCResultReceiptBytes = s.rpcResults.completedBytes.snapshot()
|
||||
if s.rpcResults.subscriberBudget != nil {
|
||||
result.RPCResultSubscribers = s.rpcResults.subscriberBudget.global.snapshot()
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
27
internal/mtprotoedge/runtime_metrics_test.go
Normal file
27
internal/mtprotoedge/runtime_metrics_test.go
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
package mtprotoedge
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestRuntimeSnapshotIsNilSafeAndReportsConfiguredLimits(t *testing.T) {
|
||||
if got := (*Server)(nil).RuntimeSnapshot(); got != (RuntimeSnapshot{}) {
|
||||
t.Fatalf("nil server snapshot = %#v, want zero", got)
|
||||
}
|
||||
if got := (&Server{}).RuntimeSnapshot(); got != (RuntimeSnapshot{}) {
|
||||
t.Fatalf("partial server snapshot = %#v, want zero", got)
|
||||
}
|
||||
|
||||
server := New(Options{})
|
||||
snapshot := server.RuntimeSnapshot()
|
||||
if snapshot.RawConnectionLimit <= 0 || snapshot.HandshakeLimit <= 0 {
|
||||
t.Fatalf("admission limits not reported: %#v", snapshot)
|
||||
}
|
||||
if snapshot.InboundRPCMaxTasks <= 0 || snapshot.InboundRPCMaxBytes <= 0 {
|
||||
t.Fatalf("inbound RPC limits not reported: %#v", snapshot)
|
||||
}
|
||||
if snapshot.InboundFrameMaxBytes <= 0 || snapshot.OutboundTrackedMaxBytes <= 0 || snapshot.OutboundWriteMaxBytes <= 0 {
|
||||
t.Fatalf("byte limits not reported: %#v", snapshot)
|
||||
}
|
||||
if snapshot.RawConnections != 0 || snapshot.ActiveSessions != 0 || snapshot.LogicalOutboxBytes != 0 {
|
||||
t.Fatalf("fresh server reported live ownership: %#v", snapshot)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue