package loadharness import ( "context" "crypto/md5" cryptorand "crypto/rand" "crypto/rsa" "encoding/binary" "encoding/hex" "errors" "fmt" "io" "math" "net" "os" "runtime" "sort" "strings" "sync" "sync/atomic" "syscall" "time" "github.com/iamxvbaba/td/mtproto" "github.com/iamxvbaba/td/pool" tdrpc "github.com/iamxvbaba/td/rpc" "github.com/iamxvbaba/td/telegram" "github.com/iamxvbaba/td/tg" ) const ( workerConnecting int32 = iota workerReady workerDisconnected workerOffline workerStopped ) type RunConfig struct { ManifestPath string SessionKeyPath string RSAKeyOverride string ReportPath string EventsPath string FileFixturePath string ServerMetricsURL string StartOrder string StartOrderSeed int64 SessionLimit int Duration time.Duration RecoveryDuration time.Duration RampDuration time.Duration RPCInterval time.Duration MessageInterval time.Duration MessageRate float64 MessageQueueDepth int DeliverySettle time.Duration FileInterval time.Duration FileSizeBytes int FileChunkBytes int SetupTimeout time.Duration OperationTimeout time.Duration SampleInterval time.Duration OfflineFraction float64 OfflineAt time.Duration OfflineFor time.Duration MinimumReadyRatio float64 ExpectServerRestart bool } const RunReportVersion = 7 func (c RunConfig) validate() error { if c.ManifestPath == "" || c.SessionKeyPath == "" || c.ReportPath == "" { return errors.New("manifest, session-key and report paths are required") } if c.Duration <= 0 || c.RecoveryDuration < 0 || c.RampDuration < 0 || c.RPCInterval <= 0 || c.OperationTimeout <= 0 || c.SampleInterval <= 0 { return errors.New("run durations and intervals are invalid") } if c.MessageRate < 0 || c.MessageRate > 100000 || c.MessageQueueDepth < 0 || c.MessageQueueDepth > 1024 || c.DeliverySettle < 0 { return errors.New("message rate, queue depth or delivery settle is invalid") } if c.MessageRate > 0 && c.MessageInterval > 0 { return errors.New("message-rate and message-interval workloads are mutually exclusive") } if c.MessageRate > 0 && (c.MessageQueueDepth == 0 || c.RampDuration >= c.Duration) { return errors.New("fixed-rate workload requires a queue depth and load duration beyond the connection ramp") } if c.FileSizeBytes < 0 || c.FileChunkBytes < 0 || c.FileChunkBytes > 1<<20 || c.FileSizeBytes > 64<<20 { return errors.New("file size must be <=64MiB and chunk size must be <=1MiB") } if c.FileSizeBytes > 0 && (c.FileInterval <= 0 || c.FileChunkBytes <= 0) { return errors.New("enabled file workload requires positive interval and chunk size") } if c.FileSizeBytes > 0 && c.SetupTimeout <= 0 { return errors.New("enabled file workload requires a positive setup timeout") } if c.OfflineFraction < 0 || c.OfflineFraction > 1 || c.MinimumReadyRatio <= 0 || c.MinimumReadyRatio > 1 { return errors.New("offline fraction and minimum ready ratio must be within [0,1]") } if c.OfflineFraction > 0 && (c.OfflineAt <= 0 || c.OfflineFor <= 0 || c.OfflineAt+c.OfflineFor >= c.Duration) { return errors.New("offline window must be positive and fit inside load duration") } if c.StartOrder != "" && c.StartOrder != StartupOrderShuffled && c.StartOrder != StartupOrderAccountIndex { return fmt.Errorf("unknown run start order %q", c.StartOrder) } return nil } type updateState struct { mu sync.Mutex value tg.UpdatesState valid bool } func (s *updateState) load() (tg.UpdatesState, bool) { s.mu.Lock() defer s.mu.Unlock() return s.value, s.valid } func (s *updateState) store(value tg.UpdatesState) { s.mu.Lock() s.value = value s.valid = true s.mu.Unlock() } type harnessCounters struct { connectionAttempts atomic.Uint64 reconnects atomic.Uint64 disconnects atomic.Uint64 updates atomic.Uint64 fatalErrors atomic.Uint64 downloadBytes atomic.Uint64 messageScheduled atomic.Uint64 messageEnqueued atomic.Uint64 messageCompleted atomic.Uint64 messageQueueFull atomic.Uint64 messageNotReady atomic.Uint64 } var debugConnectionErrors atomic.Uint64 var debugOperationErrors atomic.Uint64 func debugConnectionError(sessionIndex int, err error) { if os.Getenv("TELESRV_LOAD_DEBUG_ERRORS") != "1" || err == nil || debugConnectionErrors.Add(1) > 20 { return } // Explicit opt-in diagnostics go only to stderr and are bounded. They are // never copied into the attachable report or event stream. fmt.Fprintf(os.Stderr, "load debug: session=%d error_type=%T error=%v\n", sessionIndex, err, err) } func debugOperationError(operation string, err error) { // Connection lifecycle errors have their own bounded diagnostic path. Do not // let an expected reconnect storm consume the operation-error budget that is // needed to diagnose actual RPC failures. if operation == "connection.dead" || os.Getenv("TELESRV_LOAD_DEBUG_ERRORS") != "1" || err == nil || debugOperationErrors.Add(1) > 20 { return } // Operation names are a code-owned finite vocabulary. Raw errors are // opt-in, stderr-only and bounded; reports/events retain only classifications. fmt.Fprintf(os.Stderr, "load debug: operation=%s error_type=%T error=%v\n", operation, err, err) } type downloadFixture struct { location *tg.InputDocumentFileLocation size int chunk int } type loadWorker struct { record SessionRecord target SessionRecord endpoint Endpoint publicKey *rsa.PublicKey storage *EncryptedFileStorage metrics *metricSet counters *harnessCounters events *eventWriter rpcInterval time.Duration msgInterval time.Duration fileInterval time.Duration operationTimeout time.Duration fileFixture *downloadFixture delivery *deliveryTracker desired atomic.Bool state atomic.Int32 everReady atomic.Bool signal chan struct{} lastUpdate updateState deliveryState updateState messageSeq atomic.Uint64 sendQueue chan struct{} reconcile chan chan struct{} } func newLoadWorker(record, target SessionRecord, endpoint Endpoint, publicKey *rsa.PublicKey, storage *EncryptedFileStorage, metrics *metricSet, counters *harnessCounters, events *eventWriter, rpcInterval, messageInterval, fileInterval, operationTimeout time.Duration, fixture *downloadFixture, delivery *deliveryTracker, messageQueueDepth int) *loadWorker { w := &loadWorker{ record: record, target: target, endpoint: endpoint, publicKey: publicKey, storage: storage, metrics: metrics, counters: counters, events: events, rpcInterval: rpcInterval, msgInterval: messageInterval, fileInterval: fileInterval, operationTimeout: operationTimeout, fileFixture: fixture, delivery: delivery, signal: make(chan struct{}, 1), sendQueue: make(chan struct{}, messageQueueDepth), reconcile: make(chan chan struct{}), } w.state.Store(workerStopped) return w } func (w *loadWorker) operationContext(parent context.Context) (context.Context, context.CancelFunc) { return context.WithTimeout(parent, w.operationTimeout) } func (w *loadWorker) setOnline(online bool) { w.desired.Store(online) select { case w.signal <- struct{}{}: default: } } func (w *loadWorker) supervise(ctx context.Context, wg *sync.WaitGroup) { defer wg.Done() for { if err := ctx.Err(); err != nil { w.state.Store(workerStopped) return } if !w.desired.Load() { w.state.Store(workerOffline) select { case <-ctx.Done(): w.state.Store(workerStopped) return case <-w.signal: continue } } clientCtx, cancel := context.WithCancel(ctx) done := make(chan error, 1) go func() { done <- w.runClient(clientCtx) }() for w.desired.Load() { select { case <-ctx.Done(): cancel() <-done w.state.Store(workerStopped) return case <-w.signal: if !w.desired.Load() { cancel() <-done w.state.Store(workerOffline) } case err := <-done: cancel() if ctx.Err() != nil { w.state.Store(workerStopped) return } if !w.desired.Load() { w.state.Store(workerOffline) goto nextClient } w.counters.fatalErrors.Add(1) w.events.write(map[string]any{ "type": "worker_error", "at": time.Now().UTC(), "session_index": w.record.Index, "class": classifyError(err), }) select { case <-ctx.Done(): return case <-time.After(time.Second): } goto nextClient } } cancel() select { case <-done: default: } nextClient: } } func (w *loadWorker) runClient(ctx context.Context) error { reconnectSignal := make(chan struct{}, 1) var readySeen atomic.Bool client, err := newClient(w.endpoint, w.publicKey, w.storage, clientHooks{ Update: telegram.UpdateHandlerFunc(func(_ context.Context, updates tg.UpdatesClass) error { w.counters.updates.Add(1) observeUpdatesClass(w.delivery, w.record.UserID, updates, deliveryLive) return nil }), ConnectionState: func(state telegram.ConnectionState) { switch state { case telegram.ConnectionStateConnecting: w.state.Store(workerConnecting) w.counters.connectionAttempts.Add(1) if w.everReady.Load() { w.counters.reconnects.Add(1) } case telegram.ConnectionStateReady: needsCatchUp := markClientReady(&w.everReady, &readySeen) w.state.Store(workerReady) if needsCatchUp { select { case reconnectSignal <- struct{}{}: default: } } case telegram.ConnectionStateDisconnected: w.state.Store(workerDisconnected) w.counters.disconnects.Add(1) } }, Dead: func(err error) { debugConnectionError(w.record.Index, err) w.metrics.observe("connection.dead", time.Now(), err) w.events.write(map[string]any{ "type": "connection_dead", "at": time.Now().UTC(), "session_index": w.record.Index, "class": classifyError(err), "reason": classifyErrorReason(err), }) }, }) if err != nil { return err } return client.Run(ctx, func(ctx context.Context) error { statusStart := time.Now() operationCtx, cancelOperation := w.operationContext(ctx) status, err := client.Auth().Status(operationCtx) cancelOperation() w.metrics.observe("auth.status", statusStart, err) if err != nil { return err } if !status.Authorized || status.User == nil || status.User.ID != w.record.UserID { return errors.New("session is not authorized for its manifest user") } raw := tg.NewClient(client) if _, valid := w.lastUpdate.load(); valid { w.catchUp(ctx, raw) } else { w.refreshUpdateState(ctx, raw) } if _, valid := w.deliveryState.load(); !valid { w.refreshDeliveryState(ctx, raw) } rpcTicker := time.NewTicker(w.rpcInterval) defer rpcTicker.Stop() var messageTicker *time.Ticker var messageC <-chan time.Time if w.msgInterval > 0 && w.record.DeviceIndex == 0 && w.target.UserID > 0 { messageTicker = time.NewTicker(w.msgInterval) messageC = messageTicker.C defer messageTicker.Stop() } var fileTicker *time.Ticker var fileC <-chan time.Time if w.fileFixture != nil && w.fileInterval > 0 { fileTicker = time.NewTicker(w.fileInterval) fileC = fileTicker.C defer fileTicker.Stop() } cycle := 0 for { select { case <-ctx.Done(): return ctx.Err() case <-reconnectSignal: w.catchUp(ctx, raw) case <-rpcTicker.C: w.runRPC(ctx, client, raw, cycle) cycle++ case <-messageC: w.sendMessage(ctx, raw) case <-w.sendQueue: w.sendMessage(ctx, raw) case done := <-w.reconcile: w.catchUpDelivery(ctx, raw) close(done) case <-fileC: w.downloadFileChunk(ctx, raw) } } }) } // markClientReady distinguishes a transport reconnect inside one live gotd // Client from the first Ready transition of a newly constructed Client. The // client.Run callback already performs one cursor catch-up when it starts, so // enqueueing a second catch-up for that first transition would duplicate every // explicit offline->online getDifference request. Later Ready transitions do // need the signal because the callback remains running across transport-level // reconnects. func markClientReady(everReady, readySeen *atomic.Bool) bool { firstForClient := !readySeen.Swap(true) wasReady := everReady.Swap(true) return wasReady && !firstForClient } func (w *loadWorker) runRPC(ctx context.Context, client *telegram.Client, raw *tg.Client, cycle int) { start := time.Now() operationCtx, cancel := w.operationContext(ctx) defer cancel() var err error switch cycle % 4 { case 0: err = client.Ping(operationCtx) w.metrics.observe("ping", start, err) case 1: var state *tg.UpdatesState state, err = raw.UpdatesGetState(operationCtx) if err == nil { w.lastUpdate.store(*state) } w.metrics.observe("updates.getState", start, err) case 2: _, err = raw.MessagesGetDialogs(operationCtx, &tg.MessagesGetDialogsRequest{OffsetPeer: &tg.InputPeerEmpty{}, Limit: 20}) w.metrics.observe("messages.getDialogs", start, err) case 3: _, err = raw.HelpGetConfig(operationCtx) w.metrics.observe("help.getConfig", start, err) } } func (w *loadWorker) refreshUpdateState(ctx context.Context, raw *tg.Client) { start := time.Now() operationCtx, cancel := w.operationContext(ctx) state, err := raw.UpdatesGetState(operationCtx) cancel() w.metrics.observe("updates.getState", start, err) if err == nil { w.lastUpdate.store(*state) if _, valid := w.deliveryState.load(); !valid { w.deliveryState.store(*state) } } } func (w *loadWorker) refreshDeliveryState(ctx context.Context, raw *tg.Client) { start := time.Now() operationCtx, cancel := w.operationContext(ctx) state, err := raw.UpdatesGetState(operationCtx) cancel() w.metrics.observe("updates.getState.delivery", start, err) if err == nil { w.deliveryState.store(*state) } } func (w *loadWorker) catchUp(ctx context.Context, raw *tg.Client) { state, valid := w.lastUpdate.load() if !valid { w.refreshUpdateState(ctx, raw) return } for page := 0; page < 32; page++ { start := time.Now() operationCtx, cancel := w.operationContext(ctx) difference, err := raw.UpdatesGetDifference(operationCtx, &tg.UpdatesGetDifferenceRequest{Pts: state.Pts, Date: state.Date, Qts: state.Qts}) cancel() w.metrics.observe("updates.getDifference", start, err) if err != nil { return } switch value := difference.(type) { case *tg.UpdatesDifferenceEmpty: state.Date, state.Seq = value.Date, value.Seq w.lastUpdate.store(state) return case *tg.UpdatesDifference: state = value.State w.counters.updates.Add(uint64(len(value.NewMessages) + len(value.NewEncryptedMessages) + len(value.OtherUpdates))) w.lastUpdate.store(state) return case *tg.UpdatesDifferenceSlice: state = value.IntermediateState w.counters.updates.Add(uint64(len(value.NewMessages) + len(value.NewEncryptedMessages) + len(value.OtherUpdates))) w.lastUpdate.store(state) case *tg.UpdatesDifferenceTooLong: state.Pts = value.Pts w.lastUpdate.store(state) return default: return } } } func (w *loadWorker) catchUpDelivery(ctx context.Context, raw *tg.Client) { state, valid := w.deliveryState.load() if !valid { w.refreshDeliveryState(ctx, raw) return } for page := 0; page < 256; page++ { start := time.Now() operationCtx, cancel := w.operationContext(ctx) difference, err := raw.UpdatesGetDifference(operationCtx, &tg.UpdatesGetDifferenceRequest{Pts: state.Pts, Date: state.Date, Qts: state.Qts}) cancel() w.metrics.observe("updates.getDifference.delivery", start, err) if err != nil { return } switch value := difference.(type) { case *tg.UpdatesDifferenceEmpty: state.Date, state.Seq = value.Date, value.Seq w.deliveryState.store(state) return case *tg.UpdatesDifference: observeMessageClasses(w.delivery, w.record.UserID, value.NewMessages, deliveryDifference) observeUpdateClasses(w.delivery, w.record.UserID, value.OtherUpdates, deliveryDifference) state = value.State w.deliveryState.store(state) return case *tg.UpdatesDifferenceSlice: observeMessageClasses(w.delivery, w.record.UserID, value.NewMessages, deliveryDifference) observeUpdateClasses(w.delivery, w.record.UserID, value.OtherUpdates, deliveryDifference) state = value.IntermediateState w.deliveryState.store(state) case *tg.UpdatesDifferenceTooLong: state.Pts = value.Pts w.deliveryState.store(state) return default: return } } } func (w *loadWorker) sendMessage(ctx context.Context, raw *tg.Client) { defer w.counters.messageCompleted.Add(1) sequence := w.messageSeq.Add(1) var randomBytes [8]byte if _, err := cryptorand.Read(randomBytes[:]); err != nil { return } randomID := int64(binary.LittleEndian.Uint64(randomBytes[:])) if randomID == 0 { randomID = int64(sequence) } start := time.Now() marker := w.delivery.marker(w.record.Index, sequence) w.delivery.begin(marker, w.record.UserID, w.target.UserID, start) operationCtx, cancel := w.operationContext(ctx) _, err := raw.MessagesSendMessage(operationCtx, &tg.MessagesSendMessageRequest{ Peer: &tg.InputPeerUser{UserID: w.target.UserID, AccessHash: w.target.AccessHash}, Message: marker, RandomID: randomID, }) cancel() w.metrics.observe("messages.sendMessage", start, err) w.delivery.finish(marker, err == nil) } func (w *loadWorker) downloadFileChunk(ctx context.Context, raw *tg.Client) { fixture := w.fileFixture if fixture == nil || fixture.size <= 0 || fixture.chunk <= 0 { return } sequence := w.messageSeq.Add(1) offset := int64((int(sequence-1) * fixture.chunk) % fixture.size) limit := min(fixture.chunk, fixture.size-int(offset)) start := time.Now() operationCtx, cancel := w.operationContext(ctx) result, err := raw.UploadGetFile(operationCtx, &tg.UploadGetFileRequest{ Location: fixture.location, Offset: offset, Limit: limit, }) cancel() if err == nil { file, ok := result.(*tg.UploadFile) switch { case !ok: err = fmt.Errorf("upload.getFile returned %T", result) case len(file.Bytes) != limit: err = fmt.Errorf("upload.getFile bytes=%d want=%d", len(file.Bytes), limit) case !validFixtureBytes(file.Bytes, offset): err = errors.New("upload.getFile payload mismatch") default: w.counters.downloadBytes.Add(uint64(len(file.Bytes))) } } w.metrics.observe("upload.getFile", start, err) } func prepareDownloadFixture(ctx context.Context, cfg RunConfig, manifest *Manifest, record SessionRecord, key [32]byte, publicKey *rsa.PublicKey, metrics *metricSet) (*downloadFixture, error) { data := make([]byte, cfg.FileSizeBytes) for i := range data { data[i] = fixtureByte(int64(i)) } fileID, err := randomNonZeroInt64() if err != nil { return nil, err } storage := &EncryptedFileStorage{Path: resolveSessionPath(cfg.ManifestPath, record), Key: key} client, err := newClient(manifest.Endpoint, publicKey, storage, clientHooks{}) if err != nil { return nil, err } var location *tg.InputDocumentFileLocation err = client.Run(ctx, func(ctx context.Context) error { status, err := client.Auth().Status(ctx) if err != nil { return err } if !status.Authorized || status.User == nil || status.User.ID != record.UserID { return errors.New("fixture session is not authorized") } raw := tg.NewClient(client) const partSize = 512 << 10 parts := (len(data) + partSize - 1) / partSize big := len(data) > 10<<20 for part := 0; part < parts; part++ { startOffset := part * partSize endOffset := min(len(data), startOffset+partSize) start := time.Now() var saved bool if big { saved, err = raw.UploadSaveBigFilePart(ctx, &tg.UploadSaveBigFilePartRequest{ FileID: fileID, FilePart: part, FileTotalParts: parts, Bytes: data[startOffset:endOffset], }) } else { saved, err = raw.UploadSaveFilePart(ctx, &tg.UploadSaveFilePartRequest{ FileID: fileID, FilePart: part, Bytes: data[startOffset:endOffset], }) } metrics.observe("upload.saveFilePart", start, err) if err != nil { return err } if !saved { return fmt.Errorf("upload part %d was not saved", part) } } var file tg.InputFileClass if big { file = &tg.InputFileBig{ID: fileID, Parts: parts, Name: "telesrv-load.bin"} } else { digest := md5.Sum(data) file = &tg.InputFile{ID: fileID, Parts: parts, Name: "telesrv-load.bin", MD5Checksum: hex.EncodeToString(digest[:])} } start := time.Now() media, err := raw.MessagesUploadMedia(ctx, &tg.MessagesUploadMediaRequest{ Peer: &tg.InputPeerEmpty{}, Media: &tg.InputMediaUploadedDocument{ File: file, MimeType: "application/octet-stream", Attributes: []tg.DocumentAttributeClass{&tg.DocumentAttributeFilename{FileName: "telesrv-load.bin"}}, }, }) metrics.observe("messages.uploadMedia", start, err) if err != nil { return err } documentMedia, ok := media.(*tg.MessageMediaDocument) if !ok { return fmt.Errorf("messages.uploadMedia returned %T", media) } documentClass, ok := documentMedia.GetDocument() if !ok { return errors.New("messages.uploadMedia omitted document") } document, ok := documentClass.(*tg.Document) if !ok { return fmt.Errorf("uploaded document is %T", documentClass) } location = &tg.InputDocumentFileLocation{ ID: document.ID, AccessHash: document.AccessHash, FileReference: append([]byte(nil), document.FileReference...), } return nil }) if err != nil { return nil, err } if location == nil { return nil, errors.New("file fixture completed without a location") } return &downloadFixture{location: location, size: len(data), chunk: cfg.FileChunkBytes}, nil } func loadOrCreateDownloadFixture(ctx context.Context, cfg RunConfig, manifest *Manifest, record SessionRecord, key [32]byte, publicKey *rsa.PublicKey, metrics *metricSet) (*downloadFixture, error) { path := resolveFileFixturePath(cfg.ManifestPath, cfg.FileFixturePath) fixture, err := loadPersistedFileFixture(path, manifest.Endpoint, cfg.FileSizeBytes, cfg.FileChunkBytes) if err == nil { return fixture, nil } if !os.IsNotExist(err) { return nil, fmt.Errorf("load file fixture %q: %w", path, err) } setupCtx, cancel := context.WithTimeout(ctx, cfg.SetupTimeout) defer cancel() fixture, err = prepareDownloadFixture(setupCtx, cfg, manifest, record, key, publicKey, metrics) if err != nil { if errors.Is(err, context.DeadlineExceeded) || errors.Is(setupCtx.Err(), context.DeadlineExceeded) { return nil, fmt.Errorf("create file fixture timed out after %s", cfg.SetupTimeout) } return nil, err } if err := writePersistedFileFixture(path, manifest.Endpoint, fixture); err != nil { return nil, fmt.Errorf("persist file fixture: %w", err) } return fixture, nil } func randomNonZeroInt64() (int64, error) { var data [8]byte if _, err := cryptorand.Read(data[:]); err != nil { return 0, err } value := int64(binary.LittleEndian.Uint64(data[:]) & math.MaxInt64) if value == 0 { value = 1 } return value, nil } func fixtureByte(offset int64) byte { return byte((uint64(offset)*31 + 17) % 251) } func validFixtureBytes(data []byte, offset int64) bool { for i, value := range data { if value != fixtureByte(offset+int64(i)) { return false } } return true } // Run executes a bounded real-client load and keeps scraping after all workers // stop so logical-session/outbox reclamation can be proven rather than assumed. func Run(ctx context.Context, cfg RunConfig) (*RunReport, error) { if err := cfg.validate(); err != nil { return nil, err } manifest, err := LoadManifest(cfg.ManifestPath) if err != nil { return nil, err } key, err := LoadSessionKey(cfg.SessionKeyPath) if err != nil { return nil, err } publicKey, err := loadManifestPublicKey(cfg.ManifestPath, manifest.Endpoint, cfg.RSAKeyOverride) if err != nil { return nil, err } records := manifest.Sessions if cfg.SessionLimit > 0 && cfg.SessionLimit < len(records) { records = records[:cfg.SessionLimit] } if len(records) == 0 { return nil, errors.New("manifest has no selected sessions") } if err := validateProcessCapacity(len(records)); err != nil { return nil, err } events, err := newEventWriter(cfg.EventsPath) if err != nil { return nil, err } defer events.close() metrics := newMetricSet("auth.status", "connection.dead", "ping", "updates.getState", "updates.getDifference", "updates.getState.delivery", "updates.getDifference.delivery", "messages.getDialogs", "help.getConfig", "messages.sendMessage", "upload.saveFilePart", "messages.uploadMedia", "upload.getFile") counters := &harnessCounters{} runID, err := newLoadRunID() if err != nil { return nil, fmt.Errorf("create load run id: %w", err) } delivery := newDeliveryTracker(runID) serverMetrics := newServerMetricsClient(cfg.ServerMetricsURL) var baselineServerMetrics map[string]float64 if serverMetrics != nil { if sample, scrapeErr := serverMetrics.scrape(ctx); scrapeErr == nil { baselineServerMetrics = sample events.write(map[string]any{"type": "server_baseline", "at": time.Now().UTC(), "server_metrics": sample}) } else { events.write(map[string]any{"type": "server_baseline_error", "at": time.Now().UTC(), "class": classifyError(scrapeErr)}) } } var fixture *downloadFixture if cfg.FileSizeBytes > 0 { fixture, err = loadOrCreateDownloadFixture(ctx, cfg, manifest, records[0], key, publicKey, metrics) if err != nil { return nil, fmt.Errorf("prepare file fixture: %w", err) } } targets := primaryTargets(records) workers := make([]*loadWorker, 0, len(records)) for _, record := range records { target := targets[(record.AccountIndex+1)%len(targets)] workers = append(workers, newLoadWorker( record, target, manifest.Endpoint, publicKey, &EncryptedFileStorage{Path: resolveSessionPath(cfg.ManifestPath, record), Key: key}, metrics, counters, events, cfg.RPCInterval, cfg.MessageInterval, cfg.FileInterval, cfg.OperationTimeout, fixture, delivery, cfg.MessageQueueDepth, )) } messageWorkers := primaryWorkers(workers) startedAt := time.Now().UTC() loadCtx, stopLoad := context.WithCancel(ctx) var workerWG sync.WaitGroup for _, worker := range workers { workerWG.Add(1) go worker.supervise(loadCtx, &workerWG) } startOrder := cfg.StartOrder if startOrder == "" { startOrder = StartupOrderAccountIndex } startOrderSeed := cfg.StartOrderSeed if startOrderSeed == 0 { startOrderSeed = 20260827 } launchOrder := startupAccountOrder(len(workers), startOrder, startOrderSeed) for position, workerIndex := range launchOrder { worker := workers[workerIndex] delay := time.Duration(0) if len(workers) > 1 { delay = time.Duration(position) * cfg.RampDuration / time.Duration(len(workers)-1) } go func(w *loadWorker, d time.Duration) { timer := time.NewTimer(d) defer timer.Stop() select { case <-loadCtx.Done(): case <-timer.C: w.setOnline(true) } }(worker, delay) } if cfg.OfflineFraction > 0 { go runOfflineWindow(loadCtx, workers, cfg.OfflineFraction, cfg.OfflineAt, cfg.OfflineFor, events) } messageCtx, stopMessages := context.WithCancel(loadCtx) defer stopMessages() var messageWG sync.WaitGroup if cfg.MessageRate > 0 { messageWG.Add(1) go runFixedMessageSchedule(messageCtx, &messageWG, cfg.RampDuration, cfg.MessageRate, messageWorkers, counters, events) } loadTimer := time.NewTimer(cfg.Duration) sampleTicker := time.NewTicker(cfg.SampleInterval) peakReady := 0 steadySamples := 0 steadyReadySum := 0 steadyReadyMinimum := len(workers) var finalServerMetrics map[string]float64 for { select { case <-ctx.Done(): stopLoad() workerWG.Wait() return nil, ctx.Err() case <-sampleTicker.C: ready := countWorkerState(workers, workerReady) peakReady = max(peakReady, ready) elapsed := time.Since(startedAt) outsideOffline := cfg.OfflineFraction == 0 || elapsed < cfg.OfflineAt || elapsed > cfg.OfflineAt+cfg.OfflineFor+30*time.Second if elapsed >= cfg.RampDuration+30*time.Second && outsideOffline { steadySamples++ steadyReadySum += ready steadyReadyMinimum = min(steadyReadyMinimum, ready) } finalServerMetrics = writeSample(ctx, events, "load", workers, metrics, counters, serverMetrics) case <-loadTimer.C: goto loadFinished } } loadFinished: sampleTicker.Stop() stopMessages() messageWG.Wait() loadEndedAt := time.Now().UTC() if cfg.MessageRate > 0 { drainCtx, cancelDrain := context.WithTimeout(ctx, cfg.OperationTimeout+time.Duration(cfg.MessageQueueDepth)*cfg.OperationTimeout) waitMessageDrain(drainCtx, counters) cancelDrain() } if cfg.DeliverySettle > 0 && delivery.report().Expected > delivery.report().Delivered { settleTimer := time.NewTimer(cfg.DeliverySettle) settleTicker := time.NewTicker(min(cfg.SampleInterval, time.Second)) settling: for { select { case <-ctx.Done(): settleTimer.Stop() settleTicker.Stop() stopLoad() workerWG.Wait() return nil, ctx.Err() case <-settleTicker.C: if current := delivery.report(); current.Missing == 0 { settleTimer.Stop() settleTicker.Stop() break settling } case <-settleTimer.C: settleTicker.Stop() break settling } } } if delivery.report().Missing > 0 { reconcileCtx, cancelReconcile := context.WithTimeout(ctx, cfg.OperationTimeout*2) reconcileDeliveries(reconcileCtx, workers) cancelReconcile() } // Take the authoritative business-work cutoff before canceling clients. // Coordinated teardown can cancel an RPC already in flight; both server // outcome counters and client operation counters must therefore stop before // that cancellation begins. FinalServerMetrics remains the post-recovery // resource-reclamation snapshot. var workloadEndServerMetrics map[string]float64 if serverMetrics != nil { if sample, scrapeErr := serverMetrics.scrape(ctx); scrapeErr == nil { workloadEndServerMetrics = sample finalServerMetrics = sample events.write(map[string]any{"type": "server_workload_end", "at": time.Now().UTC(), "server_metrics": sample}) } else { events.write(map[string]any{"type": "server_workload_end_error", "at": time.Now().UTC(), "class": classifyError(scrapeErr)}) } } workloadEndOperations := metrics.freeze() finalReady := countWorkerState(workers, workerReady) stopLoad() workerWG.Wait() if finalReady > peakReady { peakReady = finalReady } if cfg.RecoveryDuration > 0 { recoveryDeadline := time.NewTimer(cfg.RecoveryDuration) recoveryTicker := time.NewTicker(cfg.SampleInterval) for { select { case <-ctx.Done(): recoveryTicker.Stop() recoveryDeadline.Stop() return nil, ctx.Err() case <-recoveryTicker.C: finalServerMetrics = writeSample(ctx, events, "recovery", workers, metrics, counters, serverMetrics) case <-recoveryDeadline.C: recoveryTicker.Stop() goto recoveryFinished } } } recoveryFinished: if serverMetrics != nil { if sample, scrapeErr := serverMetrics.scrape(ctx); scrapeErr == nil { finalServerMetrics = sample } else { finalServerMetrics = nil } } steadyRatio := float64(0) if steadySamples > 0 { steadyRatio = float64(steadyReadySum) / float64(steadySamples*len(workers)) } report := &RunReport{ Version: RunReportVersion, StartedAt: startedAt, LoadEndedAt: loadEndedAt, FinishedAt: time.Now().UTC(), StartOrder: startOrder, StartOrderSeed: startOrderSeed, RequestedDuration: cfg.Duration.String(), RecoveryDuration: cfg.RecoveryDuration.String(), ExpectedSessions: len(workers), PeakReadySessions: peakReady, FinalReadySessions: finalReady, ConnectionAttempts: counters.connectionAttempts.Load(), Reconnects: counters.reconnects.Load(), Disconnects: counters.disconnects.Load(), UpdatesReceived: counters.updates.Load(), DownloadedBytes: counters.downloadBytes.Load(), WorkerFatalErrors: counters.fatalErrors.Load(), Operations: workloadEndOperations, BaselineServerMetrics: baselineServerMetrics, WorkloadEndServerMetrics: workloadEndServerMetrics, FinalServerMetrics: finalServerMetrics, ServerMetricsScrapes: serverMetrics.successes(), ServerMetricsErrors: serverMetrics.failures(), SteadySamples: steadySamples, SteadyReadyRatio: steadyRatio, MinSteadyReadySessions: steadyReadyMinimum, MessageRatePerSecond: cfg.MessageRate, MessageScheduled: counters.messageScheduled.Load(), MessageEnqueued: counters.messageEnqueued.Load(), MessageCompleted: counters.messageCompleted.Load(), MessageQueueFull: counters.messageQueueFull.Load(), MessageNotReady: counters.messageNotReady.Load(), Delivery: delivery.report(), } report.ResponseBytes = startupResponseBytes(baselineServerMetrics, workloadEndServerMetrics) report.RPCDeliveryOutcomes = startupRPCDeliveryOutcomes(baselineServerMetrics, workloadEndServerMetrics) report.DatabaseWork = startupDatabaseWork(baselineServerMetrics, workloadEndServerMetrics) report.EventsWritten, report.EventsDropped = events.counts() evaluateReport(report, cfg) if err := WriteReport(cfg.ReportPath, report); err != nil { return nil, err } return report, nil } func primaryTargets(records []SessionRecord) []SessionRecord { byAccount := make(map[int]SessionRecord) for _, record := range records { if existing, ok := byAccount[record.AccountIndex]; !ok || record.DeviceIndex < existing.DeviceIndex { byAccount[record.AccountIndex] = record } } maxAccount := -1 for account := range byAccount { maxAccount = max(maxAccount, account) } targets := make([]SessionRecord, maxAccount+1) for account, record := range byAccount { targets[account] = record } return targets } func newLoadRunID() (string, error) { var value [8]byte if _, err := cryptorand.Read(value[:]); err != nil { return "", err } return hex.EncodeToString(value[:]), nil } func primaryWorkers(workers []*loadWorker) []*loadWorker { primary := make([]*loadWorker, 0, len(workers)) for _, worker := range workers { if worker.record.DeviceIndex == 0 && worker.target.UserID > 0 { primary = append(primary, worker) } } return primary } func runFixedMessageSchedule(ctx context.Context, wg *sync.WaitGroup, startDelay time.Duration, rate float64, workers []*loadWorker, counters *harnessCounters, events *eventWriter) { defer wg.Done() if rate <= 0 || len(workers) == 0 { return } startTimer := time.NewTimer(startDelay) defer startTimer.Stop() select { case <-ctx.Done(): return case <-startTimer.C: } readyTicker := time.NewTicker(10 * time.Millisecond) for countWorkerState(workers, workerReady) != len(workers) { select { case <-ctx.Done(): readyTicker.Stop() return case <-readyTicker.C: } } readyTicker.Stop() interval := time.Duration(float64(time.Second) / rate) if interval < time.Microsecond { interval = time.Microsecond } events.write(map[string]any{"type": "fixed_message_rate_start", "at": time.Now().UTC(), "rate_per_second": rate, "senders": len(workers)}) next := time.Now() workerIndex := 0 timer := time.NewTimer(0) defer timer.Stop() for { select { case <-ctx.Done(): events.write(map[string]any{ "type": "fixed_message_rate_stop", "at": time.Now().UTC(), "scheduled": counters.messageScheduled.Load(), "enqueued": counters.messageEnqueued.Load(), "queue_full": counters.messageQueueFull.Load(), "not_ready": counters.messageNotReady.Load(), }) return case <-timer.C: worker := workers[workerIndex] workerIndex = (workerIndex + 1) % len(workers) counters.messageScheduled.Add(1) if worker.state.Load() != workerReady { counters.messageNotReady.Add(1) } else { select { case worker.sendQueue <- struct{}{}: counters.messageEnqueued.Add(1) default: counters.messageQueueFull.Add(1) } } next = next.Add(interval) timer.Reset(max(time.Until(next), time.Duration(0))) } } } func reconcileDeliveries(ctx context.Context, workers []*loadWorker) { waits := make([]chan struct{}, 0, len(workers)) for _, worker := range workers { if worker.state.Load() != workerReady { continue } done := make(chan struct{}) select { case <-ctx.Done(): return case worker.reconcile <- done: waits = append(waits, done) } } for _, done := range waits { select { case <-ctx.Done(): return case <-done: } } } func waitMessageDrain(ctx context.Context, counters *harnessCounters) { ticker := time.NewTicker(10 * time.Millisecond) defer ticker.Stop() for counters.messageCompleted.Load() < counters.messageEnqueued.Load() { select { case <-ctx.Done(): return case <-ticker.C: } } } func minimumOpenFiles(sessions int) int { if sessions < 0 { sessions = 0 } // gotd may transiently hold primary, PFS and replacement sockets together; // retain fixed room for the process, resolver, event/report files and scrapes. return 256 + sessions*6 } func runOfflineWindow(ctx context.Context, workers []*loadWorker, fraction float64, at, duration time.Duration, events *eventWriter) { timer := time.NewTimer(at) defer timer.Stop() select { case <-ctx.Done(): return case <-timer.C: } count := int(math.Ceil(float64(len(workers)) * fraction)) selected := make([]*loadWorker, 0, count) for i, worker := range workers { if i%len(workers) < count { worker.setOnline(false) selected = append(selected, worker) } } events.write(map[string]any{"type": "offline_start", "at": time.Now().UTC(), "sessions": len(selected)}) timer.Reset(duration) select { case <-ctx.Done(): return case <-timer.C: } for _, worker := range selected { worker.setOnline(true) } events.write(map[string]any{"type": "offline_end", "at": time.Now().UTC(), "sessions": len(selected)}) } func countWorkerState(workers []*loadWorker, state int32) int { count := 0 for _, worker := range workers { if worker.state.Load() == state { count++ } } return count } func writeSample(ctx context.Context, events *eventWriter, phase string, workers []*loadWorker, metrics *metricSet, counters *harnessCounters, server *serverMetricsClient) map[string]float64 { var mem runtime.MemStats runtime.ReadMemStats(&mem) serverValues, scrapeErr := server.scrape(ctx) value := map[string]any{ "type": "sample", "phase": phase, "at": time.Now().UTC(), "workers": map[string]int{ "connecting": countWorkerState(workers, workerConnecting), "ready": countWorkerState(workers, workerReady), "disconnected": countWorkerState(workers, workerDisconnected), "offline": countWorkerState(workers, workerOffline), "stopped": countWorkerState(workers, workerStopped), }, "client_runtime": map[string]uint64{"goroutines": uint64(runtime.NumGoroutine()), "heap_alloc_bytes": mem.HeapAlloc, "sys_bytes": mem.Sys}, "connections": map[string]uint64{ "attempts": counters.connectionAttempts.Load(), "reconnects": counters.reconnects.Load(), "disconnects": counters.disconnects.Load(), "fatal_errors": counters.fatalErrors.Load(), "downloaded_bytes": counters.downloadBytes.Load(), }, "operations": metrics.report(), "server_metrics": serverValues, } if scrapeErr != nil { value["server_metrics_error"] = classifyError(scrapeErr) } events.write(value) return serverValues } func evaluateReport(report *RunReport, cfg RunConfig) { requiredReady := int(math.Ceil(float64(report.ExpectedSessions) * cfg.MinimumReadyRatio)) if report.PeakReadySessions < requiredReady { report.Failures = append(report.Failures, fmt.Sprintf("peak ready sessions %d below required %d", report.PeakReadySessions, requiredReady)) } if report.SteadySamples == 0 { report.Failures = append(report.Failures, "no post-ramp steady-state samples were collected") } else if report.SteadyReadyRatio < cfg.MinimumReadyRatio { report.Failures = append(report.Failures, fmt.Sprintf("steady ready ratio %.4f below required %.4f", report.SteadyReadyRatio, cfg.MinimumReadyRatio)) } if report.WorkerFatalErrors > 0 { report.Failures = append(report.Failures, fmt.Sprintf("worker fatal errors: %d", report.WorkerFatalErrors)) } if cfg.MessageRate > 0 { if report.MessageScheduled == 0 { report.Failures = append(report.Failures, "fixed-rate message scheduler produced no arrivals") } if report.MessageNotReady > 0 { report.Failures = append(report.Failures, fmt.Sprintf("fixed-rate arrivals rejected because sender was not ready: %d", report.MessageNotReady)) } if report.MessageQueueFull > 0 { report.Failures = append(report.Failures, fmt.Sprintf("fixed-rate arrivals rejected by bounded sender queues: %d", report.MessageQueueFull)) } if report.MessageEnqueued != report.MessageScheduled { report.Failures = append(report.Failures, fmt.Sprintf("fixed-rate scheduler enqueued %d of %d arrivals", report.MessageEnqueued, report.MessageScheduled)) } if report.MessageCompleted != report.MessageEnqueued { report.Failures = append(report.Failures, fmt.Sprintf("message workers completed %d of %d enqueued sends", report.MessageCompleted, report.MessageEnqueued)) } sendOperation := report.Operations["messages.sendMessage"] if sendOperation.Count != report.MessageCompleted { report.Failures = append(report.Failures, fmt.Sprintf("messages.sendMessage recorded %d of %d completed jobs", sendOperation.Count, report.MessageCompleted)) } successfulSends := sendOperation.Count - min(sendOperation.Count, sendOperation.Errors+sendOperation.Canceled) if report.Delivery.Expected != successfulSends { report.Failures = append(report.Failures, fmt.Sprintf("delivery tracker committed %d of %d successful send RPCs", report.Delivery.Expected, successfulSends)) } if report.Delivery.Missing > 0 || report.Delivery.Delivered != report.Delivery.Expected { report.Failures = append(report.Failures, fmt.Sprintf("recipient delivery incomplete: delivered %d of %d, missing %d", report.Delivery.Delivered, report.Delivery.Expected, report.Delivery.Missing)) } if cfg.OfflineFraction == 0 && report.Delivery.DifferenceRecovered > 0 { report.Failures = append(report.Failures, fmt.Sprintf("online recipients recovered %d messages only through updates.getDifference", report.Delivery.DifferenceRecovered)) } if report.Delivery.DuplicateObservations > 0 { report.Failures = append(report.Failures, fmt.Sprintf("recipient observed %d duplicate message updates", report.Delivery.DuplicateObservations)) } if report.Delivery.WrongAccountObserved > 0 { report.Failures = append(report.Failures, fmt.Sprintf("load markers appeared on %d wrong recipient accounts", report.Delivery.WrongAccountObserved)) } if report.Delivery.UnmatchedMarkers > 0 { report.Failures = append(report.Failures, fmt.Sprintf("observed %d load markers without a successful send RPC", report.Delivery.UnmatchedMarkers)) } } for name, operation := range report.Operations { if operation.FloodWaits > 0 { report.Failures = append(report.Failures, fmt.Sprintf("%s returned FLOOD_WAIT %d times", name, operation.FloodWaits)) } unexpectedErrors := operation.Errors if cfg.ExpectServerRestart { unexpectedErrors -= min(unexpectedErrors, operation.ConnectionErrors) } if unexpectedErrors > 0 { report.Failures = append(report.Failures, fmt.Sprintf("%s returned %d unexpected non-cancel errors", name, unexpectedErrors)) } } methods := make([]string, 0, len(report.RPCDeliveryOutcomes)) for method := range report.RPCDeliveryOutcomes { methods = append(methods, method) } sort.Strings(methods) for _, method := range methods { outcomes := report.RPCDeliveryOutcomes[method] outcomeNames := make([]string, 0, len(outcomes)) for outcome := range outcomes { outcomeNames = append(outcomeNames, outcome) } sort.Strings(outcomeNames) for _, outcome := range outcomeNames { if count := outcomes[outcome]; outcome != "ok" && count > 0 { report.Failures = append(report.Failures, fmt.Sprintf("%s rpc_result delivery outcome %s: %d", method, outcome, count)) } } } methods = methods[:0] for method := range report.DatabaseWork { methods = append(methods, method) } sort.Strings(methods) for _, method := range methods { if errors := report.DatabaseWork[method].Errors; errors > 0 { report.Failures = append(report.Failures, fmt.Sprintf("%s database errors: %d", method, errors)) } } if cfg.ExpectServerRestart && report.Reconnects < uint64(requiredReady) { report.Failures = append(report.Failures, fmt.Sprintf("server restart expected at least %d reconnect attempts, observed %d", requiredReady, report.Reconnects)) } if report.FinalServerMetrics != nil && report.BaselineServerMetrics != nil && cfg.RecoveryDuration > 0 { checks := []string{ "telesrv_mtproto_raw_connections", "telesrv_mtproto_logical_sessions", "telesrv_mtproto_logical_outbox_bytes", "telesrv_mtproto_pending_push_bytes", "telesrv_mtproto_outbound_tracked_bytes", "telesrv_mtproto_rpc_execution_owners", "telesrv_mtproto_rpc_execution_reserved_entries", "telesrv_mtproto_rpc_execution_receipts", "telesrv_mtproto_rpc_execution_receipt_budget_bytes", "telesrv_mtproto_rpc_execution_subscribers", } for _, name := range checks { baseline := metricValue(report.BaselineServerMetrics, name) final := metricValue(report.FinalServerMetrics, name) if final > baseline { report.Failures = append(report.Failures, fmt.Sprintf("server retained %.0f above baseline %.0f in %s after recovery", final-baseline, baseline, name)) } } } if strings.TrimSpace(cfg.ServerMetricsURL) != "" && report.ServerMetricsScrapes == 0 { report.Failures = append(report.Failures, "server metrics endpoint produced no successful scrapes") } if strings.TrimSpace(cfg.ServerMetricsURL) != "" && report.FinalServerMetrics == nil { report.Failures = append(report.Failures, "final post-recovery server metrics scrape failed") } if strings.TrimSpace(cfg.ServerMetricsURL) != "" && report.WorkloadEndServerMetrics == nil { report.Failures = append(report.Failures, "pre-teardown workload-end server metrics scrape failed") } if strings.TrimSpace(cfg.ServerMetricsURL) != "" && report.BaselineServerMetrics == nil { report.Failures = append(report.Failures, "pre-load server metrics baseline scrape failed") } report.Pass = len(report.Failures) == 0 } func metricValue(values map[string]float64, name string) float64 { // The scraper always stores an aggregate family value in the bare key and // may additionally retain bounded state/method label breakdowns. Prefer that // aggregate; summing both would double-count every labeled family in resource // recovery checks (for example retained/offline logical sessions). if value, ok := values[name]; ok { return value } var total float64 for key, value := range values { if strings.HasPrefix(key, name+"{") { total += value } } return total } func classifyError(err error) string { if err == nil { return "ok" } if errors.Is(err, context.Canceled) { return "canceled" } if errors.Is(err, context.DeadlineExceeded) { return "timeout" } if errors.Is(err, mtproto.ErrPFSReconnectRequired) || errors.Is(err, mtproto.ErrPFSDropKeysRequired) || errors.Is(err, mtproto.ErrTransportNotReady) { return "connection" } if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) || errors.Is(err, pool.ErrConnDead) || errors.Is(err, tdrpc.ErrEngineClosed) || errors.Is(err, syscall.ECONNABORTED) || errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EHOSTUNREACH) || errors.Is(err, syscall.ENETUNREACH) || errors.Is(err, syscall.EPIPE) { return "connection" } var netErr net.Error if errors.As(err, &netErr) { if netErr.Timeout() { return "timeout" } return "connection" } message := strings.ToUpper(err.Error()) switch { case strings.Contains(message, "FLOOD_WAIT"): return "flood_wait" case strings.Contains(message, "ENCRYPTED_MESSAGE_INVALID"): return "encrypted_message_invalid" case strings.Contains(message, "AUTH_KEY"): return "auth_key" case strings.Contains(message, "CONNECTION"), strings.Contains(message, "CONNECT"), strings.Contains(message, "EOF"), strings.Contains(message, "ENGINE WAS CLOSED"), strings.Contains(message, "BROKEN PIPE"), strings.Contains(message, "CLOSED NETWORK"), strings.Contains(message, "NO ROUTE TO HOST"), strings.Contains(message, "NETWORK IS UNREACHABLE"): return "connection" default: return "error" } } // classifyErrorReason intentionally returns a finite vocabulary. It preserves // enough transport/PFS evidence to diagnose a failed load without persisting // raw error strings, addresses, auth-key IDs or request payloads. func classifyErrorReason(err error) string { if err == nil { return "ok" } message := strings.ToUpper(err.Error()) switch { case errors.Is(err, mtproto.ErrPFSDropKeysRequired): return "pfs_drop_keys" case errors.Is(err, mtproto.ErrPFSReconnectRequired): return "pfs_reconnect" case errors.Is(err, mtproto.ErrTransportNotReady): return "transport_not_ready" case strings.Contains(message, "TOO MANY OPEN FILES"): return "file_descriptor_limit" case strings.Contains(message, "PFS RECONNECT"): return "pfs_reconnect" case strings.Contains(message, "AUTH KEY NOT FOUND"), strings.Contains(message, "AUTH_KEY_NOT_FOUND"), strings.Contains(message, "PROTOCOL ERROR 404"): return "auth_key_not_found" case strings.Contains(message, "ENCRYPTED_MESSAGE_INVALID"): return "encrypted_message_invalid" case strings.Contains(message, "FINGERPRINT"): return "rsa_fingerprint" case strings.Contains(message, "CONNECTION REFUSED"): return "connection_refused" case strings.Contains(message, "CONNECTION RESET"): return "connection_reset" case strings.Contains(message, "NETWORK IS UNREACHABLE"), strings.Contains(message, "NO ROUTE TO HOST"): return "network_unreachable" case strings.Contains(message, "NO SUCH HOST"): return "dns" case strings.Contains(message, "BROKEN PIPE"): return "broken_pipe" case strings.Contains(message, "ENDED BEFORE BUSINESS READINESS"): return "business_readiness_incomplete" case strings.Contains(message, "EOF"): return "eof" case errors.Is(err, context.DeadlineExceeded): return "timeout" case errors.Is(err, context.Canceled): return "canceled" default: return classifyError(err) } }