merged from gramsrv upstream

This commit is contained in:
onysd 2026-09-01 12:06:31 +03:00
parent 79c64ee916
commit 21a0856587
651 changed files with 54774 additions and 4590 deletions

View file

@ -12,8 +12,12 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/iamxvbaba/td/exchange"
"github.com/iamxvbaba/td/mtproto"
"github.com/iamxvbaba/td/proto"
"github.com/iamxvbaba/td/telegram"
"github.com/iamxvbaba/td/telegram/dcs"
"github.com/iamxvbaba/td/tg"
@ -24,6 +28,40 @@ type clientHooks struct {
Update telegram.UpdateHandler
ConnectionState func(telegram.ConnectionState)
Dead func(error)
Device *telegram.DeviceConfig
}
// loadMessageIDSource keeps the load generator on the same MTProto message-id
// rules as a production client even when the host clock lands exactly on an
// integral second. The underlying gotd generator can emit a client id whose
// lower 32 bits are zero in that narrow window; Telegram explicitly forbids
// that value as replay protection. Retrying also fences any encoded duplicate
// caused by a very low-resolution clock without weakening the DUT validator.
type loadMessageIDSource struct {
mu sync.Mutex
source mtproto.MessageIDSource
last int64
}
func newLoadMessageIDSource(now func() time.Time) *loadMessageIDSource {
return &loadMessageIDSource{source: proto.NewMessageIDGen(now)}
}
func (s *loadMessageIDSource) New(messageType proto.MessageType) int64 {
s.mu.Lock()
defer s.mu.Unlock()
for {
messageID := s.source.New(messageType)
if messageID <= s.last {
continue
}
if messageType == proto.MessageFromClient && uint32(messageID) == 0 {
continue
}
s.last = messageID
return messageID
}
}
func newClient(endpoint Endpoint, publicKey *rsa.PublicKey, storage telegram.SessionStorage, hooks clientHooks) (*telegram.Client, error) {
@ -44,6 +82,10 @@ func newClient(endpoint Endpoint, publicKey *rsa.PublicKey, storage telegram.Ses
if updateHandler == nil {
updateHandler = telegram.UpdateHandlerFunc(func(context.Context, tg.UpdatesClass) error { return nil })
}
device := telegram.DeviceTDesktopWindows()
if hooks.Device != nil {
device = *hooks.Device
}
return telegram.NewClient(endpoint.APIID, endpoint.APIHash, telegram.Options{
PublicKeys: []exchange.PublicKey{{RSA: publicKey}},
DC: endpoint.DC,
@ -55,7 +97,8 @@ func newClient(endpoint Endpoint, publicKey *rsa.PublicKey, storage telegram.Ses
UpdateHandler: updateHandler,
EnablePFS: endpoint.PFS,
TempKeyTTL: endpoint.TempKeyTTL,
Device: telegram.DeviceTDesktopWindows(),
Device: device,
MessageID: newLoadMessageIDSource(time.Now),
OnConnectionState: hooks.ConnectionState,
OnDead: hooks.Dead,
}), nil

View file

@ -0,0 +1,60 @@
package loadharness
import (
"sync"
"testing"
"time"
"github.com/iamxvbaba/td/proto"
)
func TestLoadMessageIDSourceSkipsEmptyClientFraction(t *testing.T) {
second := time.Unix(1_800_000_000, 0)
source := newLoadMessageIDSource(func() time.Time { return second })
first := source.New(proto.MessageFromClient)
secondID := source.New(proto.MessageFromClient)
for index, messageID := range []int64{first, secondID} {
if uint32(messageID) == 0 {
t.Fatalf("message id %d lower 32 bits are empty", index)
}
if proto.MessageID(messageID).Type() != proto.MessageFromClient {
t.Fatalf("message id %d type = %v, want client", index, proto.MessageID(messageID).Type())
}
}
if secondID <= first {
t.Fatalf("message ids are not strictly increasing: %d then %d", first, secondID)
}
}
func TestLoadMessageIDSourceConcurrentMonotonicUnique(t *testing.T) {
second := time.Unix(1_800_000_000, 0)
source := newLoadMessageIDSource(func() time.Time { return second })
const calls = 1_000
ids := make(chan int64, calls)
var workers sync.WaitGroup
workers.Add(calls)
for range calls {
go func() {
defer workers.Done()
ids <- source.New(proto.MessageFromClient)
}()
}
workers.Wait()
close(ids)
seen := make(map[int64]struct{}, calls)
for messageID := range ids {
if uint32(messageID) == 0 || proto.MessageID(messageID).Type() != proto.MessageFromClient {
t.Fatalf("invalid client message id %d", messageID)
}
if _, duplicate := seen[messageID]; duplicate {
t.Fatalf("duplicate client message id %d", messageID)
}
seen[messageID] = struct{}{}
}
if len(seen) != calls {
t.Fatalf("unique message ids = %d, want %d", len(seen), calls)
}
}

View file

@ -0,0 +1,446 @@
package loadharness
import (
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"sort"
"strings"
"time"
)
const (
DatasetVersion = 1
maxDatasetGroups = 10000
maxDatasetMembership = 2_000_000
maxDatasetMessages = 2_000_000
)
// DatasetConfig describes a deterministic social graph whose durable facts are
// later materialized exclusively through real MTProto RPCs. Planning does not
// contact the server and never embeds auth/session material.
type DatasetConfig struct {
Accounts int `json:"accounts"`
Seed int64 `json:"seed"`
PrivateFanout int `json:"private_fanout"`
HotGroups int `json:"hot_groups"`
HotMembers int `json:"hot_members"`
HotHistory int `json:"hot_history"`
MediumGroups int `json:"medium_groups"`
MediumMembers int `json:"medium_members"`
MediumHistory int `json:"medium_history"`
SmallGroups int `json:"small_groups"`
SmallMembers int `json:"small_members"`
SmallHistory int `json:"small_history"`
HeavyGroups int `json:"heavy_groups"`
HeavyAccounts int `json:"heavy_accounts"`
HeavyHistory int `json:"heavy_history"`
}
func DefaultDatasetConfig(accounts int) DatasetConfig {
return DatasetConfig{
Accounts: accounts, Seed: 20260827, PrivateFanout: min(10, max(accounts-1, 0)),
HotGroups: 10, HotMembers: accounts, HotHistory: 100,
MediumGroups: 100, MediumMembers: min(100, accounts), MediumHistory: 30,
SmallGroups: 200, SmallMembers: min(20, accounts), SmallHistory: 10,
HeavyGroups: 200, HeavyAccounts: min(100, accounts), HeavyHistory: 30,
}
}
type DatasetPrivateEdge struct {
SenderAccount int `json:"sender_account"`
RecipientAccount int `json:"recipient_account"`
RandomID int64 `json:"random_id"`
Marker string `json:"marker"`
}
type DatasetGroup struct {
Index int `json:"index"`
Tier string `json:"tier"`
Title string `json:"title"`
About string `json:"about"`
CreatorAccount int `json:"creator_account"`
MemberAccounts []int `json:"member_accounts"`
HistoryMessages int `json:"history_messages"`
}
// Dataset is the immutable plan. Real server identities and resumable progress
// live in the separate compact DatasetSeedState so a large plan is not rewritten
// after every reconciled RPC batch.
type Dataset struct {
Version int `json:"version"`
CreatedAt time.Time `json:"created_at"`
RunID string `json:"run_id"`
Config DatasetConfig `json:"config"`
PlanSHA256 string `json:"plan_sha256"`
PrivateEdges []DatasetPrivateEdge `json:"private_edges"`
Groups []DatasetGroup `json:"groups"`
}
type DatasetSeedGroupState struct {
GroupIndex int `json:"group_index"`
ChannelID int64 `json:"channel_id,omitempty"`
AccessHash int64 `json:"access_hash,omitempty"`
CreatePending bool `json:"create_pending,omitempty"`
InviteCursor int `json:"invite_cursor,omitempty"`
InvitePendingEnd int `json:"invite_pending_end,omitempty"`
}
type DatasetSeedState struct {
Version int `json:"version"`
PlanSHA256 string `json:"plan_sha256"`
UpdatedAt time.Time `json:"updated_at"`
PrivateSentByAccount []int `json:"private_sent_by_account"`
HistorySentByAccount []int `json:"history_sent_by_account"`
RichStateByAccount []bool `json:"rich_state_by_account,omitempty"`
Groups []DatasetSeedGroupState `json:"groups"`
}
func PlanDataset(cfg DatasetConfig) (*Dataset, error) {
if err := cfg.validate(); err != nil {
return nil, err
}
runID := fmt.Sprintf("rpc-startup-%d-%016x", cfg.Accounts, uint64(cfg.Seed))
dataset := &Dataset{
Version: DatasetVersion, CreatedAt: time.Now().UTC(),
RunID: runID, Config: cfg,
PrivateEdges: make([]DatasetPrivateEdge, 0, cfg.Accounts*cfg.PrivateFanout),
}
for account := 0; account < cfg.Accounts; account++ {
for offset := 1; offset <= cfg.PrivateFanout; offset++ {
recipient := (account + offset) % cfg.Accounts
dataset.PrivateEdges = append(dataset.PrivateEdges, DatasetPrivateEdge{
SenderAccount: account, RecipientAccount: recipient,
RandomID: stableDatasetID(cfg.Seed, "private", cfg.Accounts, account, offset),
Marker: fmt.Sprintf("[%s private %04d/%02d]", runID, account, offset),
})
}
}
groupIndex := 0
appendTier := func(tier string, count, members, history int, memberSet func(int) []int) {
for i := 0; i < count; i++ {
groupMembers := memberSet(i)
creator := groupMembers[i%len(groupMembers)]
dataset.Groups = append(dataset.Groups, DatasetGroup{
Index: groupIndex, Tier: tier,
Title: fmt.Sprintf("%s %s %04d", runID, tier, i+1),
About: fmt.Sprintf("telesrv real-RPC load dataset %s group %d", tier, i+1),
CreatorAccount: creator, MemberAccounts: groupMembers, HistoryMessages: history,
})
groupIndex++
}
}
appendTier("hot", cfg.HotGroups, cfg.HotMembers, cfg.HotHistory, func(i int) []int {
return cyclicMembers(cfg.Accounts, i*max(cfg.HotMembers, 1), cfg.HotMembers)
})
appendTier("medium", cfg.MediumGroups, cfg.MediumMembers, cfg.MediumHistory, func(i int) []int {
return cyclicMembers(cfg.Accounts, i*max(cfg.MediumMembers, 1), cfg.MediumMembers)
})
appendTier("small", cfg.SmallGroups, cfg.SmallMembers, cfg.SmallHistory, func(i int) []int {
return cyclicMembers(cfg.Accounts, i*max(cfg.SmallMembers, 1), cfg.SmallMembers)
})
appendTier("heavy", cfg.HeavyGroups, cfg.HeavyAccounts, cfg.HeavyHistory, func(int) []int {
members := make([]int, cfg.HeavyAccounts)
for i := range members {
members[i] = i
}
return members
})
planHash, err := dataset.planHash()
if err != nil {
return nil, err
}
dataset.PlanSHA256 = planHash
if err := dataset.Validate(); err != nil {
return nil, err
}
return dataset, nil
}
func (c DatasetConfig) validate() error {
if c.Accounts < 2 || c.Accounts > 100000 {
return errors.New("dataset accounts must be between 2 and 100000")
}
if c.PrivateFanout < 0 || c.PrivateFanout >= c.Accounts {
return errors.New("private fanout must be non-negative and smaller than accounts")
}
groups := c.HotGroups + c.MediumGroups + c.SmallGroups + c.HeavyGroups
if groups <= 0 || groups > maxDatasetGroups {
return fmt.Errorf("dataset group count must be between 1 and %d", maxDatasetGroups)
}
tiers := []struct {
name string
groups, members, history int
}{
{"hot", c.HotGroups, c.HotMembers, c.HotHistory},
{"medium", c.MediumGroups, c.MediumMembers, c.MediumHistory},
{"small", c.SmallGroups, c.SmallMembers, c.SmallHistory},
{"heavy", c.HeavyGroups, c.HeavyAccounts, c.HeavyHistory},
}
memberships := 0
messages := c.Accounts * c.PrivateFanout
for _, tier := range tiers {
if tier.groups < 0 || tier.members < 0 || tier.members > c.Accounts || tier.history < 0 {
return fmt.Errorf("invalid %s dataset tier", tier.name)
}
if tier.groups > 0 && tier.members == 0 {
return fmt.Errorf("%s dataset tier has groups without members", tier.name)
}
memberships += tier.groups * tier.members
messages += tier.groups * tier.history
}
if memberships > maxDatasetMembership {
return fmt.Errorf("dataset memberships %d exceed hard limit %d", memberships, maxDatasetMembership)
}
if messages > maxDatasetMessages {
return fmt.Errorf("dataset messages %d exceed hard limit %d", messages, maxDatasetMessages)
}
return nil
}
func (d *Dataset) Validate() error {
if d == nil || d.Version != DatasetVersion {
return errors.New("invalid dataset version")
}
if strings.TrimSpace(d.RunID) == "" || strings.TrimSpace(d.PlanSHA256) == "" {
return errors.New("dataset is missing run id or plan hash")
}
if err := d.Config.validate(); err != nil {
return err
}
if len(d.PrivateEdges) != d.Config.Accounts*d.Config.PrivateFanout {
return errors.New("dataset private graph size does not match config")
}
seenGroups := make(map[int]struct{}, len(d.Groups))
for _, group := range d.Groups {
if group.Index < 0 || group.CreatorAccount < 0 || group.CreatorAccount >= d.Config.Accounts || group.HistoryMessages < 0 {
return fmt.Errorf("invalid group %d", group.Index)
}
if _, exists := seenGroups[group.Index]; exists {
return fmt.Errorf("duplicate group index %d", group.Index)
}
seenGroups[group.Index] = struct{}{}
members := make(map[int]struct{}, len(group.MemberAccounts))
for _, account := range group.MemberAccounts {
if account < 0 || account >= d.Config.Accounts {
return fmt.Errorf("group %d has invalid account %d", group.Index, account)
}
if _, exists := members[account]; exists {
return fmt.Errorf("group %d has duplicate account %d", group.Index, account)
}
members[account] = struct{}{}
}
if _, ok := members[group.CreatorAccount]; !ok {
return fmt.Errorf("group %d creator is not a member", group.Index)
}
}
wantHash, err := d.planHash()
if err != nil {
return err
}
if wantHash != d.PlanSHA256 {
return errors.New("dataset immutable plan hash mismatch")
}
return nil
}
func WriteDataset(path string, dataset *Dataset) error {
if err := dataset.Validate(); err != nil {
return err
}
data, err := json.MarshalIndent(dataset, "", " ")
if err != nil {
return fmt.Errorf("encode dataset: %w", err)
}
return writeFileAtomic(path, append(data, '\n'), 0o600)
}
func NewDatasetSeedState(dataset *Dataset) (*DatasetSeedState, error) {
if err := dataset.Validate(); err != nil {
return nil, err
}
state := &DatasetSeedState{
Version: DatasetVersion, PlanSHA256: dataset.PlanSHA256, UpdatedAt: time.Now().UTC(),
PrivateSentByAccount: make([]int, dataset.Config.Accounts),
HistorySentByAccount: make([]int, dataset.Config.Accounts),
RichStateByAccount: make([]bool, dataset.Config.Accounts),
Groups: make([]DatasetSeedGroupState, len(dataset.Groups)),
}
for i, group := range dataset.Groups {
state.Groups[i].GroupIndex = group.Index
}
return state, nil
}
func (s *DatasetSeedState) Validate(dataset *Dataset) error {
if s == nil || s.Version != DatasetVersion || dataset == nil || s.PlanSHA256 != dataset.PlanSHA256 {
return errors.New("seed state does not match dataset plan")
}
if len(s.PrivateSentByAccount) != dataset.Config.Accounts || len(s.HistorySentByAccount) != dataset.Config.Accounts || len(s.Groups) != len(dataset.Groups) {
return errors.New("seed state dimensions do not match dataset plan")
}
// A nil rich-state vector is accepted only for seed journals created before
// the comprehensive startup workload added this phase. New journals always
// allocate the full vector and therefore cannot silently skip it.
if len(s.RichStateByAccount) != 0 && len(s.RichStateByAccount) != dataset.Config.Accounts {
return errors.New("seed rich-state dimensions do not match dataset plan")
}
historyTasks := datasetHistoryTaskCounts(dataset)
for account := 0; account < dataset.Config.Accounts; account++ {
if s.PrivateSentByAccount[account] < 0 || s.PrivateSentByAccount[account] > dataset.Config.PrivateFanout {
return fmt.Errorf("invalid private seed progress for account %d", account)
}
if s.HistorySentByAccount[account] < 0 || s.HistorySentByAccount[account] > historyTasks[account] {
return fmt.Errorf("invalid history seed progress for account %d", account)
}
}
for i, groupState := range s.Groups {
group := dataset.Groups[i]
invitees := len(group.MemberAccounts) - 1
if groupState.GroupIndex != group.Index || groupState.InviteCursor < 0 || groupState.InviteCursor > invitees {
return fmt.Errorf("invalid seed state for group %d", group.Index)
}
if (groupState.ChannelID == 0) != (groupState.AccessHash == 0) {
return fmt.Errorf("group %d has partial channel identity", group.Index)
}
if groupState.ChannelID != 0 && groupState.CreatePending {
return fmt.Errorf("group %d has both channel identity and pending create", group.Index)
}
if groupState.InvitePendingEnd < groupState.InviteCursor || groupState.InvitePendingEnd > invitees {
return fmt.Errorf("group %d has invalid pending invite range", group.Index)
}
if groupState.ChannelID == 0 && (groupState.InviteCursor != 0 || groupState.InvitePendingEnd != 0) {
return fmt.Errorf("group %d has invite progress without a channel identity", group.Index)
}
}
return nil
}
func WriteDatasetSeedState(path string, dataset *Dataset, state *DatasetSeedState) error {
if err := state.Validate(dataset); err != nil {
return err
}
state.UpdatedAt = time.Now().UTC()
data, err := json.MarshalIndent(state, "", " ")
if err != nil {
return fmt.Errorf("encode dataset seed state: %w", err)
}
return writeFileAtomic(path, append(data, '\n'), 0o600)
}
func LoadDatasetSeedState(path string, dataset *Dataset) (*DatasetSeedState, error) {
data, err := os.ReadFile(path)
if os.IsNotExist(err) {
return NewDatasetSeedState(dataset)
}
if err != nil {
return nil, fmt.Errorf("read dataset seed state: %w", err)
}
var state DatasetSeedState
decoder := json.NewDecoder(strings.NewReader(string(data)))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&state); err != nil {
return nil, fmt.Errorf("decode dataset seed state: %w", err)
}
if err := state.Validate(dataset); err != nil {
return nil, err
}
return &state, nil
}
func datasetHistoryTaskCounts(dataset *Dataset) []int {
counts := make([]int, dataset.Config.Accounts)
for _, group := range dataset.Groups {
for message := 0; message < group.HistoryMessages; message++ {
counts[group.MemberAccounts[message%len(group.MemberAccounts)]]++
}
}
return counts
}
func LoadDataset(path string) (*Dataset, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read dataset: %w", err)
}
var dataset Dataset
decoder := json.NewDecoder(strings.NewReader(string(data)))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&dataset); err != nil {
return nil, fmt.Errorf("decode dataset: %w", err)
}
if err := dataset.Validate(); err != nil {
return nil, err
}
return &dataset, nil
}
func (d *Dataset) planHash() (string, error) {
type immutableDataset struct {
Version int `json:"version"`
RunID string `json:"run_id"`
Config DatasetConfig `json:"config"`
PrivateEdges []DatasetPrivateEdge `json:"private_edges"`
Groups []struct {
Index int `json:"index"`
Tier string `json:"tier"`
Title string `json:"title"`
About string `json:"about"`
CreatorAccount int `json:"creator_account"`
MemberAccounts []int `json:"member_accounts"`
HistoryMessages int `json:"history_messages"`
} `json:"groups"`
}
immutable := immutableDataset{Version: d.Version, RunID: d.RunID, Config: d.Config, PrivateEdges: d.PrivateEdges}
for _, group := range d.Groups {
immutable.Groups = append(immutable.Groups, struct {
Index int `json:"index"`
Tier string `json:"tier"`
Title string `json:"title"`
About string `json:"about"`
CreatorAccount int `json:"creator_account"`
MemberAccounts []int `json:"member_accounts"`
HistoryMessages int `json:"history_messages"`
}{group.Index, group.Tier, group.Title, group.About, group.CreatorAccount, group.MemberAccounts, group.HistoryMessages})
}
data, err := json.Marshal(immutable)
if err != nil {
return "", err
}
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:]), nil
}
func cyclicMembers(accounts, start, count int) []int {
if count <= 0 {
return nil
}
members := make([]int, count)
for i := range members {
members[i] = (start + i) % accounts
}
sort.Ints(members)
return members
}
func stableDatasetID(seed int64, namespace string, values ...int) int64 {
h := sha256.New()
var buf [8]byte
binary.LittleEndian.PutUint64(buf[:], uint64(seed))
_, _ = h.Write(buf[:])
_, _ = h.Write([]byte(namespace))
for _, value := range values {
binary.LittleEndian.PutUint64(buf[:], uint64(value))
_, _ = h.Write(buf[:])
}
sum := h.Sum(nil)
id := int64(binary.LittleEndian.Uint64(sum[:8]) & ^(uint64(1) << 63))
if id == 0 {
return 1
}
return id
}

View file

@ -0,0 +1,179 @@
package loadharness
import (
"path/filepath"
"testing"
)
func TestPlanDatasetDefaultTopology(t *testing.T) {
dataset, err := PlanDataset(DefaultDatasetConfig(1000))
if err != nil {
t.Fatal(err)
}
if got, want := len(dataset.PrivateEdges), 10000; got != want {
t.Fatalf("private edges = %d, want %d", got, want)
}
if got, want := len(dataset.Groups), 510; got != want {
t.Fatalf("groups = %d, want %d", got, want)
}
memberships := make([]int, dataset.Config.Accounts)
totalMemberships := 0
for _, group := range dataset.Groups {
totalMemberships += len(group.MemberAccounts)
for _, account := range group.MemberAccounts {
memberships[account]++
}
}
if totalMemberships != 44000 {
t.Fatalf("memberships = %d, want 44000", totalMemberships)
}
if got, want := memberships[999]+20, 44; got != want {
t.Fatalf("regular account dialogs = %d, want %d", got, want)
}
if got, want := memberships[0]+20, 244; got != want {
t.Fatalf("heavy account dialogs = %d, want %d", got, want)
}
if dataset.PlanSHA256 == "" || dataset.PrivateEdges[0].RandomID == 0 {
t.Fatal("dataset is missing stable identity")
}
}
func TestPlanDatasetIsDeterministic(t *testing.T) {
cfg := DefaultDatasetConfig(100)
first, err := PlanDataset(cfg)
if err != nil {
t.Fatal(err)
}
second, err := PlanDataset(cfg)
if err != nil {
t.Fatal(err)
}
if first.PlanSHA256 != second.PlanSHA256 {
t.Fatalf("plan hash changed: %s != %s", first.PlanSHA256, second.PlanSHA256)
}
if first.PrivateEdges[37] != second.PrivateEdges[37] {
t.Fatal("private edge plan changed")
}
}
func TestDatasetRandomIDsAreNamespacedByScale(t *testing.T) {
ten, err := PlanDataset(DefaultDatasetConfig(10))
if err != nil {
t.Fatal(err)
}
hundred, err := PlanDataset(DefaultDatasetConfig(100))
if err != nil {
t.Fatal(err)
}
if ten.PrivateEdges[0].RandomID == hundred.PrivateEdges[0].RandomID {
t.Fatal("private random_id collided across dataset scales")
}
if stableDatasetID(7, "offline", 10, 0) == stableDatasetID(7, "offline", 100, 0) {
t.Fatal("offline random_id collided across dataset scales")
}
}
func TestDatasetRoundTripAndImmutableHash(t *testing.T) {
dataset, err := PlanDataset(DefaultDatasetConfig(20))
if err != nil {
t.Fatal(err)
}
path := filepath.Join(t.TempDir(), "dataset.json")
if err := WriteDataset(path, dataset); err != nil {
t.Fatal(err)
}
loaded, err := LoadDataset(path)
if err != nil {
t.Fatal(err)
}
if loaded.PlanSHA256 != dataset.PlanSHA256 || len(loaded.Groups) != len(dataset.Groups) {
t.Fatal("dataset round trip changed the plan")
}
loaded.Groups[0].Title += " changed"
if err := loaded.Validate(); err == nil {
t.Fatal("mutated immutable plan passed validation")
}
}
func TestDatasetSeedStateRoundTrip(t *testing.T) {
dataset, err := PlanDataset(DefaultDatasetConfig(20))
if err != nil {
t.Fatal(err)
}
state, err := NewDatasetSeedState(dataset)
if err != nil {
t.Fatal(err)
}
state.PrivateSentByAccount[0] = 2
state.Groups[0].ChannelID = 11
state.Groups[0].AccessHash = 22
state.Groups[0].InviteCursor = 3
state.Groups[0].InvitePendingEnd = 7
path := filepath.Join(t.TempDir(), "seed-state.json")
if err := WriteDatasetSeedState(path, dataset, state); err != nil {
t.Fatal(err)
}
loaded, err := LoadDatasetSeedState(path, dataset)
if err != nil {
t.Fatal(err)
}
if loaded.PrivateSentByAccount[0] != 2 || loaded.Groups[0].ChannelID != 11 || loaded.Groups[0].InviteCursor != 3 || loaded.Groups[0].InvitePendingEnd != 7 {
t.Fatal("seed state round trip changed progress")
}
}
func TestDatasetSeedStateAcceptsLegacyMissingRichVector(t *testing.T) {
dataset, err := PlanDataset(DefaultDatasetConfig(20))
if err != nil {
t.Fatal(err)
}
state, err := NewDatasetSeedState(dataset)
if err != nil {
t.Fatal(err)
}
state.RichStateByAccount = nil
if err := state.Validate(dataset); err != nil {
t.Fatal(err)
}
state.RichStateByAccount = []bool{true}
if err := state.Validate(dataset); err == nil {
t.Fatal("partial rich-state vector passed validation")
}
}
func TestDatasetSeedStateRejectsImpossiblePendingOperation(t *testing.T) {
dataset, err := PlanDataset(DefaultDatasetConfig(20))
if err != nil {
t.Fatal(err)
}
state, err := NewDatasetSeedState(dataset)
if err != nil {
t.Fatal(err)
}
state.Groups[0].CreatePending = true
state.Groups[0].ChannelID = 11
state.Groups[0].AccessHash = 22
if err := state.Validate(dataset); err == nil {
t.Fatal("channel identity plus pending create passed validation")
}
state.Groups[0].CreatePending = false
state.Groups[0].ChannelID = 0
state.Groups[0].AccessHash = 0
state.Groups[0].InvitePendingEnd = 1
if err := state.Validate(dataset); err == nil {
t.Fatal("invite progress without channel identity passed validation")
}
}
func TestDatasetConfigRejectsUnsafeScale(t *testing.T) {
cfg := DefaultDatasetConfig(1000)
cfg.HotGroups = maxDatasetGroups + 1
if _, err := PlanDataset(cfg); err == nil {
t.Fatal("oversized dataset passed validation")
}
cfg = DefaultDatasetConfig(10)
cfg.PrivateFanout = 10
if _, err := PlanDataset(cfg); err == nil {
t.Fatal("self-wrapping private fanout passed validation")
}
}

View file

@ -0,0 +1,230 @@
package loadharness
import (
"fmt"
"math"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/iamxvbaba/td/tg"
)
const deliveryMarkerPrefix = "telesrv-load-v3"
type deliverySource uint8
const (
deliveryLive deliverySource = 1 << iota
deliveryDifference
)
type deliveryExpectation struct {
senderUserID int64
targetUserID int64
startedAt time.Time
committed bool
}
type deliveryObservation struct {
sources deliverySource
repeats uint64
firstAt time.Time
}
type deliveryTracker struct {
mu sync.Mutex
runID string
expected map[string]deliveryExpectation
observations map[string]map[int64]deliveryObservation
}
type DeliveryReport struct {
RunID string `json:"run_id"`
Expected uint64 `json:"expected"`
Delivered uint64 `json:"delivered"`
Missing uint64 `json:"missing"`
LiveDelivered uint64 `json:"live_delivered"`
DifferenceRecovered uint64 `json:"difference_recovered"`
DuplicateObservations uint64 `json:"duplicate_observations"`
WrongAccountObserved uint64 `json:"wrong_account_observed"`
UnmatchedMarkers uint64 `json:"unmatched_markers"`
E2EP50MS float64 `json:"e2e_p50_ms"`
E2EP95MS float64 `json:"e2e_p95_ms"`
E2EP99MS float64 `json:"e2e_p99_ms"`
E2EMaxMS float64 `json:"e2e_max_ms"`
}
func newDeliveryTracker(runID string) *deliveryTracker {
return &deliveryTracker{
runID: runID,
expected: make(map[string]deliveryExpectation),
observations: make(map[string]map[int64]deliveryObservation),
}
}
func (t *deliveryTracker) marker(senderIndex int, sequence uint64) string {
return fmt.Sprintf("%s/%s/%d/%d", deliveryMarkerPrefix, t.runID, senderIndex, sequence)
}
func (t *deliveryTracker) expect(marker string, senderUserID, targetUserID int64) {
t.begin(marker, senderUserID, targetUserID, time.Now())
t.finish(marker, true)
}
func (t *deliveryTracker) begin(marker string, senderUserID, targetUserID int64, startedAt time.Time) {
if t == nil || !t.matches(marker) {
return
}
t.mu.Lock()
t.expected[marker] = deliveryExpectation{senderUserID: senderUserID, targetUserID: targetUserID, startedAt: startedAt}
t.mu.Unlock()
}
func (t *deliveryTracker) finish(marker string, success bool) {
if t == nil {
return
}
t.mu.Lock()
expectation, ok := t.expected[marker]
if ok && success {
expectation.committed = true
t.expected[marker] = expectation
} else if ok {
delete(t.expected, marker)
}
t.mu.Unlock()
}
func (t *deliveryTracker) observe(marker string, accountUserID int64, source deliverySource) {
if t == nil || accountUserID <= 0 || !t.matches(marker) {
return
}
t.mu.Lock()
byAccount := t.observations[marker]
if byAccount == nil {
byAccount = make(map[int64]deliveryObservation)
t.observations[marker] = byAccount
}
observation := byAccount[accountUserID]
if observation.firstAt.IsZero() {
observation.firstAt = time.Now()
}
if observation.sources&source != 0 {
observation.repeats++
}
observation.sources |= source
byAccount[accountUserID] = observation
t.mu.Unlock()
}
func (t *deliveryTracker) matches(marker string) bool {
parts := strings.Split(marker, "/")
if len(parts) != 4 || parts[0] != deliveryMarkerPrefix || parts[1] != t.runID {
return false
}
if _, err := strconv.Atoi(parts[2]); err != nil {
return false
}
sequence, err := strconv.ParseUint(parts[3], 10, 64)
return err == nil && sequence > 0
}
func (t *deliveryTracker) report() DeliveryReport {
t.mu.Lock()
defer t.mu.Unlock()
report := DeliveryReport{RunID: t.runID}
latencies := make([]time.Duration, 0, len(t.expected))
for marker, expectation := range t.expected {
if !expectation.committed {
continue
}
report.Expected++
byAccount := t.observations[marker]
observation, delivered := byAccount[expectation.targetUserID]
if delivered {
report.Delivered++
switch {
case observation.sources&deliveryLive != 0:
report.LiveDelivered++
case observation.sources&deliveryDifference != 0:
report.DifferenceRecovered++
}
report.DuplicateObservations += observation.repeats
if !expectation.startedAt.IsZero() && !observation.firstAt.Before(expectation.startedAt) {
latencies = append(latencies, observation.firstAt.Sub(expectation.startedAt))
}
} else {
report.Missing++
}
for accountUserID, other := range byAccount {
if accountUserID == expectation.targetUserID || accountUserID == expectation.senderUserID {
continue
}
report.WrongAccountObserved++
report.DuplicateObservations += other.repeats
}
}
for marker := range t.observations {
if expectation, ok := t.expected[marker]; !ok || !expectation.committed {
report.UnmatchedMarkers++
}
}
if len(latencies) > 0 {
report.E2EP50MS = deliveryQuantileMS(latencies, 0.50)
report.E2EP95MS = deliveryQuantileMS(latencies, 0.95)
report.E2EP99MS = deliveryQuantileMS(latencies, 0.99)
report.E2EMaxMS = deliveryQuantileMS(latencies, 1)
}
return report
}
func deliveryQuantileMS(values []time.Duration, quantile float64) float64 {
sorted := append([]time.Duration(nil), values...)
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
index := int(math.Ceil(float64(len(sorted))*quantile)) - 1
index = max(0, min(index, len(sorted)-1))
return durationMS(sorted[index])
}
func observeUpdatesClass(tracker *deliveryTracker, accountUserID int64, updates tg.UpdatesClass, source deliverySource) {
switch value := updates.(type) {
case *tg.Updates:
observeUpdateClasses(tracker, accountUserID, value.Updates, source)
case *tg.UpdatesCombined:
observeUpdateClasses(tracker, accountUserID, value.Updates, source)
case *tg.UpdateShort:
observeUpdateClass(tracker, accountUserID, value.Update, source)
case *tg.UpdateShortMessage:
tracker.observe(value.Message, accountUserID, source)
}
}
func observeUpdateClasses(tracker *deliveryTracker, accountUserID int64, updates []tg.UpdateClass, source deliverySource) {
for _, update := range updates {
observeUpdateClass(tracker, accountUserID, update, source)
}
}
func observeUpdateClass(tracker *deliveryTracker, accountUserID int64, update tg.UpdateClass, source deliverySource) {
switch value := update.(type) {
case *tg.UpdateNewMessage:
observeMessageClass(tracker, accountUserID, value.Message, source)
case *tg.UpdateNewChannelMessage:
observeMessageClass(tracker, accountUserID, value.Message, source)
}
}
func observeMessageClasses(tracker *deliveryTracker, accountUserID int64, messages []tg.MessageClass, source deliverySource) {
for _, message := range messages {
observeMessageClass(tracker, accountUserID, message, source)
}
}
func observeMessageClass(tracker *deliveryTracker, accountUserID int64, message tg.MessageClass, source deliverySource) {
if value, ok := message.(*tg.Message); ok {
tracker.observe(value.Message, accountUserID, source)
}
}

View file

@ -0,0 +1,83 @@
package loadharness
import (
"testing"
"time"
"github.com/iamxvbaba/td/tg"
)
func TestDeliveryTrackerReconcilesObservationBeforeSendReturn(t *testing.T) {
tracker := newDeliveryTracker("run")
marker := tracker.marker(7, 9)
tracker.observe(marker, 200, deliveryLive)
tracker.expect(marker, 100, 200)
report := tracker.report()
if report.Expected != 1 || report.Delivered != 1 || report.LiveDelivered != 1 || report.Missing != 0 {
t.Fatalf("unexpected report: %+v", report)
}
}
func TestDeliveryTrackerMeasuresFromSendStart(t *testing.T) {
tracker := newDeliveryTracker("run")
marker := tracker.marker(1, 1)
tracker.begin(marker, 100, 200, time.Now().Add(-25*time.Millisecond))
tracker.observe(marker, 200, deliveryLive)
tracker.finish(marker, true)
report := tracker.report()
if report.E2EP50MS < 20 || report.E2EP99MS < report.E2EP50MS || report.E2EMaxMS < report.E2EP99MS {
t.Fatalf("unexpected e2e latency report: %+v", report)
}
}
func TestDeliveryTrackerSeparatesDifferenceRecoveryAndDuplicates(t *testing.T) {
tracker := newDeliveryTracker("run")
liveMarker := tracker.marker(1, 1)
differenceMarker := tracker.marker(2, 1)
tracker.expect(liveMarker, 100, 200)
tracker.expect(differenceMarker, 200, 300)
tracker.observe(liveMarker, 200, deliveryLive)
tracker.observe(liveMarker, 200, deliveryLive)
tracker.observe(liveMarker, 200, deliveryDifference)
tracker.observe(differenceMarker, 300, deliveryDifference)
report := tracker.report()
if report.Delivered != 2 || report.LiveDelivered != 1 || report.DifferenceRecovered != 1 {
t.Fatalf("unexpected delivery sources: %+v", report)
}
if report.DuplicateObservations != 1 {
t.Fatalf("duplicate observations = %d, want 1", report.DuplicateObservations)
}
}
func TestObserveUpdatesClassExtractsPrivateMessages(t *testing.T) {
tracker := newDeliveryTracker("run")
shortMarker := tracker.marker(1, 1)
fullMarker := tracker.marker(2, 1)
tracker.expect(shortMarker, 100, 200)
tracker.expect(fullMarker, 300, 200)
observeUpdatesClass(tracker, 200, &tg.UpdateShortMessage{Message: shortMarker}, deliveryLive)
observeUpdatesClass(tracker, 200, &tg.Updates{Updates: []tg.UpdateClass{
&tg.UpdateNewMessage{Message: &tg.Message{Message: fullMarker}},
}}, deliveryLive)
report := tracker.report()
if report.Delivered != 2 || report.Missing != 0 {
t.Fatalf("unexpected report: %+v", report)
}
}
func TestDeliveryTrackerRejectsForeignAndMalformedMarkers(t *testing.T) {
tracker := newDeliveryTracker("run")
tracker.observe("telesrv-load-v3/other/1/1", 200, deliveryLive)
tracker.observe("telesrv-load-v3/run/not-an-index/1", 200, deliveryLive)
if report := tracker.report(); report.UnmatchedMarkers != 0 {
t.Fatalf("foreign marker entered report: %+v", report)
}
}

View file

@ -0,0 +1,778 @@
package loadharness
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"os"
"sort"
"strings"
"sync"
"time"
"github.com/iamxvbaba/td/tg"
"github.com/iamxvbaba/td/tgerr"
)
const OfflineMutationVersion = 1
type OfflineMutationChannelPlan struct {
GroupPosition int
Messages int
}
type OfflineMutationChannelState struct {
GroupIndex int `json:"group_index"`
MessageIDs []int `json:"message_ids"`
LatestPts int `json:"latest_pts,omitempty"`
EditPending bool `json:"edit_pending,omitempty"`
EditDone bool `json:"edit_done,omitempty"`
DeletePending bool `json:"delete_pending,omitempty"`
DeleteDone bool `json:"delete_done,omitempty"`
PinPending bool `json:"pin_pending,omitempty"`
PinDone bool `json:"pin_done,omitempty"`
}
type OfflineMutationState struct {
Version int `json:"version"`
DatasetSHA256 string `json:"dataset_sha256"`
SeedIdentitySHA string `json:"seed_identity_sha256"`
BaselineStateSHA string `json:"baseline_state_sha256"`
UpdatedAt time.Time `json:"updated_at"`
PrivateMessageIDs []int `json:"private_message_ids"`
AccountObservedPts []int `json:"account_observed_pts"`
Channels []OfflineMutationChannelState `json:"channels"`
}
type MutateOfflineConfig struct {
ManifestPath string
SessionKeyPath string
RSAKeyOverride string
DatasetPath string
SeedStatePath string
ClientStatePath string
MutationStatePath string
Concurrency int
OperationTimeout time.Duration
}
type MutationEvent struct {
Phase string
Completed int
Total int
Account int
Err error
}
type MutationResult struct {
PrivateMessages int
ChannelMessages int
DirtyChannels int
Edited int
Deleted int
Pinned int
}
func (c MutateOfflineConfig) validate() error {
if c.ManifestPath == "" || c.SessionKeyPath == "" || c.DatasetPath == "" || c.SeedStatePath == "" || c.ClientStatePath == "" || c.MutationStatePath == "" {
return errors.New("manifest, session-key, dataset, seed-state, client-state and mutation-state paths are required")
}
if c.Concurrency <= 0 || c.Concurrency > 64 {
return errors.New("offline mutation concurrency must be between 1 and 64")
}
if c.OperationTimeout <= 0 {
return errors.New("offline mutation operation timeout must be positive")
}
return nil
}
// MutateOffline creates gaps only after a complete baseline has been locked.
// Message sends use stable random_id values. Mutable channel operations use a
// pending journal and public read-back reconciliation before a resumed run can
// declare them complete.
func MutateOffline(ctx context.Context, cfg MutateOfflineConfig, progress func(MutationEvent)) (*MutationResult, error) {
if err := cfg.validate(); err != nil {
return nil, err
}
manifest, err := LoadManifest(cfg.ManifestPath)
if err != nil {
return nil, err
}
dataset, err := LoadDataset(cfg.DatasetPath)
if err != nil {
return nil, err
}
targets, err := seedPrimaryTargets(manifest, dataset.Config.Accounts)
if err != nil {
return nil, err
}
seedState, err := LoadDatasetSeedState(cfg.SeedStatePath, dataset)
if err != nil {
return nil, err
}
seedJournal := &seedJournal{dataset: dataset, state: seedState}
if err := seedJournal.assertComplete(); err != nil {
return nil, fmt.Errorf("offline mutation requires a complete seed: %w", err)
}
clientState, err := LoadClientState(cfg.ClientStatePath)
if err != nil {
return nil, err
}
if err := clientState.Validate(dataset, seedState, targets); err != nil {
return nil, err
}
baselineSHA, err := fileSHA256(cfg.ClientStatePath)
if err != nil {
return nil, err
}
seedIdentity, err := seedIdentitySHA256(seedState)
if err != nil {
return nil, err
}
plan := planOfflineMutation(dataset)
state, err := loadOrCreateOfflineMutationState(cfg.MutationStatePath, dataset, seedIdentity, baselineSHA, plan)
if err != nil {
return nil, err
}
journal := &mutationJournal{path: cfg.MutationStatePath, dataset: dataset, plan: plan, state: state}
if err := journal.persist(); err != nil {
return nil, err
}
key, err := LoadSessionKey(cfg.SessionKeyPath)
if err != nil {
return nil, err
}
publicKey, err := loadManifestPublicKey(cfg.ManifestPath, manifest.Endpoint, cfg.RSAKeyOverride)
if err != nil {
return nil, err
}
accounts := make([]int, dataset.Config.Accounts)
for account := range accounts {
accounts[account] = account
}
if err := runSeedAccountPhase(ctx, "mutate-private", accounts, cfg.Concurrency, mutationProgressAdapter("private", progress), func(ctx context.Context, account int) error {
if journal.privateMessageID(account) != 0 {
return nil
}
return withAuthorizedSeedSession(ctx, SeedConfig{ManifestPath: cfg.ManifestPath, OperationTimeout: cfg.OperationTimeout}, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
recipient := (account + 1) % dataset.Config.Accounts
marker := offlinePrivateMarker(dataset, account, recipient)
updates, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
return raw.MessagesSendMessage(rpcCtx, &tg.MessagesSendMessageRequest{
Peer: &tg.InputPeerUser{UserID: targets[recipient].UserID, AccessHash: targets[recipient].AccessHash},
Message: marker, RandomID: stableDatasetID(dataset.Config.Seed, "offline-private", dataset.Config.Accounts, account, recipient),
})
})
if err != nil {
return fmt.Errorf("messages.sendMessage: %w", err)
}
observation, err := sentMessageObservation(updates, clientPeerKey{typ: "user", id: targets[recipient].UserID}, marker)
if err != nil {
return err
}
return journal.commitPrivate(account, observation.ID, observation.Pts)
})
}); err != nil {
return nil, err
}
channelTasks := offlineChannelTasks(dataset, plan)
channelAccounts := make([]int, 0, len(channelTasks))
for account := range channelTasks {
channelAccounts = append(channelAccounts, account)
}
sort.Ints(channelAccounts)
if err := runSeedAccountPhase(ctx, "mutate-channel", channelAccounts, cfg.Concurrency, mutationProgressAdapter("channel", progress), func(ctx context.Context, account int) error {
return withAuthorizedSeedSession(ctx, SeedConfig{ManifestPath: cfg.ManifestPath, OperationTimeout: cfg.OperationTimeout}, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
for _, task := range channelTasks[account] {
if journal.channelMessageID(task.PlanPosition, task.MessageIndex) != 0 {
continue
}
channelPlan := plan[task.PlanPosition]
group := dataset.Groups[channelPlan.GroupPosition]
channel := seedState.Groups[channelPlan.GroupPosition]
marker := offlineChannelMarker(dataset, group, task.MessageIndex)
updates, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
return raw.MessagesSendMessage(rpcCtx, &tg.MessagesSendMessageRequest{
Peer: &tg.InputPeerChannel{ChannelID: channel.ChannelID, AccessHash: channel.AccessHash},
Message: marker, RandomID: stableDatasetID(dataset.Config.Seed, "offline-channel", dataset.Config.Accounts, group.Index, task.MessageIndex),
})
})
if err != nil {
return fmt.Errorf("group %d message %d: %w", group.Index, task.MessageIndex, err)
}
observation, err := sentMessageObservation(updates, clientPeerKey{typ: "channel", id: channel.ChannelID}, marker)
if err != nil {
return fmt.Errorf("group %d message %d: %w", group.Index, task.MessageIndex, err)
}
if err := journal.commitChannelMessage(task.PlanPosition, task.MessageIndex, observation.ID, observation.Pts); err != nil {
return err
}
}
return nil
})
}); err != nil {
return nil, err
}
// The first dirty channel deliberately exceeds the page limit and owns the
// edit, delete and pin events. The creator authors the edited message and can
// administratively delete/pin the two newest messages in one session.
if len(plan) == 0 || plan[0].Messages < 120 {
return nil, errors.New("offline mutation plan has no multi-page channel")
}
actionGroup := dataset.Groups[plan[0].GroupPosition]
if err := runSeedAccountPhase(ctx, "mutate-actions", []int{actionGroup.CreatorAccount}, 1, mutationProgressAdapter("actions", progress), func(ctx context.Context, account int) error {
return withAuthorizedSeedSession(ctx, SeedConfig{ManifestPath: cfg.ManifestPath, OperationTimeout: cfg.OperationTimeout}, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
return applyOfflineChannelActions(ctx, cfg.OperationTimeout, dataset, seedState, plan, journal, raw)
})
}); err != nil {
return nil, err
}
if err := journal.assertComplete(); err != nil {
return nil, err
}
return offlineMutationResult(plan, state), nil
}
func planOfflineMutation(dataset *Dataset) []OfflineMutationChannelPlan {
plan := make([]OfflineMutationChannelPlan, 0, 40)
counts := map[string]int{"hot": 10, "medium": 10, "small": 10, "heavy": 10}
messages := map[string]int{"hot": 3, "medium": 2, "small": 1, "heavy": 3}
seen := make(map[string]int)
for position, group := range dataset.Groups {
if seen[group.Tier] >= counts[group.Tier] {
continue
}
count := messages[group.Tier]
switch len(plan) {
case 0:
count = 120
case 1:
// Exactly one full page complements the first channel's >limit
// channelDifferenceTooLong snapshot path.
count = 100
}
plan = append(plan, OfflineMutationChannelPlan{GroupPosition: position, Messages: count})
seen[group.Tier]++
}
if len(plan) == 0 && len(dataset.Groups) != 0 {
plan = append(plan, OfflineMutationChannelPlan{GroupPosition: 0, Messages: 120})
}
return plan
}
type offlineChannelTask struct {
PlanPosition int
MessageIndex int
}
func offlineChannelTasks(dataset *Dataset, plan []OfflineMutationChannelPlan) map[int][]offlineChannelTask {
tasks := make(map[int][]offlineChannelTask)
for planPosition, channelPlan := range plan {
group := dataset.Groups[channelPlan.GroupPosition]
for message := 0; message < channelPlan.Messages; message++ {
account := offlineMutationSender(group, message)
tasks[account] = append(tasks[account], offlineChannelTask{PlanPosition: planPosition, MessageIndex: message})
}
}
return tasks
}
func offlineMutationSender(group DatasetGroup, message int) int {
if message < 3 {
return group.CreatorAccount
}
return group.MemberAccounts[message%len(group.MemberAccounts)]
}
func offlinePrivateMarker(dataset *Dataset, sender, recipient int) string {
return fmt.Sprintf("[%s offline private %04d->%04d]", dataset.RunID, sender, recipient)
}
func offlineChannelMarker(dataset *Dataset, group DatasetGroup, message int) string {
return fmt.Sprintf("[%s offline channel %04d message %04d]", dataset.RunID, group.Index, message+1)
}
type messageObservation struct {
ID int
Pts int
}
func sentMessageObservation(updates tg.UpdatesClass, peer clientPeerKey, marker string) (messageObservation, error) {
switch value := updates.(type) {
case *tg.UpdateShortSentMessage:
if value.ID > 0 {
return messageObservation{ID: value.ID, Pts: value.Pts}, nil
}
case *tg.UpdateShortMessage:
if value.ID > 0 && value.Message == marker {
return messageObservation{ID: value.ID, Pts: value.Pts}, nil
}
case *tg.Updates:
return sentMessageObservationFromUpdates(value.Updates, peer, marker)
case *tg.UpdatesCombined:
return sentMessageObservationFromUpdates(value.Updates, peer, marker)
}
return messageObservation{}, fmt.Errorf("messages.sendMessage returned %T without marker", updates)
}
func sentMessageObservationFromUpdates(updates []tg.UpdateClass, peer clientPeerKey, marker string) (messageObservation, error) {
for _, update := range updates {
var message tg.MessageClass
pts := 0
switch value := update.(type) {
case *tg.UpdateNewMessage:
message, pts = value.Message, value.Pts
case *tg.UpdateNewChannelMessage:
message, pts = value.Message, value.Pts
default:
continue
}
full, ok := message.(*tg.Message)
if !ok || full.Message != marker {
continue
}
messagePeer, ok := clientPeerFromTG(full.PeerID)
if ok && messagePeer == peer && full.ID > 0 {
return messageObservation{ID: full.ID, Pts: pts}, nil
}
}
return messageObservation{}, errors.New("messages.sendMessage updates omitted expected marker")
}
func applyOfflineChannelActions(
ctx context.Context,
timeout time.Duration,
dataset *Dataset,
seedState *DatasetSeedState,
plan []OfflineMutationChannelPlan,
journal *mutationJournal,
raw *tg.Client,
) error {
channelPlan := plan[0]
group := dataset.Groups[channelPlan.GroupPosition]
channel := seedState.Groups[channelPlan.GroupPosition]
state := journal.channel(0)
deleteIndex, pinIndex := channelPlan.Messages-2, channelPlan.Messages-1
if state.MessageIDs[0] == 0 || state.MessageIDs[deleteIndex] == 0 || state.MessageIDs[pinIndex] == 0 {
return errors.New("channel action messages are incomplete")
}
peer := &tg.InputPeerChannel{ChannelID: channel.ChannelID, AccessHash: channel.AccessHash}
inputChannel := &tg.InputChannel{ChannelID: channel.ChannelID, AccessHash: channel.AccessHash}
if !state.EditDone {
if !state.EditPending {
if err := journal.beginAction(0, "edit"); err != nil {
return err
}
}
editedMarker := offlineChannelMarker(dataset, group, 0) + " edited"
updates, err := rpcWithFloodWaitRetry(ctx, timeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
return raw.MessagesEditMessage(rpcCtx, &tg.MessagesEditMessageRequest{Peer: peer, ID: state.MessageIDs[0], Message: editedMarker})
})
pts := maxPtsFromUpdates(updates)
if err != nil {
if !tgerr.Is(err, "MESSAGE_NOT_MODIFIED") {
return fmt.Errorf("messages.editMessage pending reconciliation: %w", err)
}
matches, verifyErr := channelMessageMatches(ctx, timeout, raw, inputChannel, state.MessageIDs[0], editedMarker)
if verifyErr != nil || !matches {
return fmt.Errorf("reconcile messages.editMessage: matched=%v err=%w", matches, verifyErr)
}
}
if err := journal.commitAction(0, "edit", pts); err != nil {
return err
}
state = journal.channel(0)
}
if !state.DeleteDone {
if !state.DeletePending {
if err := journal.beginAction(0, "delete"); err != nil {
return err
}
}
affected, err := rpcWithFloodWaitRetry(ctx, timeout, func(rpcCtx context.Context) (*tg.MessagesAffectedMessages, error) {
return raw.ChannelsDeleteMessages(rpcCtx, &tg.ChannelsDeleteMessagesRequest{Channel: inputChannel, ID: []int{state.MessageIDs[deleteIndex]}})
})
pts := 0
if affected != nil {
pts = affected.Pts
}
if err != nil {
deleted, verifyErr := channelMessageDeleted(ctx, timeout, raw, inputChannel, state.MessageIDs[deleteIndex])
if verifyErr != nil || !deleted {
return fmt.Errorf("reconcile channels.deleteMessages: deleted=%v err=%w (rpc %v)", deleted, verifyErr, err)
}
}
if err := journal.commitAction(0, "delete", pts); err != nil {
return err
}
state = journal.channel(0)
}
if !state.PinDone {
if !state.PinPending {
if err := journal.beginAction(0, "pin"); err != nil {
return err
}
}
updates, err := rpcWithFloodWaitRetry(ctx, timeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
return raw.MessagesUpdatePinnedMessage(rpcCtx, &tg.MessagesUpdatePinnedMessageRequest{Silent: true, Peer: peer, ID: state.MessageIDs[pinIndex]})
})
pts := maxPtsFromUpdates(updates)
if err != nil {
pinned, verifyErr := channelMessagePinned(ctx, timeout, raw, inputChannel, state.MessageIDs[pinIndex])
if verifyErr != nil || !pinned {
return fmt.Errorf("reconcile messages.updatePinnedMessage: pinned=%v err=%w (rpc %v)", pinned, verifyErr, err)
}
}
if err := journal.commitAction(0, "pin", pts); err != nil {
return err
}
}
return nil
}
func channelMessageMatches(ctx context.Context, timeout time.Duration, raw *tg.Client, channel *tg.InputChannel, messageID int, text string) (bool, error) {
messages, err := getChannelMessages(ctx, timeout, raw, channel, messageID)
if err != nil {
return false, err
}
for _, message := range messages {
if full, ok := message.(*tg.Message); ok && full.ID == messageID {
return full.Message == text, nil
}
}
return false, nil
}
func channelMessageDeleted(ctx context.Context, timeout time.Duration, raw *tg.Client, channel *tg.InputChannel, messageID int) (bool, error) {
messages, err := getChannelMessages(ctx, timeout, raw, channel, messageID)
if err != nil {
return false, err
}
for _, message := range messages {
if full, ok := message.(*tg.Message); ok && full.ID == messageID {
return false, nil
}
}
return true, nil
}
func getChannelMessages(ctx context.Context, timeout time.Duration, raw *tg.Client, channel *tg.InputChannel, messageID int) ([]tg.MessageClass, error) {
rpcCtx, cancel := context.WithTimeout(ctx, timeout)
response, err := raw.ChannelsGetMessages(rpcCtx, &tg.ChannelsGetMessagesRequest{
Channel: channel, ID: []tg.InputMessageClass{&tg.InputMessageID{ID: messageID}},
})
cancel()
if err != nil {
return nil, err
}
modified, ok := response.AsModified()
if !ok {
return nil, fmt.Errorf("channels.getMessages returned %T", response)
}
return modified.GetMessages(), nil
}
func channelMessagePinned(ctx context.Context, timeout time.Duration, raw *tg.Client, channel *tg.InputChannel, messageID int) (bool, error) {
rpcCtx, cancel := context.WithTimeout(ctx, timeout)
response, err := raw.ChannelsGetFullChannel(rpcCtx, channel)
cancel()
if err != nil {
return false, err
}
full, ok := response.FullChat.(*tg.ChannelFull)
if !ok {
return false, fmt.Errorf("channels.getFullChannel returned %T", response.FullChat)
}
pinned, ok := full.GetPinnedMsgID()
return ok && pinned == messageID, nil
}
func maxPtsFromUpdates(updates tg.UpdatesClass) int {
if updates == nil {
return 0
}
maxPts := 0
var classes []tg.UpdateClass
switch value := updates.(type) {
case *tg.UpdateShortSentMessage:
return value.Pts
case *tg.UpdateShortMessage:
return value.Pts
case *tg.Updates:
classes = value.Updates
case *tg.UpdatesCombined:
classes = value.Updates
}
for _, class := range classes {
switch value := class.(type) {
case *tg.UpdateNewMessage:
maxPts = max(maxPts, value.Pts)
case *tg.UpdateNewChannelMessage:
maxPts = max(maxPts, value.Pts)
case *tg.UpdateEditChannelMessage:
maxPts = max(maxPts, value.Pts)
case *tg.UpdateDeleteChannelMessages:
maxPts = max(maxPts, value.Pts)
case *tg.UpdatePinnedChannelMessages:
maxPts = max(maxPts, value.Pts)
}
}
return maxPts
}
func mutationProgressAdapter(phase string, progress func(MutationEvent)) func(SeedEvent) {
if progress == nil {
return nil
}
return func(event SeedEvent) {
progress(MutationEvent{Phase: phase, Completed: event.Completed, Total: event.Total, Account: event.Account, Err: event.Err})
}
}
func fileSHA256(path string) (string, error) {
data, err := os.ReadFile(path)
if err != nil {
return "", err
}
sum := sha256.Sum256(data)
return fmt.Sprintf("%x", sum[:]), nil
}
func loadOrCreateOfflineMutationState(
path string,
dataset *Dataset,
seedIdentity, baselineSHA string,
plan []OfflineMutationChannelPlan,
) (*OfflineMutationState, error) {
data, err := os.ReadFile(path)
if os.IsNotExist(err) {
state := &OfflineMutationState{
Version: OfflineMutationVersion, DatasetSHA256: dataset.PlanSHA256,
SeedIdentitySHA: seedIdentity, BaselineStateSHA: baselineSHA,
PrivateMessageIDs: make([]int, dataset.Config.Accounts), AccountObservedPts: make([]int, dataset.Config.Accounts),
Channels: make([]OfflineMutationChannelState, len(plan)),
}
for i, channelPlan := range plan {
state.Channels[i] = OfflineMutationChannelState{
GroupIndex: dataset.Groups[channelPlan.GroupPosition].Index,
MessageIDs: make([]int, channelPlan.Messages),
}
}
return state, nil
}
if err != nil {
return nil, err
}
var state OfflineMutationState
decoder := json.NewDecoder(strings.NewReader(string(data)))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&state); err != nil {
return nil, fmt.Errorf("decode offline mutation state: %w", err)
}
if err := state.Validate(dataset, seedIdentity, baselineSHA, plan); err != nil {
return nil, err
}
return &state, nil
}
func (s *OfflineMutationState) Validate(dataset *Dataset, seedIdentity, baselineSHA string, plan []OfflineMutationChannelPlan) error {
if s == nil || s.Version != OfflineMutationVersion || s.DatasetSHA256 != dataset.PlanSHA256 || s.SeedIdentitySHA != seedIdentity || s.BaselineStateSHA != baselineSHA {
return errors.New("offline mutation state does not match baseline dataset")
}
if len(s.PrivateMessageIDs) != dataset.Config.Accounts || len(s.AccountObservedPts) != dataset.Config.Accounts || len(s.Channels) != len(plan) {
return errors.New("offline mutation state dimensions do not match plan")
}
for account := range s.PrivateMessageIDs {
if s.PrivateMessageIDs[account] < 0 || s.AccountObservedPts[account] < 0 {
return fmt.Errorf("invalid offline private mutation account %d", account)
}
}
for i, channel := range s.Channels {
if channel.GroupIndex != dataset.Groups[plan[i].GroupPosition].Index || len(channel.MessageIDs) != plan[i].Messages || channel.LatestPts < 0 {
return fmt.Errorf("invalid offline mutation channel %d", channel.GroupIndex)
}
for _, messageID := range channel.MessageIDs {
if messageID < 0 {
return fmt.Errorf("offline mutation channel %d has invalid message id", channel.GroupIndex)
}
}
if (channel.EditDone && channel.EditPending) || (channel.DeleteDone && channel.DeletePending) || (channel.PinDone && channel.PinPending) {
return fmt.Errorf("offline mutation channel %d has completed pending action", channel.GroupIndex)
}
}
return nil
}
type mutationJournal struct {
mu sync.Mutex
path string
dataset *Dataset
plan []OfflineMutationChannelPlan
state *OfflineMutationState
}
func (j *mutationJournal) persistLocked() error {
if err := j.state.Validate(j.dataset, j.state.SeedIdentitySHA, j.state.BaselineStateSHA, j.plan); err != nil {
return err
}
j.state.UpdatedAt = time.Now().UTC()
data, err := json.MarshalIndent(j.state, "", " ")
if err != nil {
return err
}
return writeFileAtomic(j.path, append(data, '\n'), 0o600)
}
func (j *mutationJournal) persist() error {
j.mu.Lock()
defer j.mu.Unlock()
return j.persistLocked()
}
func (j *mutationJournal) privateMessageID(account int) int {
j.mu.Lock()
defer j.mu.Unlock()
return j.state.PrivateMessageIDs[account]
}
func (j *mutationJournal) channelMessageID(planPosition, message int) int {
j.mu.Lock()
defer j.mu.Unlock()
return j.state.Channels[planPosition].MessageIDs[message]
}
func (j *mutationJournal) channel(planPosition int) OfflineMutationChannelState {
j.mu.Lock()
defer j.mu.Unlock()
state := j.state.Channels[planPosition]
state.MessageIDs = append([]int(nil), state.MessageIDs...)
return state
}
func (j *mutationJournal) commitPrivate(account, messageID, pts int) error {
if messageID <= 0 || pts < 0 {
return errors.New("invalid private mutation observation")
}
j.mu.Lock()
defer j.mu.Unlock()
oldID, oldPts := j.state.PrivateMessageIDs[account], j.state.AccountObservedPts[account]
j.state.PrivateMessageIDs[account] = messageID
j.state.AccountObservedPts[account] = max(oldPts, pts)
if err := j.persistLocked(); err != nil {
j.state.PrivateMessageIDs[account], j.state.AccountObservedPts[account] = oldID, oldPts
return err
}
return nil
}
func (j *mutationJournal) commitChannelMessage(planPosition, message, messageID, pts int) error {
if messageID <= 0 || pts <= 0 {
return errors.New("invalid channel mutation observation")
}
j.mu.Lock()
defer j.mu.Unlock()
channel := &j.state.Channels[planPosition]
oldID, oldPts := channel.MessageIDs[message], channel.LatestPts
channel.MessageIDs[message] = messageID
channel.LatestPts = max(channel.LatestPts, pts)
if err := j.persistLocked(); err != nil {
channel.MessageIDs[message], channel.LatestPts = oldID, oldPts
return err
}
return nil
}
func (j *mutationJournal) beginAction(planPosition int, action string) error {
j.mu.Lock()
defer j.mu.Unlock()
channel := &j.state.Channels[planPosition]
old := *channel
switch action {
case "edit":
channel.EditPending = true
case "delete":
channel.DeletePending = true
case "pin":
channel.PinPending = true
default:
return fmt.Errorf("unknown mutation action %q", action)
}
if err := j.persistLocked(); err != nil {
*channel = old
return err
}
return nil
}
func (j *mutationJournal) commitAction(planPosition int, action string, pts int) error {
j.mu.Lock()
defer j.mu.Unlock()
channel := &j.state.Channels[planPosition]
old := *channel
switch action {
case "edit":
channel.EditPending, channel.EditDone = false, true
case "delete":
channel.DeletePending, channel.DeleteDone = false, true
case "pin":
channel.PinPending, channel.PinDone = false, true
default:
return fmt.Errorf("unknown mutation action %q", action)
}
channel.LatestPts = max(channel.LatestPts, pts)
if err := j.persistLocked(); err != nil {
*channel = old
return err
}
return nil
}
func (j *mutationJournal) assertComplete() error {
j.mu.Lock()
defer j.mu.Unlock()
if err := j.state.Validate(j.dataset, j.state.SeedIdentitySHA, j.state.BaselineStateSHA, j.plan); err != nil {
return err
}
for account, messageID := range j.state.PrivateMessageIDs {
if messageID <= 0 {
return fmt.Errorf("offline private mutation account %d is incomplete", account)
}
}
for i, channel := range j.state.Channels {
for message, messageID := range channel.MessageIDs {
if messageID <= 0 {
return fmt.Errorf("offline channel %d message %d is incomplete", channel.GroupIndex, message)
}
}
if channel.LatestPts <= 0 {
return fmt.Errorf("offline channel %d has no observed pts", channel.GroupIndex)
}
if i == 0 && (!channel.EditDone || !channel.DeleteDone || !channel.PinDone) {
return fmt.Errorf("offline channel %d actions are incomplete", channel.GroupIndex)
}
}
return nil
}
func offlineMutationResult(plan []OfflineMutationChannelPlan, state *OfflineMutationState) *MutationResult {
result := &MutationResult{PrivateMessages: len(state.PrivateMessageIDs), DirtyChannels: len(plan)}
for _, channel := range state.Channels {
result.ChannelMessages += len(channel.MessageIDs)
if channel.EditDone {
result.Edited++
}
if channel.DeleteDone {
result.Deleted++
}
if channel.PinDone {
result.Pinned++
}
}
return result
}

View file

@ -0,0 +1,123 @@
package loadharness
import (
"os"
"path/filepath"
"testing"
"github.com/iamxvbaba/td/tg"
)
func TestPlanOfflineMutationDefaultTopology(t *testing.T) {
dataset, err := PlanDataset(DefaultDatasetConfig(1000))
if err != nil {
t.Fatal(err)
}
plan := planOfflineMutation(dataset)
if got, want := len(plan), 40; got != want {
t.Fatalf("dirty channels = %d, want %d", got, want)
}
total := 0
for _, channel := range plan {
total += channel.Messages
}
if got, want := total, 304; got != want {
t.Fatalf("channel mutations = %d, want %d", got, want)
}
if plan[0].Messages != 120 || dataset.Groups[plan[0].GroupPosition].Tier != "hot" {
t.Fatalf("multi-page channel plan = %+v", plan[0])
}
if plan[1].Messages != 100 {
t.Fatalf("full-page boundary channel plan = %+v", plan[1])
}
group := dataset.Groups[plan[0].GroupPosition]
for message := 0; message < 3; message++ {
if got := offlineMutationSender(group, message); got != group.CreatorAccount {
t.Fatalf("action message %d sender = %d, want creator %d", message, got, group.CreatorAccount)
}
}
}
func TestSentMessageObservation(t *testing.T) {
peer := clientPeerKey{typ: "channel", id: 22}
marker := "marker"
observation, err := sentMessageObservation(&tg.Updates{Updates: []tg.UpdateClass{
&tg.UpdateNewChannelMessage{
Message: &tg.Message{ID: 7, PeerID: &tg.PeerChannel{ChannelID: 22}, Message: marker},
Pts: 12, PtsCount: 1,
},
}}, peer, marker)
if err != nil {
t.Fatal(err)
}
if observation.ID != 7 || observation.Pts != 12 {
t.Fatalf("observation = %+v", observation)
}
short, err := sentMessageObservation(&tg.UpdateShortSentMessage{ID: 8, Pts: 13}, clientPeerKey{typ: "user", id: 1}, "private")
if err != nil || short.ID != 8 || short.Pts != 13 {
t.Fatalf("short observation = %+v err=%v", short, err)
}
if _, err := sentMessageObservation(&tg.Updates{}, peer, marker); err == nil {
t.Fatal("updates without marker passed validation")
}
}
func TestOfflineMutationJournalPersistsMessagesAndActions(t *testing.T) {
dataset, seedState, _ := snapshotFixture(t)
seedIdentity, err := seedIdentitySHA256(seedState)
if err != nil {
t.Fatal(err)
}
plan := planOfflineMutation(dataset)
path := filepath.Join(t.TempDir(), "mutation-state.json")
state, err := loadOrCreateOfflineMutationState(path, dataset, seedIdentity, "baseline", plan)
if err != nil {
t.Fatal(err)
}
journal := &mutationJournal{path: path, dataset: dataset, plan: plan, state: state}
if err := journal.persist(); err != nil {
t.Fatal(err)
}
if err := journal.commitPrivate(0, 10, 20); err != nil {
t.Fatal(err)
}
if err := journal.commitChannelMessage(0, 0, 30, 40); err != nil {
t.Fatal(err)
}
if err := journal.beginAction(0, "edit"); err != nil {
t.Fatal(err)
}
if err := journal.commitAction(0, "edit", 41); err != nil {
t.Fatal(err)
}
loaded, err := loadOrCreateOfflineMutationState(path, dataset, seedIdentity, "baseline", plan)
if err != nil {
t.Fatal(err)
}
if loaded.PrivateMessageIDs[0] != 10 || loaded.AccountObservedPts[0] != 20 || loaded.Channels[0].MessageIDs[0] != 30 || loaded.Channels[0].LatestPts != 41 || !loaded.Channels[0].EditDone || loaded.Channels[0].EditPending {
t.Fatalf("persisted mutation state = %+v", loaded)
}
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0o600 {
t.Fatalf("mutation state mode = %o, want 600", info.Mode().Perm())
}
}
func TestOfflineMutationStateRejectsDifferentBaseline(t *testing.T) {
dataset, seedState, _ := snapshotFixture(t)
seedIdentity, err := seedIdentitySHA256(seedState)
if err != nil {
t.Fatal(err)
}
plan := planOfflineMutation(dataset)
state, err := loadOrCreateOfflineMutationState(filepath.Join(t.TempDir(), "missing.json"), dataset, seedIdentity, "baseline-a", plan)
if err != nil {
t.Fatal(err)
}
if err := state.Validate(dataset, seedIdentity, "baseline-b", plan); err == nil {
t.Fatal("mutation state accepted different baseline snapshot")
}
}

View file

@ -29,6 +29,10 @@ type ProvisionConfig struct {
FirstNamePrefix string
}
// DefaultPhonePrefix plus the six-digit account index produces the repository's
// structurally possible reserved NANP range (for example +1 555 000 0001).
const DefaultPhonePrefix = "+15550"
type ProvisionEvent struct {
Completed int
Total int

View file

@ -3,6 +3,8 @@ package loadharness
import (
"path/filepath"
"testing"
"telesrv/internal/domain"
)
func TestSessionDirectoryForManifestIsolatesNamedBundles(t *testing.T) {
@ -23,6 +25,14 @@ func TestSessionDirectoryForManifestIsolatesNamedBundles(t *testing.T) {
}
}
func TestDefaultProvisionPhonePrefixProducesCanonicalPossibleNumber(t *testing.T) {
cfg := ProvisionConfig{PhonePrefix: DefaultPhonePrefix, FirstNamePrefix: "Load"}
record := desiredSessionRecord(0, 0, 0, cfg)
if got, want := domain.NormalizePhone(record.Phone), "15550000001"; got != want {
t.Fatalf("normalized default phone = %q, want %q (wire %q)", got, want, record.Phone)
}
}
func TestDesiredSessionRecordUsesManifestNamespace(t *testing.T) {
cfg := ProvisionConfig{ManifestPath: filepath.Join("data", "manifest-500.json"), PhonePrefix: "+155500", FirstNamePrefix: "Load"}
record := desiredSessionRecord(12, 12, 1, cfg)

View file

@ -111,8 +111,9 @@ func durationMS(d time.Duration) float64 {
}
type metricSet struct {
mu sync.RWMutex
ops map[string]*operationMetrics
mu sync.RWMutex
ops map[string]*operationMetrics
frozen bool
}
func newMetricSet(names ...string) *metricSet {
@ -124,29 +125,56 @@ func newMetricSet(names ...string) *metricSet {
}
func (m *metricSet) observe(name string, start time.Time, err error) {
debugOperationError(name, err)
m.mu.RLock()
if m.frozen {
m.mu.RUnlock()
return
}
op := m.ops[name]
if op != nil {
debugOperationError(name, err)
op.observe(start, err)
m.mu.RUnlock()
return
}
m.mu.RUnlock()
if op == nil {
{
// Operation names are code-owned and finite, but retain a lock-protected
// fallback for optional scenarios added by the harness.
m.mu.Lock()
defer m.mu.Unlock()
if m.frozen {
return
}
op = m.ops[name]
if op == nil && len(m.ops) < 32 {
op = &operationMetrics{}
m.ops[name] = op
}
m.mu.Unlock()
}
if op != nil {
op.observe(start, err)
if op != nil {
debugOperationError(name, err)
op.observe(start, err)
}
}
}
func (m *metricSet) report() map[string]OperationReport {
m.mu.RLock()
defer m.mu.RUnlock()
return m.reportLocked()
}
// freeze returns the immutable pre-teardown operation cut. Holding the write
// lock waits for any observer already publishing its complete metric tuple and
// prevents later coordinated-cancel outcomes from entering the business report.
func (m *metricSet) freeze() map[string]OperationReport {
m.mu.Lock()
defer m.mu.Unlock()
m.frozen = true
return m.reportLocked()
}
func (m *metricSet) reportLocked() map[string]OperationReport {
out := make(map[string]OperationReport, len(m.ops))
for name, op := range m.ops {
out[name] = op.report()
@ -155,31 +183,46 @@ func (m *metricSet) report() map[string]OperationReport {
}
type RunReport struct {
Version int `json:"version"`
StartedAt time.Time `json:"started_at"`
LoadEndedAt time.Time `json:"load_ended_at"`
FinishedAt time.Time `json:"finished_at"`
RequestedDuration string `json:"requested_duration"`
RecoveryDuration string `json:"recovery_duration"`
ExpectedSessions int `json:"expected_sessions"`
PeakReadySessions int `json:"peak_ready_sessions"`
FinalReadySessions int `json:"final_ready_sessions"`
SteadySamples int `json:"steady_samples"`
SteadyReadyRatio float64 `json:"steady_ready_ratio"`
MinSteadyReadySessions int `json:"min_steady_ready_sessions"`
ConnectionAttempts uint64 `json:"connection_attempts"`
Reconnects uint64 `json:"reconnects"`
Disconnects uint64 `json:"disconnects"`
UpdatesReceived uint64 `json:"updates_received"`
DownloadedBytes uint64 `json:"downloaded_bytes"`
WorkerFatalErrors uint64 `json:"worker_fatal_errors"`
Operations map[string]OperationReport `json:"operations"`
BaselineServerMetrics map[string]float64 `json:"baseline_server_metrics,omitempty"`
FinalServerMetrics map[string]float64 `json:"final_server_metrics,omitempty"`
ServerMetricsScrapes uint64 `json:"server_metrics_scrapes"`
ServerMetricsErrors uint64 `json:"server_metrics_errors"`
Pass bool `json:"pass"`
Failures []string `json:"failures,omitempty"`
Version int `json:"version"`
StartOrder string `json:"start_order,omitempty"`
StartOrderSeed int64 `json:"start_order_seed,omitempty"`
StartedAt time.Time `json:"started_at"`
LoadEndedAt time.Time `json:"load_ended_at"`
FinishedAt time.Time `json:"finished_at"`
RequestedDuration string `json:"requested_duration"`
RecoveryDuration string `json:"recovery_duration"`
ExpectedSessions int `json:"expected_sessions"`
PeakReadySessions int `json:"peak_ready_sessions"`
FinalReadySessions int `json:"final_ready_sessions"`
SteadySamples int `json:"steady_samples"`
SteadyReadyRatio float64 `json:"steady_ready_ratio"`
MinSteadyReadySessions int `json:"min_steady_ready_sessions"`
ConnectionAttempts uint64 `json:"connection_attempts"`
Reconnects uint64 `json:"reconnects"`
Disconnects uint64 `json:"disconnects"`
UpdatesReceived uint64 `json:"updates_received"`
DownloadedBytes uint64 `json:"downloaded_bytes"`
WorkerFatalErrors uint64 `json:"worker_fatal_errors"`
MessageRatePerSecond float64 `json:"message_rate_per_second"`
MessageScheduled uint64 `json:"message_scheduled"`
MessageEnqueued uint64 `json:"message_enqueued"`
MessageCompleted uint64 `json:"message_completed"`
MessageQueueFull uint64 `json:"message_queue_full"`
MessageNotReady uint64 `json:"message_not_ready"`
Delivery DeliveryReport `json:"delivery"`
Operations map[string]OperationReport `json:"operations"`
ResponseBytes map[string]StartupResponseBytes `json:"response_bytes,omitempty"`
RPCDeliveryOutcomes map[string]map[string]uint64 `json:"rpc_delivery_outcomes,omitempty"`
DatabaseWork map[string]StartupDatabaseWork `json:"database_work,omitempty"`
BaselineServerMetrics map[string]float64 `json:"baseline_server_metrics,omitempty"`
WorkloadEndServerMetrics map[string]float64 `json:"workload_end_server_metrics,omitempty"`
FinalServerMetrics map[string]float64 `json:"final_server_metrics,omitempty"`
ServerMetricsScrapes uint64 `json:"server_metrics_scrapes"`
ServerMetricsErrors uint64 `json:"server_metrics_errors"`
EventsWritten uint64 `json:"events_written"`
EventsDropped uint64 `json:"events_dropped"`
Pass bool `json:"pass"`
Failures []string `json:"failures,omitempty"`
}
func WriteReport(path string, report *RunReport) error {
@ -191,12 +234,18 @@ func WriteReport(path string, report *RunReport) error {
}
type eventWriter struct {
mu sync.Mutex
f *os.File
written uint64
dropped uint64
mu sync.Mutex
f *os.File
written uint64
dropped uint64
connectionDeadWritten uint64
}
const (
maxEventLines = 100000
maxConnectionDeadEventLines = 5000
)
func newEventWriter(path string) (*eventWriter, error) {
if path == "" {
return &eventWriter{}, nil
@ -220,7 +269,15 @@ func (w *eventWriter) write(value any) {
return
}
w.mu.Lock()
if w.written >= 10000 {
if event, ok := value.(map[string]any); ok && event["type"] == "connection_dead" {
if w.connectionDeadWritten >= maxConnectionDeadEventLines {
w.dropped++
w.mu.Unlock()
return
}
w.connectionDeadWritten++
}
if w.written >= maxEventLines {
w.dropped++
w.mu.Unlock()
return
@ -230,6 +287,16 @@ func (w *eventWriter) write(value any) {
w.mu.Unlock()
}
func (w *eventWriter) counts() (written, dropped uint64) {
if w == nil {
return 0, 0
}
w.mu.Lock()
written, dropped = w.written, w.dropped
w.mu.Unlock()
return written, dropped
}
func (w *eventWriter) close() error {
if w == nil || w.f == nil {
return nil

View file

@ -5,6 +5,8 @@ import (
"errors"
"fmt"
"net"
"path/filepath"
"sync/atomic"
"testing"
"time"
@ -12,6 +14,43 @@ import (
tdrpc "github.com/iamxvbaba/td/rpc"
)
func TestMarkClientReadyAvoidsDuplicateInitialCatchUp(t *testing.T) {
var everReady atomic.Bool
var firstClient atomic.Bool
if markClientReady(&everReady, &firstClient) {
t.Fatal("first Ready transition requested a catch-up")
}
if !markClientReady(&everReady, &firstClient) {
t.Fatal("transport reconnect did not request a catch-up")
}
var replacementClient atomic.Bool
if markClientReady(&everReady, &replacementClient) {
t.Fatal("replacement Client duplicated its callback-owned initial catch-up")
}
if !markClientReady(&everReady, &replacementClient) {
t.Fatal("replacement Client transport reconnect did not request a catch-up")
}
}
func TestEventWriterCapsConnectionDetailsWithoutDroppingSamples(t *testing.T) {
w, err := newEventWriter(filepath.Join(t.TempDir(), "events.ndjson"))
if err != nil {
t.Fatal(err)
}
for i := 0; i < maxConnectionDeadEventLines+1; i++ {
w.write(map[string]any{"type": "connection_dead", "class": "connection"})
}
w.write(map[string]any{"type": "sample", "ready": 0})
written, dropped := w.counts()
if written != maxConnectionDeadEventLines+1 || dropped != 1 {
t.Fatalf("event counts = %d/%d", written, dropped)
}
if err := w.close(); err != nil {
t.Fatal(err)
}
}
func TestOperationMetricsUsesBoundedHistogramAndFixedErrorClasses(t *testing.T) {
metrics := &operationMetrics{}
metrics.observe(time.Now().Add(-20*time.Millisecond), nil)
@ -33,6 +72,7 @@ func TestClassifyErrorReasonUsesFiniteRedactedVocabulary(t *testing.T) {
{errors.New("dial tcp 10.0.0.1:2398: socket: too many open files"), "file_descriptor_limit"},
{errors.New("read: temporary auth key not found: pfs reconnect required"), "pfs_reconnect"},
{errors.New("read tcp: EOF auth_key_id=secret"), "eof"},
{errors.New("startup session ended before business readiness"), "business_readiness_incomplete"},
}
for _, test := range tests {
if got := classifyErrorReason(test.err); got != test.want {
@ -64,6 +104,34 @@ func TestOperationMetricsSeparatesHarnessCancellation(t *testing.T) {
}
}
func TestMetricSetFreezeExcludesCoordinatedTeardown(t *testing.T) {
metrics := newMetricSet("ping")
metrics.observe("ping", time.Now(), nil)
cut := metrics.freeze()
metrics.observe("ping", time.Now(), fmt.Errorf("engine forcibly closed: %w", context.Canceled))
if got := cut["ping"]; got.Count != 1 || got.Errors != 0 || got.Canceled != 0 {
t.Fatalf("workload cut = %#v", got)
}
if got := metrics.report()["ping"]; got != cut["ping"] {
t.Fatalf("post-freeze metrics changed: got %#v want %#v", got, cut["ping"])
}
}
func TestEvaluateReportRejectsFixedRateDeliveryLoss(t *testing.T) {
report := &RunReport{
ExpectedSessions: 1, PeakReadySessions: 1,
SteadySamples: 1, SteadyReadyRatio: 1, MinSteadyReadySessions: 1,
MessageScheduled: 2, MessageEnqueued: 2, MessageCompleted: 2,
Delivery: DeliveryReport{Expected: 2, Delivered: 1, Missing: 1},
Operations: map[string]OperationReport{},
}
evaluateReport(report, RunConfig{MinimumReadyRatio: 1, MessageRate: 1})
if report.Pass {
t.Fatal("fixed-rate report with a missing recipient delivery passed")
}
}
func TestEvaluateReportAllowsOnlyConnectionErrorsForExpectedRestart(t *testing.T) {
report := &RunReport{
ExpectedSessions: 2, PeakReadySessions: 2, Reconnects: 2,
@ -84,6 +152,24 @@ func TestEvaluateReportAllowsOnlyConnectionErrorsForExpectedRestart(t *testing.T
}
}
func TestEvaluateReportRejectsServerDeliveryAndDatabaseErrors(t *testing.T) {
report := &RunReport{
ExpectedSessions: 1, PeakReadySessions: 1,
SteadySamples: 1, SteadyReadyRatio: 1, MinSteadyReadySessions: 1,
Operations: map[string]OperationReport{},
RPCDeliveryOutcomes: map[string]map[string]uint64{
"updates.getState": {"ok": 1, "edge_overload": 2},
},
DatabaseWork: map[string]StartupDatabaseWork{
"messages.getDialogs": {Errors: 3},
},
}
evaluateReport(report, RunConfig{MinimumReadyRatio: 1})
if report.Pass || len(report.Failures) != 2 {
t.Fatalf("report = %#v", report)
}
}
func TestEvaluateReportRequiresReclamationAndNoFloodWait(t *testing.T) {
report := &RunReport{
ExpectedSessions: 10, PeakReadySessions: 10, ServerMetricsScrapes: 1,
@ -97,6 +183,7 @@ func TestEvaluateReportRequiresReclamationAndNoFloodWait(t *testing.T) {
"telesrv_mtproto_raw_connections": 2,
"telesrv_mtproto_logical_outbox_bytes": 4,
},
WorkloadEndServerMetrics: map[string]float64{},
}
evaluateReport(report, RunConfig{MinimumReadyRatio: 1, RecoveryDuration: time.Minute, ServerMetricsURL: "http://metrics"})
if report.Pass || len(report.Failures) != 1 {
@ -119,6 +206,7 @@ func TestEvaluateReportAcceptsReturnToNonZeroSharedServerBaseline(t *testing.T)
"telesrv_mtproto_logical_sessions": 2,
"telesrv_mtproto_logical_outbox_bytes": 1024,
},
WorkloadEndServerMetrics: map[string]float64{},
}
evaluateReport(report, RunConfig{MinimumReadyRatio: 1, RecoveryDuration: time.Minute, ServerMetricsURL: "http://metrics"})
if !report.Pass || len(report.Failures) != 0 {

View file

@ -0,0 +1,232 @@
package loadharness
import (
"context"
"crypto/rsa"
"errors"
"fmt"
"time"
"github.com/iamxvbaba/td/tg"
)
type richAccountStatePlan struct {
PinnedPeerAccount int
ReadPeerAccount int
ReadGroupPosition int
DraftMarker string
}
func planRichAccountState(dataset *Dataset, account int) (richAccountStatePlan, error) {
if dataset == nil || account < 0 || account >= dataset.Config.Accounts {
return richAccountStatePlan{}, errors.New("invalid rich-state account")
}
if dataset.Config.PrivateFanout < 1 {
return richAccountStatePlan{}, errors.New("rich startup dataset requires at least one private peer per account")
}
groupPosition := -1
for position, group := range dataset.Groups {
if group.HistoryMessages > 0 && datasetGroupHasAccount(group, account) {
groupPosition = position
break
}
}
if groupPosition < 0 {
return richAccountStatePlan{}, errors.New("rich startup dataset requires a non-empty supergroup per account")
}
return richAccountStatePlan{
PinnedPeerAccount: (account + 1) % dataset.Config.Accounts,
ReadPeerAccount: (account - 1 + dataset.Config.Accounts) % dataset.Config.Accounts,
ReadGroupPosition: groupPosition,
DraftMarker: fmt.Sprintf("[%s draft account %04d]", dataset.RunID, account),
}, nil
}
// seedRichAccountState creates synchronized dialog state exclusively through
// public MTProto RPCs. Every mutation is absolute and then read back through
// messages.getPeerDialogs before the resumable journal is committed.
func seedRichAccountState(
ctx context.Context,
cfg SeedConfig,
manifest *Manifest,
dataset *Dataset,
journal *seedJournal,
targets []SessionRecord,
key [32]byte,
publicKey *rsa.PublicKey,
account int,
) error {
if journal.richStateComplete(account) {
return nil
}
plan, err := planRichAccountState(dataset, account)
if err != nil {
return err
}
return withAuthorizedSeedSession(ctx, cfg, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
pinnedTarget := targets[plan.PinnedPeerAccount]
readTarget := targets[plan.ReadPeerAccount]
pinnedPeer := &tg.InputPeerUser{UserID: pinnedTarget.UserID, AccessHash: pinnedTarget.AccessHash}
readPeer := &tg.InputPeerUser{UserID: readTarget.UserID, AccessHash: readTarget.AccessHash}
groupIdentity := journal.group(plan.ReadGroupPosition)
if groupIdentity.ChannelID <= 0 || groupIdentity.AccessHash == 0 {
return errors.New("rich-state group identity is incomplete")
}
channelPeer := &tg.InputPeerChannel{ChannelID: groupIdentity.ChannelID, AccessHash: groupIdentity.AccessHash}
before, err := getRichStateDialogs(ctx, cfg.OperationTimeout, raw, pinnedPeer, readPeer, channelPeer)
if err != nil {
return fmt.Errorf("read rich-state cursors: %w", err)
}
readPrivate, ok := before[clientPeerKey{typ: "user", id: readTarget.UserID}]
if !ok || readPrivate.TopMessage <= 0 {
return errors.New("read private peer omitted its top message")
}
readChannel, ok := before[clientPeerKey{typ: "channel", id: groupIdentity.ChannelID}]
if !ok || readChannel.TopMessage <= 0 {
return errors.New("read channel omitted its top message")
}
pinned, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (bool, error) {
return raw.MessagesToggleDialogPin(rpcCtx, &tg.MessagesToggleDialogPinRequest{
Pinned: true, Peer: &tg.InputDialogPeer{Peer: pinnedPeer},
})
})
if err != nil || !pinned {
return rpcBooleanError("messages.toggleDialogPin", pinned, err)
}
saved, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (bool, error) {
return raw.MessagesSaveDraft(rpcCtx, &tg.MessagesSaveDraftRequest{Peer: pinnedPeer, Message: plan.DraftMarker})
})
if err != nil || !saved {
return rpcBooleanError("messages.saveDraft", saved, err)
}
if _, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (*tg.MessagesAffectedMessages, error) {
return raw.MessagesReadHistory(rpcCtx, &tg.MessagesReadHistoryRequest{Peer: readPeer, MaxID: readPrivate.TopMessage})
}); err != nil {
return fmt.Errorf("messages.readHistory: %w", err)
}
channelRead, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (bool, error) {
return raw.ChannelsReadHistory(rpcCtx, &tg.ChannelsReadHistoryRequest{
Channel: &tg.InputChannel{ChannelID: groupIdentity.ChannelID, AccessHash: groupIdentity.AccessHash},
MaxID: readChannel.TopMessage,
})
})
if err != nil || !channelRead {
return rpcBooleanError("channels.readHistory", channelRead, err)
}
after, err := getRichStateDialogs(ctx, cfg.OperationTimeout, raw, pinnedPeer, readPeer, channelPeer)
if err != nil {
return fmt.Errorf("verify rich-state dialogs: %w", err)
}
pinnedDialog, ok := after[clientPeerKey{typ: "user", id: pinnedTarget.UserID}]
if !ok || !pinnedDialog.Pinned || !pinnedDialog.HasDraft || pinnedDialog.DraftText != plan.DraftMarker {
return errors.New("pinned private dialog or exact draft was not persisted")
}
readPrivate = after[clientPeerKey{typ: "user", id: readTarget.UserID}]
if readPrivate.ReadInboxMaxID < before[clientPeerKey{typ: "user", id: readTarget.UserID}].TopMessage || readPrivate.UnreadCount != 0 {
return errors.New("private read boundary did not converge")
}
readChannel = after[clientPeerKey{typ: "channel", id: groupIdentity.ChannelID}]
if readChannel.ReadInboxMaxID < before[clientPeerKey{typ: "channel", id: groupIdentity.ChannelID}].TopMessage || readChannel.UnreadCount != 0 {
return errors.New("channel read boundary did not converge")
}
return journal.setRichStateComplete(account)
})
}
func rpcBooleanError(operation string, result bool, err error) error {
if err != nil {
return fmt.Errorf("%s result=%v: %w", operation, result, err)
}
return fmt.Errorf("%s returned false", operation)
}
func getRichStateDialogs(
ctx context.Context,
timeout time.Duration,
raw *tg.Client,
peers ...tg.InputPeerClass,
) (map[clientPeerKey]ClientDialogState, error) {
requests := make([]tg.InputDialogPeerClass, 0, len(peers))
seen := make(map[clientPeerKey]struct{}, len(peers))
for _, peer := range peers {
key, ok := clientPeerFromInput(peer)
if !ok {
return nil, errors.New("invalid rich-state input peer")
}
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
requests = append(requests, &tg.InputDialogPeer{Peer: peer})
}
response, err := rpcWithFloodWaitRetry(ctx, timeout, func(rpcCtx context.Context) (*tg.MessagesPeerDialogs, error) {
return raw.MessagesGetPeerDialogs(rpcCtx, requests)
})
if err != nil {
return nil, err
}
dialogs := make(map[clientPeerKey]ClientDialogState, len(response.Dialogs))
if _, err := mergeDialogPage(dialogs, response.Dialogs, response.Messages, response.Chats, response.Users, false); err != nil {
return nil, err
}
if len(dialogs) != len(requests) {
return nil, fmt.Errorf("messages.getPeerDialogs returned %d/%d dialogs", len(dialogs), len(requests))
}
return dialogs, nil
}
func clientPeerFromInput(peer tg.InputPeerClass) (clientPeerKey, bool) {
switch value := peer.(type) {
case *tg.InputPeerUser:
return clientPeerKey{typ: "user", id: value.UserID}, value.UserID > 0 && value.AccessHash != 0
case *tg.InputPeerChannel:
return clientPeerKey{typ: "channel", id: value.ChannelID}, value.ChannelID > 0 && value.AccessHash != 0
default:
return clientPeerKey{}, false
}
}
func validateSeededRichDialogs(
dataset *Dataset,
seedState *DatasetSeedState,
targets []SessionRecord,
account int,
dialogs []ClientDialogState,
requireReadBoundaries bool,
) error {
if len(seedState.RichStateByAccount) == 0 {
return nil
}
if len(seedState.RichStateByAccount) != dataset.Config.Accounts || !seedState.RichStateByAccount[account] {
return errors.New("rich-state seed is incomplete")
}
plan, err := planRichAccountState(dataset, account)
if err != nil {
return err
}
byPeer := make(map[clientPeerKey]ClientDialogState, len(dialogs))
for _, dialog := range dialogs {
byPeer[clientPeerKey{typ: dialog.PeerType, id: dialog.PeerID}] = dialog
}
pinned := byPeer[clientPeerKey{typ: "user", id: targets[plan.PinnedPeerAccount].UserID}]
if !pinned.Pinned || !pinned.HasDraft || pinned.DraftText != plan.DraftMarker {
return fmt.Errorf("seeded pinned dialog state mismatch: present=%v pinned=%v has_draft=%v draft_matches=%v got_draft=%q want_draft=%q",
pinned.PeerID != 0, pinned.Pinned, pinned.HasDraft, pinned.DraftText == plan.DraftMarker, pinned.DraftText, plan.DraftMarker)
}
if !requireReadBoundaries {
return nil
}
readPrivate := byPeer[clientPeerKey{typ: "user", id: targets[plan.ReadPeerAccount].UserID}]
if readPrivate.TopMessage <= 0 || readPrivate.ReadInboxMaxID < readPrivate.TopMessage || readPrivate.UnreadCount != 0 {
return errors.New("seeded private read boundary is stale")
}
group := seedState.Groups[plan.ReadGroupPosition]
readChannel := byPeer[clientPeerKey{typ: "channel", id: group.ChannelID}]
if readChannel.TopMessage <= 0 || readChannel.ReadInboxMaxID < readChannel.TopMessage || readChannel.UnreadCount != 0 {
return errors.New("seeded channel read boundary is stale")
}
return nil
}

View file

@ -0,0 +1,89 @@
package loadharness
import (
"context"
"fmt"
"time"
"github.com/iamxvbaba/td/tgerr"
)
const (
maxDatasetFloodWaitRetries = 16
maxDatasetFloodWait = 2 * time.Minute
datasetFloodWaitPadding = time.Second
)
type floodWaitPolicy struct {
maxRetries int
maxWait time.Duration
padding time.Duration
wait func(context.Context, time.Duration) error
}
func defaultFloodWaitPolicy() floodWaitPolicy {
return floodWaitPolicy{
maxRetries: maxDatasetFloodWaitRetries,
maxWait: maxDatasetFloodWait,
padding: datasetFloodWaitPadding,
wait: waitForContext,
}
}
// rpcWithFloodWaitRetry is reserved for dataset preparation. Startup-run must
// observe FLOOD_WAIT as a measured failure instead of hiding it behind a retry.
func rpcWithFloodWaitRetry[T any](
ctx context.Context,
timeout time.Duration,
call func(context.Context) (T, error),
) (T, error) {
return rpcWithFloodWaitPolicy(ctx, timeout, defaultFloodWaitPolicy(), call)
}
func rpcWithFloodWaitPolicy[T any](
ctx context.Context,
timeout time.Duration,
policy floodWaitPolicy,
call func(context.Context) (T, error),
) (T, error) {
var zero T
if timeout <= 0 {
return zero, fmt.Errorf("RPC timeout must be positive")
}
if policy.maxRetries < 0 || policy.maxWait < 0 || policy.padding < 0 || policy.wait == nil {
return zero, fmt.Errorf("invalid FLOOD_WAIT retry policy")
}
for retry := 0; ; retry++ {
rpcCtx, cancel := context.WithTimeout(ctx, timeout)
result, err := call(rpcCtx)
cancel()
if err == nil {
return result, nil
}
wait, ok := tgerr.AsFloodWait(err)
if !ok {
return zero, err
}
if retry >= policy.maxRetries {
return zero, fmt.Errorf("FLOOD_WAIT retry limit %d exhausted: %w", policy.maxRetries, err)
}
wait += policy.padding
if wait > policy.maxWait {
return zero, fmt.Errorf("FLOOD_WAIT %s exceeds dataset preparation limit %s: %w", wait, policy.maxWait, err)
}
if err := policy.wait(ctx, wait); err != nil {
return zero, err
}
}
}
func waitForContext(ctx context.Context, delay time.Duration) error {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-timer.C:
return nil
case <-ctx.Done():
return ctx.Err()
}
}

View file

@ -0,0 +1,63 @@
package loadharness
import (
"context"
"errors"
"testing"
"time"
"github.com/iamxvbaba/td/tgerr"
)
func TestRPCWithFloodWaitPolicyRetriesTheSameCall(t *testing.T) {
attempts := 0
waits := make([]time.Duration, 0, 1)
policy := floodWaitPolicy{
maxRetries: 2,
maxWait: 5 * time.Second,
padding: 100 * time.Millisecond,
wait: func(_ context.Context, delay time.Duration) error {
waits = append(waits, delay)
return nil
},
}
result, err := rpcWithFloodWaitPolicy(context.Background(), time.Second, policy, func(context.Context) (int, error) {
attempts++
if attempts == 1 {
return 0, tgerr.New(420, "FLOOD_WAIT_2")
}
return 42, nil
})
if err != nil {
t.Fatal(err)
}
if result != 42 || attempts != 2 {
t.Fatalf("result=%d attempts=%d", result, attempts)
}
if len(waits) != 1 || waits[0] != 2100*time.Millisecond {
t.Fatalf("waits=%v", waits)
}
}
func TestRPCWithFloodWaitPolicyDoesNotRetryOrdinaryError(t *testing.T) {
want := errors.New("boom")
attempts := 0
policy := floodWaitPolicy{maxRetries: 2, maxWait: time.Second, wait: waitForContext}
_, err := rpcWithFloodWaitPolicy(context.Background(), time.Second, policy, func(context.Context) (int, error) {
attempts++
return 0, want
})
if !errors.Is(err, want) || attempts != 1 {
t.Fatalf("err=%v attempts=%d", err, attempts)
}
}
func TestRPCWithFloodWaitPolicyRejectsExcessiveWait(t *testing.T) {
policy := floodWaitPolicy{maxRetries: 2, maxWait: time.Second, wait: waitForContext}
_, err := rpcWithFloodWaitPolicy(context.Background(), time.Second, policy, func(context.Context) (int, error) {
return 0, tgerr.New(420, "FLOOD_WAIT_2")
})
if err == nil {
t.Fatal("excessive FLOOD_WAIT unexpectedly retried")
}
}

View file

@ -14,6 +14,7 @@ import (
"net"
"os"
"runtime"
"sort"
"strings"
"sync"
"sync/atomic"
@ -43,12 +44,17 @@ type RunConfig struct {
EventsPath string
FileFixturePath string
ServerMetricsURL string
StartOrder string
StartOrderSeed int64
SessionLimit int
Duration time.Duration
RecoveryDuration time.Duration
RampDuration time.Duration
RPCInterval time.Duration
MessageInterval time.Duration
MessageRate float64
MessageQueueDepth int
DeliverySettle time.Duration
FileInterval time.Duration
FileSizeBytes int
FileChunkBytes int
@ -62,6 +68,8 @@ type RunConfig struct {
ExpectServerRestart bool
}
const RunReportVersion = 7
func (c RunConfig) validate() error {
if c.ManifestPath == "" || c.SessionKeyPath == "" || c.ReportPath == "" {
return errors.New("manifest, session-key and report paths are required")
@ -69,6 +77,15 @@ func (c RunConfig) validate() error {
if c.Duration <= 0 || c.RecoveryDuration < 0 || c.RampDuration < 0 || c.RPCInterval <= 0 || c.OperationTimeout <= 0 || c.SampleInterval <= 0 {
return errors.New("run durations and intervals are invalid")
}
if c.MessageRate < 0 || c.MessageRate > 100000 || c.MessageQueueDepth < 0 || c.MessageQueueDepth > 1024 || c.DeliverySettle < 0 {
return errors.New("message rate, queue depth or delivery settle is invalid")
}
if c.MessageRate > 0 && c.MessageInterval > 0 {
return errors.New("message-rate and message-interval workloads are mutually exclusive")
}
if c.MessageRate > 0 && (c.MessageQueueDepth == 0 || c.RampDuration >= c.Duration) {
return errors.New("fixed-rate workload requires a queue depth and load duration beyond the connection ramp")
}
if c.FileSizeBytes < 0 || c.FileChunkBytes < 0 || c.FileChunkBytes > 1<<20 || c.FileSizeBytes > 64<<20 {
return errors.New("file size must be <=64MiB and chunk size must be <=1MiB")
}
@ -84,6 +101,9 @@ func (c RunConfig) validate() error {
if c.OfflineFraction > 0 && (c.OfflineAt <= 0 || c.OfflineFor <= 0 || c.OfflineAt+c.OfflineFor >= c.Duration) {
return errors.New("offline window must be positive and fit inside load duration")
}
if c.StartOrder != "" && c.StartOrder != StartupOrderShuffled && c.StartOrder != StartupOrderAccountIndex {
return fmt.Errorf("unknown run start order %q", c.StartOrder)
}
return nil
}
@ -113,6 +133,11 @@ type harnessCounters struct {
updates atomic.Uint64
fatalErrors atomic.Uint64
downloadBytes atomic.Uint64
messageScheduled atomic.Uint64
messageEnqueued atomic.Uint64
messageCompleted atomic.Uint64
messageQueueFull atomic.Uint64
messageNotReady atomic.Uint64
}
var debugConnectionErrors atomic.Uint64
@ -159,21 +184,26 @@ type loadWorker struct {
fileInterval time.Duration
operationTimeout time.Duration
fileFixture *downloadFixture
delivery *deliveryTracker
desired atomic.Bool
state atomic.Int32
everReady atomic.Bool
signal chan struct{}
lastUpdate updateState
messageSeq atomic.Uint64
desired atomic.Bool
state atomic.Int32
everReady atomic.Bool
signal chan struct{}
lastUpdate updateState
deliveryState updateState
messageSeq atomic.Uint64
sendQueue chan struct{}
reconcile chan chan struct{}
}
func newLoadWorker(record, target SessionRecord, endpoint Endpoint, publicKey *rsa.PublicKey, storage *EncryptedFileStorage, metrics *metricSet, counters *harnessCounters, events *eventWriter, rpcInterval, messageInterval, fileInterval, operationTimeout time.Duration, fixture *downloadFixture) *loadWorker {
func newLoadWorker(record, target SessionRecord, endpoint Endpoint, publicKey *rsa.PublicKey, storage *EncryptedFileStorage, metrics *metricSet, counters *harnessCounters, events *eventWriter, rpcInterval, messageInterval, fileInterval, operationTimeout time.Duration, fixture *downloadFixture, delivery *deliveryTracker, messageQueueDepth int) *loadWorker {
w := &loadWorker{
record: record, target: target, endpoint: endpoint, publicKey: publicKey, storage: storage,
metrics: metrics, counters: counters, events: events, rpcInterval: rpcInterval,
msgInterval: messageInterval, fileInterval: fileInterval, operationTimeout: operationTimeout, fileFixture: fixture,
signal: make(chan struct{}, 1),
delivery: delivery, signal: make(chan struct{}, 1), sendQueue: make(chan struct{}, messageQueueDepth),
reconcile: make(chan chan struct{}),
}
w.state.Store(workerStopped)
return w
@ -259,9 +289,11 @@ func (w *loadWorker) supervise(ctx context.Context, wg *sync.WaitGroup) {
func (w *loadWorker) runClient(ctx context.Context) error {
reconnectSignal := make(chan struct{}, 1)
var readySeen atomic.Bool
client, err := newClient(w.endpoint, w.publicKey, w.storage, clientHooks{
Update: telegram.UpdateHandlerFunc(func(context.Context, tg.UpdatesClass) error {
Update: telegram.UpdateHandlerFunc(func(_ context.Context, updates tg.UpdatesClass) error {
w.counters.updates.Add(1)
observeUpdatesClass(w.delivery, w.record.UserID, updates, deliveryLive)
return nil
}),
ConnectionState: func(state telegram.ConnectionState) {
@ -273,9 +305,9 @@ func (w *loadWorker) runClient(ctx context.Context) error {
w.counters.reconnects.Add(1)
}
case telegram.ConnectionStateReady:
wasReady := w.everReady.Swap(true)
needsCatchUp := markClientReady(&w.everReady, &readySeen)
w.state.Store(workerReady)
if wasReady {
if needsCatchUp {
select {
case reconnectSignal <- struct{}{}:
default:
@ -316,6 +348,9 @@ func (w *loadWorker) runClient(ctx context.Context) error {
} else {
w.refreshUpdateState(ctx, raw)
}
if _, valid := w.deliveryState.load(); !valid {
w.refreshDeliveryState(ctx, raw)
}
rpcTicker := time.NewTicker(w.rpcInterval)
defer rpcTicker.Stop()
@ -345,6 +380,11 @@ func (w *loadWorker) runClient(ctx context.Context) error {
cycle++
case <-messageC:
w.sendMessage(ctx, raw)
case <-w.sendQueue:
w.sendMessage(ctx, raw)
case done := <-w.reconcile:
w.catchUpDelivery(ctx, raw)
close(done)
case <-fileC:
w.downloadFileChunk(ctx, raw)
}
@ -352,6 +392,19 @@ func (w *loadWorker) runClient(ctx context.Context) error {
})
}
// markClientReady distinguishes a transport reconnect inside one live gotd
// Client from the first Ready transition of a newly constructed Client. The
// client.Run callback already performs one cursor catch-up when it starts, so
// enqueueing a second catch-up for that first transition would duplicate every
// explicit offline->online getDifference request. Later Ready transitions do
// need the signal because the callback remains running across transport-level
// reconnects.
func markClientReady(everReady, readySeen *atomic.Bool) bool {
firstForClient := !readySeen.Swap(true)
wasReady := everReady.Swap(true)
return wasReady && !firstForClient
}
func (w *loadWorker) runRPC(ctx context.Context, client *telegram.Client, raw *tg.Client, cycle int) {
start := time.Now()
operationCtx, cancel := w.operationContext(ctx)
@ -385,6 +438,20 @@ func (w *loadWorker) refreshUpdateState(ctx context.Context, raw *tg.Client) {
w.metrics.observe("updates.getState", start, err)
if err == nil {
w.lastUpdate.store(*state)
if _, valid := w.deliveryState.load(); !valid {
w.deliveryState.store(*state)
}
}
}
func (w *loadWorker) refreshDeliveryState(ctx context.Context, raw *tg.Client) {
start := time.Now()
operationCtx, cancel := w.operationContext(ctx)
state, err := raw.UpdatesGetState(operationCtx)
cancel()
w.metrics.observe("updates.getState.delivery", start, err)
if err == nil {
w.deliveryState.store(*state)
}
}
@ -427,7 +494,49 @@ func (w *loadWorker) catchUp(ctx context.Context, raw *tg.Client) {
}
}
func (w *loadWorker) catchUpDelivery(ctx context.Context, raw *tg.Client) {
state, valid := w.deliveryState.load()
if !valid {
w.refreshDeliveryState(ctx, raw)
return
}
for page := 0; page < 256; page++ {
start := time.Now()
operationCtx, cancel := w.operationContext(ctx)
difference, err := raw.UpdatesGetDifference(operationCtx, &tg.UpdatesGetDifferenceRequest{Pts: state.Pts, Date: state.Date, Qts: state.Qts})
cancel()
w.metrics.observe("updates.getDifference.delivery", start, err)
if err != nil {
return
}
switch value := difference.(type) {
case *tg.UpdatesDifferenceEmpty:
state.Date, state.Seq = value.Date, value.Seq
w.deliveryState.store(state)
return
case *tg.UpdatesDifference:
observeMessageClasses(w.delivery, w.record.UserID, value.NewMessages, deliveryDifference)
observeUpdateClasses(w.delivery, w.record.UserID, value.OtherUpdates, deliveryDifference)
state = value.State
w.deliveryState.store(state)
return
case *tg.UpdatesDifferenceSlice:
observeMessageClasses(w.delivery, w.record.UserID, value.NewMessages, deliveryDifference)
observeUpdateClasses(w.delivery, w.record.UserID, value.OtherUpdates, deliveryDifference)
state = value.IntermediateState
w.deliveryState.store(state)
case *tg.UpdatesDifferenceTooLong:
state.Pts = value.Pts
w.deliveryState.store(state)
return
default:
return
}
}
}
func (w *loadWorker) sendMessage(ctx context.Context, raw *tg.Client) {
defer w.counters.messageCompleted.Add(1)
sequence := w.messageSeq.Add(1)
var randomBytes [8]byte
if _, err := cryptorand.Read(randomBytes[:]); err != nil {
@ -438,13 +547,16 @@ func (w *loadWorker) sendMessage(ctx context.Context, raw *tg.Client) {
randomID = int64(sequence)
}
start := time.Now()
marker := w.delivery.marker(w.record.Index, sequence)
w.delivery.begin(marker, w.record.UserID, w.target.UserID, start)
operationCtx, cancel := w.operationContext(ctx)
_, err := raw.MessagesSendMessage(operationCtx, &tg.MessagesSendMessageRequest{
Peer: &tg.InputPeerUser{UserID: w.target.UserID, AccessHash: w.target.AccessHash},
Message: fmt.Sprintf("load/%d/%d", w.record.Index, sequence), RandomID: randomID,
Message: marker, RandomID: randomID,
})
cancel()
w.metrics.observe("messages.sendMessage", start, err)
w.delivery.finish(marker, err == nil)
}
func (w *loadWorker) downloadFileChunk(ctx context.Context, raw *tg.Client) {
@ -654,8 +766,13 @@ func Run(ctx context.Context, cfg RunConfig) (*RunReport, error) {
return nil, err
}
defer events.close()
metrics := newMetricSet("auth.status", "connection.dead", "ping", "updates.getState", "updates.getDifference", "messages.getDialogs", "help.getConfig", "messages.sendMessage", "upload.saveFilePart", "messages.uploadMedia", "upload.getFile")
metrics := newMetricSet("auth.status", "connection.dead", "ping", "updates.getState", "updates.getDifference", "updates.getState.delivery", "updates.getDifference.delivery", "messages.getDialogs", "help.getConfig", "messages.sendMessage", "upload.saveFilePart", "messages.uploadMedia", "upload.getFile")
counters := &harnessCounters{}
runID, err := newLoadRunID()
if err != nil {
return nil, fmt.Errorf("create load run id: %w", err)
}
delivery := newDeliveryTracker(runID)
serverMetrics := newServerMetricsClient(cfg.ServerMetricsURL)
var baselineServerMetrics map[string]float64
if serverMetrics != nil {
@ -681,8 +798,10 @@ func Run(ctx context.Context, cfg RunConfig) (*RunReport, error) {
record, target, manifest.Endpoint, publicKey,
&EncryptedFileStorage{Path: resolveSessionPath(cfg.ManifestPath, record), Key: key},
metrics, counters, events, cfg.RPCInterval, cfg.MessageInterval, cfg.FileInterval, cfg.OperationTimeout, fixture,
delivery, cfg.MessageQueueDepth,
))
}
messageWorkers := primaryWorkers(workers)
startedAt := time.Now().UTC()
loadCtx, stopLoad := context.WithCancel(ctx)
@ -691,10 +810,20 @@ func Run(ctx context.Context, cfg RunConfig) (*RunReport, error) {
workerWG.Add(1)
go worker.supervise(loadCtx, &workerWG)
}
for i, worker := range workers {
startOrder := cfg.StartOrder
if startOrder == "" {
startOrder = StartupOrderAccountIndex
}
startOrderSeed := cfg.StartOrderSeed
if startOrderSeed == 0 {
startOrderSeed = 20260827
}
launchOrder := startupAccountOrder(len(workers), startOrder, startOrderSeed)
for position, workerIndex := range launchOrder {
worker := workers[workerIndex]
delay := time.Duration(0)
if len(workers) > 1 {
delay = time.Duration(i) * cfg.RampDuration / time.Duration(len(workers)-1)
delay = time.Duration(position) * cfg.RampDuration / time.Duration(len(workers)-1)
}
go func(w *loadWorker, d time.Duration) {
timer := time.NewTimer(d)
@ -710,6 +839,13 @@ func Run(ctx context.Context, cfg RunConfig) (*RunReport, error) {
if cfg.OfflineFraction > 0 {
go runOfflineWindow(loadCtx, workers, cfg.OfflineFraction, cfg.OfflineAt, cfg.OfflineFor, events)
}
messageCtx, stopMessages := context.WithCancel(loadCtx)
defer stopMessages()
var messageWG sync.WaitGroup
if cfg.MessageRate > 0 {
messageWG.Add(1)
go runFixedMessageSchedule(messageCtx, &messageWG, cfg.RampDuration, cfg.MessageRate, messageWorkers, counters, events)
}
loadTimer := time.NewTimer(cfg.Duration)
sampleTicker := time.NewTicker(cfg.SampleInterval)
peakReady := 0
@ -741,11 +877,64 @@ func Run(ctx context.Context, cfg RunConfig) (*RunReport, error) {
loadFinished:
sampleTicker.Stop()
stopMessages()
messageWG.Wait()
loadEndedAt := time.Now().UTC()
if cfg.MessageRate > 0 {
drainCtx, cancelDrain := context.WithTimeout(ctx, cfg.OperationTimeout+time.Duration(cfg.MessageQueueDepth)*cfg.OperationTimeout)
waitMessageDrain(drainCtx, counters)
cancelDrain()
}
if cfg.DeliverySettle > 0 && delivery.report().Expected > delivery.report().Delivered {
settleTimer := time.NewTimer(cfg.DeliverySettle)
settleTicker := time.NewTicker(min(cfg.SampleInterval, time.Second))
settling:
for {
select {
case <-ctx.Done():
settleTimer.Stop()
settleTicker.Stop()
stopLoad()
workerWG.Wait()
return nil, ctx.Err()
case <-settleTicker.C:
if current := delivery.report(); current.Missing == 0 {
settleTimer.Stop()
settleTicker.Stop()
break settling
}
case <-settleTimer.C:
settleTicker.Stop()
break settling
}
}
}
if delivery.report().Missing > 0 {
reconcileCtx, cancelReconcile := context.WithTimeout(ctx, cfg.OperationTimeout*2)
reconcileDeliveries(reconcileCtx, workers)
cancelReconcile()
}
// Take the authoritative business-work cutoff before canceling clients.
// Coordinated teardown can cancel an RPC already in flight; both server
// outcome counters and client operation counters must therefore stop before
// that cancellation begins. FinalServerMetrics remains the post-recovery
// resource-reclamation snapshot.
var workloadEndServerMetrics map[string]float64
if serverMetrics != nil {
if sample, scrapeErr := serverMetrics.scrape(ctx); scrapeErr == nil {
workloadEndServerMetrics = sample
finalServerMetrics = sample
events.write(map[string]any{"type": "server_workload_end", "at": time.Now().UTC(), "server_metrics": sample})
} else {
events.write(map[string]any{"type": "server_workload_end_error", "at": time.Now().UTC(), "class": classifyError(scrapeErr)})
}
}
workloadEndOperations := metrics.freeze()
finalReady := countWorkerState(workers, workerReady)
stopLoad()
workerWG.Wait()
loadEndedAt := time.Now().UTC()
if ready := countWorkerState(workers, workerReady); ready > peakReady {
peakReady = ready
if finalReady > peakReady {
peakReady = finalReady
}
if cfg.RecoveryDuration > 0 {
@ -779,16 +968,24 @@ recoveryFinished:
steadyRatio = float64(steadyReadySum) / float64(steadySamples*len(workers))
}
report := &RunReport{
Version: 2, StartedAt: startedAt, LoadEndedAt: loadEndedAt, FinishedAt: time.Now().UTC(),
Version: RunReportVersion, StartedAt: startedAt, LoadEndedAt: loadEndedAt, FinishedAt: time.Now().UTC(),
StartOrder: startOrder, StartOrderSeed: startOrderSeed,
RequestedDuration: cfg.Duration.String(), RecoveryDuration: cfg.RecoveryDuration.String(),
ExpectedSessions: len(workers), PeakReadySessions: peakReady, FinalReadySessions: countWorkerState(workers, workerReady),
ExpectedSessions: len(workers), PeakReadySessions: peakReady, FinalReadySessions: finalReady,
ConnectionAttempts: counters.connectionAttempts.Load(), Reconnects: counters.reconnects.Load(),
Disconnects: counters.disconnects.Load(), UpdatesReceived: counters.updates.Load(), DownloadedBytes: counters.downloadBytes.Load(),
WorkerFatalErrors: counters.fatalErrors.Load(), Operations: metrics.report(),
BaselineServerMetrics: baselineServerMetrics, FinalServerMetrics: finalServerMetrics,
WorkerFatalErrors: counters.fatalErrors.Load(), Operations: workloadEndOperations,
BaselineServerMetrics: baselineServerMetrics, WorkloadEndServerMetrics: workloadEndServerMetrics, FinalServerMetrics: finalServerMetrics,
ServerMetricsScrapes: serverMetrics.successes(), ServerMetricsErrors: serverMetrics.failures(),
SteadySamples: steadySamples, SteadyReadyRatio: steadyRatio, MinSteadyReadySessions: steadyReadyMinimum,
MessageRatePerSecond: cfg.MessageRate, MessageScheduled: counters.messageScheduled.Load(),
MessageEnqueued: counters.messageEnqueued.Load(), MessageCompleted: counters.messageCompleted.Load(), MessageQueueFull: counters.messageQueueFull.Load(),
MessageNotReady: counters.messageNotReady.Load(), Delivery: delivery.report(),
}
report.ResponseBytes = startupResponseBytes(baselineServerMetrics, workloadEndServerMetrics)
report.RPCDeliveryOutcomes = startupRPCDeliveryOutcomes(baselineServerMetrics, workloadEndServerMetrics)
report.DatabaseWork = startupDatabaseWork(baselineServerMetrics, workloadEndServerMetrics)
report.EventsWritten, report.EventsDropped = events.counts()
evaluateReport(report, cfg)
if err := WriteReport(cfg.ReportPath, report); err != nil {
return nil, err
@ -814,6 +1011,119 @@ func primaryTargets(records []SessionRecord) []SessionRecord {
return targets
}
func newLoadRunID() (string, error) {
var value [8]byte
if _, err := cryptorand.Read(value[:]); err != nil {
return "", err
}
return hex.EncodeToString(value[:]), nil
}
func primaryWorkers(workers []*loadWorker) []*loadWorker {
primary := make([]*loadWorker, 0, len(workers))
for _, worker := range workers {
if worker.record.DeviceIndex == 0 && worker.target.UserID > 0 {
primary = append(primary, worker)
}
}
return primary
}
func runFixedMessageSchedule(ctx context.Context, wg *sync.WaitGroup, startDelay time.Duration, rate float64, workers []*loadWorker, counters *harnessCounters, events *eventWriter) {
defer wg.Done()
if rate <= 0 || len(workers) == 0 {
return
}
startTimer := time.NewTimer(startDelay)
defer startTimer.Stop()
select {
case <-ctx.Done():
return
case <-startTimer.C:
}
readyTicker := time.NewTicker(10 * time.Millisecond)
for countWorkerState(workers, workerReady) != len(workers) {
select {
case <-ctx.Done():
readyTicker.Stop()
return
case <-readyTicker.C:
}
}
readyTicker.Stop()
interval := time.Duration(float64(time.Second) / rate)
if interval < time.Microsecond {
interval = time.Microsecond
}
events.write(map[string]any{"type": "fixed_message_rate_start", "at": time.Now().UTC(), "rate_per_second": rate, "senders": len(workers)})
next := time.Now()
workerIndex := 0
timer := time.NewTimer(0)
defer timer.Stop()
for {
select {
case <-ctx.Done():
events.write(map[string]any{
"type": "fixed_message_rate_stop", "at": time.Now().UTC(),
"scheduled": counters.messageScheduled.Load(), "enqueued": counters.messageEnqueued.Load(),
"queue_full": counters.messageQueueFull.Load(), "not_ready": counters.messageNotReady.Load(),
})
return
case <-timer.C:
worker := workers[workerIndex]
workerIndex = (workerIndex + 1) % len(workers)
counters.messageScheduled.Add(1)
if worker.state.Load() != workerReady {
counters.messageNotReady.Add(1)
} else {
select {
case worker.sendQueue <- struct{}{}:
counters.messageEnqueued.Add(1)
default:
counters.messageQueueFull.Add(1)
}
}
next = next.Add(interval)
timer.Reset(max(time.Until(next), time.Duration(0)))
}
}
}
func reconcileDeliveries(ctx context.Context, workers []*loadWorker) {
waits := make([]chan struct{}, 0, len(workers))
for _, worker := range workers {
if worker.state.Load() != workerReady {
continue
}
done := make(chan struct{})
select {
case <-ctx.Done():
return
case worker.reconcile <- done:
waits = append(waits, done)
}
}
for _, done := range waits {
select {
case <-ctx.Done():
return
case <-done:
}
}
}
func waitMessageDrain(ctx context.Context, counters *harnessCounters) {
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for counters.messageCompleted.Load() < counters.messageEnqueued.Load() {
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func minimumOpenFiles(sessions int) int {
if sessions < 0 {
sessions = 0
@ -901,6 +1211,46 @@ func evaluateReport(report *RunReport, cfg RunConfig) {
if report.WorkerFatalErrors > 0 {
report.Failures = append(report.Failures, fmt.Sprintf("worker fatal errors: %d", report.WorkerFatalErrors))
}
if cfg.MessageRate > 0 {
if report.MessageScheduled == 0 {
report.Failures = append(report.Failures, "fixed-rate message scheduler produced no arrivals")
}
if report.MessageNotReady > 0 {
report.Failures = append(report.Failures, fmt.Sprintf("fixed-rate arrivals rejected because sender was not ready: %d", report.MessageNotReady))
}
if report.MessageQueueFull > 0 {
report.Failures = append(report.Failures, fmt.Sprintf("fixed-rate arrivals rejected by bounded sender queues: %d", report.MessageQueueFull))
}
if report.MessageEnqueued != report.MessageScheduled {
report.Failures = append(report.Failures, fmt.Sprintf("fixed-rate scheduler enqueued %d of %d arrivals", report.MessageEnqueued, report.MessageScheduled))
}
if report.MessageCompleted != report.MessageEnqueued {
report.Failures = append(report.Failures, fmt.Sprintf("message workers completed %d of %d enqueued sends", report.MessageCompleted, report.MessageEnqueued))
}
sendOperation := report.Operations["messages.sendMessage"]
if sendOperation.Count != report.MessageCompleted {
report.Failures = append(report.Failures, fmt.Sprintf("messages.sendMessage recorded %d of %d completed jobs", sendOperation.Count, report.MessageCompleted))
}
successfulSends := sendOperation.Count - min(sendOperation.Count, sendOperation.Errors+sendOperation.Canceled)
if report.Delivery.Expected != successfulSends {
report.Failures = append(report.Failures, fmt.Sprintf("delivery tracker committed %d of %d successful send RPCs", report.Delivery.Expected, successfulSends))
}
if report.Delivery.Missing > 0 || report.Delivery.Delivered != report.Delivery.Expected {
report.Failures = append(report.Failures, fmt.Sprintf("recipient delivery incomplete: delivered %d of %d, missing %d", report.Delivery.Delivered, report.Delivery.Expected, report.Delivery.Missing))
}
if cfg.OfflineFraction == 0 && report.Delivery.DifferenceRecovered > 0 {
report.Failures = append(report.Failures, fmt.Sprintf("online recipients recovered %d messages only through updates.getDifference", report.Delivery.DifferenceRecovered))
}
if report.Delivery.DuplicateObservations > 0 {
report.Failures = append(report.Failures, fmt.Sprintf("recipient observed %d duplicate message updates", report.Delivery.DuplicateObservations))
}
if report.Delivery.WrongAccountObserved > 0 {
report.Failures = append(report.Failures, fmt.Sprintf("load markers appeared on %d wrong recipient accounts", report.Delivery.WrongAccountObserved))
}
if report.Delivery.UnmatchedMarkers > 0 {
report.Failures = append(report.Failures, fmt.Sprintf("observed %d load markers without a successful send RPC", report.Delivery.UnmatchedMarkers))
}
}
for name, operation := range report.Operations {
if operation.FloodWaits > 0 {
report.Failures = append(report.Failures, fmt.Sprintf("%s returned FLOOD_WAIT %d times", name, operation.FloodWaits))
@ -913,6 +1263,34 @@ func evaluateReport(report *RunReport, cfg RunConfig) {
report.Failures = append(report.Failures, fmt.Sprintf("%s returned %d unexpected non-cancel errors", name, unexpectedErrors))
}
}
methods := make([]string, 0, len(report.RPCDeliveryOutcomes))
for method := range report.RPCDeliveryOutcomes {
methods = append(methods, method)
}
sort.Strings(methods)
for _, method := range methods {
outcomes := report.RPCDeliveryOutcomes[method]
outcomeNames := make([]string, 0, len(outcomes))
for outcome := range outcomes {
outcomeNames = append(outcomeNames, outcome)
}
sort.Strings(outcomeNames)
for _, outcome := range outcomeNames {
if count := outcomes[outcome]; outcome != "ok" && count > 0 {
report.Failures = append(report.Failures, fmt.Sprintf("%s rpc_result delivery outcome %s: %d", method, outcome, count))
}
}
}
methods = methods[:0]
for method := range report.DatabaseWork {
methods = append(methods, method)
}
sort.Strings(methods)
for _, method := range methods {
if errors := report.DatabaseWork[method].Errors; errors > 0 {
report.Failures = append(report.Failures, fmt.Sprintf("%s database errors: %d", method, errors))
}
}
if cfg.ExpectServerRestart && report.Reconnects < uint64(requiredReady) {
report.Failures = append(report.Failures, fmt.Sprintf("server restart expected at least %d reconnect attempts, observed %d", requiredReady, report.Reconnects))
}
@ -938,6 +1316,9 @@ func evaluateReport(report *RunReport, cfg RunConfig) {
if strings.TrimSpace(cfg.ServerMetricsURL) != "" && report.FinalServerMetrics == nil {
report.Failures = append(report.Failures, "final post-recovery server metrics scrape failed")
}
if strings.TrimSpace(cfg.ServerMetricsURL) != "" && report.WorkloadEndServerMetrics == nil {
report.Failures = append(report.Failures, "pre-teardown workload-end server metrics scrape failed")
}
if strings.TrimSpace(cfg.ServerMetricsURL) != "" && report.BaselineServerMetrics == nil {
report.Failures = append(report.Failures, "pre-load server metrics baseline scrape failed")
}
@ -945,9 +1326,16 @@ func evaluateReport(report *RunReport, cfg RunConfig) {
}
func metricValue(values map[string]float64, name string) float64 {
// The scraper always stores an aggregate family value in the bare key and
// may additionally retain bounded state/method label breakdowns. Prefer that
// aggregate; summing both would double-count every labeled family in resource
// recovery checks (for example retained/offline logical sessions).
if value, ok := values[name]; ok {
return value
}
var total float64
for key, value := range values {
if key == name || strings.HasPrefix(key, name+"{") {
if strings.HasPrefix(key, name+"{") {
total += value
}
}
@ -1032,6 +1420,8 @@ func classifyErrorReason(err error) string {
return "dns"
case strings.Contains(message, "BROKEN PIPE"):
return "broken_pipe"
case strings.Contains(message, "ENDED BEFORE BUSINESS READINESS"):
return "business_readiness_incomplete"
case strings.Contains(message, "EOF"):
return "eof"
case errors.Is(err, context.DeadlineExceeded):

View file

@ -0,0 +1,714 @@
package loadharness
import (
"context"
"crypto/rsa"
"errors"
"fmt"
"sort"
"sync"
"time"
"github.com/iamxvbaba/td/tg"
"github.com/iamxvbaba/td/tgerr"
)
const maxSeedInviteBatch = 200
type SeedConfig struct {
ManifestPath string
SessionKeyPath string
RSAKeyOverride string
DatasetPath string
SeedStatePath string
Concurrency int
OperationTimeout time.Duration
}
type SeedEvent struct {
Phase string
Completed int
Total int
Account int
Err error
}
type SeedResult struct {
PrivateMessages int
Groups int
InvitedMembers int
GroupMessages int
RichStateAccounts int
}
func (c SeedConfig) validate() error {
if c.ManifestPath == "" || c.SessionKeyPath == "" || c.DatasetPath == "" || c.SeedStatePath == "" {
return errors.New("manifest, session-key, dataset and seed-state paths are required")
}
if c.Concurrency <= 0 || c.Concurrency > 64 {
return errors.New("seed concurrency must be between 1 and 64")
}
if c.OperationTimeout <= 0 {
return errors.New("seed operation timeout must be positive")
}
return nil
}
// Seed materializes a Dataset only through authenticated MTProto RPCs. It never
// calls a server-internal handler or store. The journal is persisted before any
// non-idempotent channel operation, so an interrupted run can reconcile using
// the same account's public RPC view before it proceeds.
func Seed(ctx context.Context, cfg SeedConfig, progress func(SeedEvent)) (*SeedResult, error) {
if err := cfg.validate(); err != nil {
return nil, err
}
manifest, err := LoadManifest(cfg.ManifestPath)
if err != nil {
return nil, err
}
dataset, err := LoadDataset(cfg.DatasetPath)
if err != nil {
return nil, err
}
targets, err := seedPrimaryTargets(manifest, dataset.Config.Accounts)
if err != nil {
return nil, err
}
key, err := LoadSessionKey(cfg.SessionKeyPath)
if err != nil {
return nil, err
}
publicKey, err := loadManifestPublicKey(cfg.ManifestPath, manifest.Endpoint, cfg.RSAKeyOverride)
if err != nil {
return nil, err
}
state, err := LoadDatasetSeedState(cfg.SeedStatePath, dataset)
if err != nil {
return nil, err
}
journal := &seedJournal{path: cfg.SeedStatePath, dataset: dataset, state: state}
if err := journal.enableRichState(); err != nil {
return nil, err
}
if err := journal.persist(); err != nil {
return nil, err
}
accounts := make([]int, dataset.Config.Accounts)
for account := range accounts {
accounts[account] = account
}
if err := runSeedAccountPhase(ctx, "private", accounts, cfg.Concurrency, progress, func(ctx context.Context, account int) error {
return seedPrivateMessages(ctx, cfg, manifest, dataset, journal, targets, key, publicKey, account)
}); err != nil {
return nil, err
}
groupsByCreator := make(map[int][]int)
for position, group := range dataset.Groups {
groupsByCreator[group.CreatorAccount] = append(groupsByCreator[group.CreatorAccount], position)
}
creators := make([]int, 0, len(groupsByCreator))
for account := range groupsByCreator {
creators = append(creators, account)
}
sort.Ints(creators)
if err := runSeedAccountPhase(ctx, "groups", creators, cfg.Concurrency, progress, func(ctx context.Context, account int) error {
return withAuthorizedSeedSession(ctx, cfg, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
for _, position := range groupsByCreator[account] {
if err := seedGroup(ctx, cfg, dataset, journal, targets, raw, position); err != nil {
return fmt.Errorf("group %d: %w", dataset.Groups[position].Index, err)
}
}
return nil
})
}); err != nil {
return nil, err
}
historyTasks := datasetHistoryTasks(dataset)
if err := runSeedAccountPhase(ctx, "group-history", accounts, cfg.Concurrency, progress, func(ctx context.Context, account int) error {
return seedGroupHistory(ctx, cfg, manifest, dataset, journal, targets, key, publicKey, account, historyTasks[account])
}); err != nil {
return nil, err
}
if err := runSeedAccountPhase(ctx, "rich-state", accounts, cfg.Concurrency, progress, func(ctx context.Context, account int) error {
return seedRichAccountState(ctx, cfg, manifest, dataset, journal, targets, key, publicKey, account)
}); err != nil {
return nil, err
}
if err := journal.assertComplete(); err != nil {
return nil, err
}
result := &SeedResult{PrivateMessages: len(dataset.PrivateEdges), Groups: len(dataset.Groups), RichStateAccounts: dataset.Config.Accounts}
for _, group := range dataset.Groups {
result.InvitedMembers += len(group.MemberAccounts) - 1
result.GroupMessages += group.HistoryMessages
}
return result, nil
}
func seedPrimaryTargets(manifest *Manifest, accounts int) ([]SessionRecord, error) {
targets := primaryTargets(manifest.Sessions)
if len(targets) < accounts {
return nil, fmt.Errorf("dataset requires %d primary accounts, manifest has account range 0..%d", accounts, len(targets)-1)
}
targets = targets[:accounts]
for account, target := range targets {
if target.AccountIndex != account || target.UserID <= 0 || target.AccessHash == 0 || target.SessionFile == "" {
return nil, fmt.Errorf("manifest has no complete primary session for account %d", account)
}
}
return targets, nil
}
func seedPrivateMessages(
ctx context.Context,
cfg SeedConfig,
manifest *Manifest,
dataset *Dataset,
journal *seedJournal,
targets []SessionRecord,
key [32]byte,
publicKey *rsa.PublicKey,
account int,
) error {
cursor := journal.privateCursor(account)
if cursor == dataset.Config.PrivateFanout {
return nil
}
start := account * dataset.Config.PrivateFanout
edges := dataset.PrivateEdges[start : start+dataset.Config.PrivateFanout]
return withAuthorizedSeedSession(ctx, cfg, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
for i := cursor; i < len(edges); i++ {
edge := edges[i]
target := targets[edge.RecipientAccount]
_, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
return raw.MessagesSendMessage(rpcCtx, &tg.MessagesSendMessageRequest{
Peer: &tg.InputPeerUser{UserID: target.UserID, AccessHash: target.AccessHash},
Message: edge.Marker, RandomID: edge.RandomID,
})
})
if err != nil {
return fmt.Errorf("messages.sendMessage edge %d: %w", i, err)
}
}
return journal.setPrivateCursor(account, len(edges))
})
}
func seedGroup(
ctx context.Context,
cfg SeedConfig,
dataset *Dataset,
journal *seedJournal,
targets []SessionRecord,
raw *tg.Client,
position int,
) error {
group := dataset.Groups[position]
groupState := journal.group(position)
if groupState.ChannelID == 0 && groupState.CreatePending {
channel, found, err := reconcilePendingChannelCreate(ctx, cfg.OperationTimeout, raw, group)
if err != nil {
return err
}
if found {
if err := journal.commitChannel(position, channel.ID, channel.AccessHash); err != nil {
return err
}
} else if err := journal.clearCreatePending(position); err != nil {
return err
}
groupState = journal.group(position)
}
if groupState.ChannelID == 0 {
if err := journal.beginCreate(position); err != nil {
return err
}
updates, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
return raw.ChannelsCreateChannel(rpcCtx, &tg.ChannelsCreateChannelRequest{
Megagroup: true, Title: group.Title, About: group.About,
})
})
if err != nil {
return fmt.Errorf("channels.createChannel pending reconciliation: %w", err)
}
channel, err := createdChannelFromUpdates(updates, group.Title)
if err != nil {
return err
}
if err := journal.commitChannel(position, channel.ID, channel.AccessHash); err != nil {
return err
}
groupState = journal.group(position)
}
invitees := groupInvitees(group)
for groupState.InviteCursor < len(invitees) {
if groupState.InvitePendingEnd > groupState.InviteCursor {
if err := reconcilePendingInvites(ctx, cfg.OperationTimeout, raw, groupState, invitees, targets); err != nil {
return err
}
if err := journal.commitInvite(position, groupState.InvitePendingEnd); err != nil {
return err
}
groupState = journal.group(position)
continue
}
end := min(groupState.InviteCursor+maxSeedInviteBatch, len(invitees))
if err := journal.beginInvite(position, end); err != nil {
return err
}
if err := inviteAccounts(ctx, cfg.OperationTimeout, raw, groupState, invitees[groupState.InviteCursor:end], targets); err != nil {
return fmt.Errorf("channels.inviteToChannel pending reconciliation: %w", err)
}
if err := journal.commitInvite(position, end); err != nil {
return err
}
groupState = journal.group(position)
}
return nil
}
func reconcilePendingChannelCreate(ctx context.Context, timeout time.Duration, raw *tg.Client, group DatasetGroup) (*tg.Channel, bool, error) {
rpcCtx, cancel := context.WithTimeout(ctx, timeout)
dialogs, err := raw.MessagesGetDialogs(rpcCtx, &tg.MessagesGetDialogsRequest{
OffsetPeer: &tg.InputPeerEmpty{}, Limit: 500,
})
cancel()
if err != nil {
return nil, false, fmt.Errorf("reconcile create with messages.getDialogs: %w", err)
}
var chats []tg.ChatClass
switch value := dialogs.(type) {
case *tg.MessagesDialogs:
chats = value.Chats
case *tg.MessagesDialogsSlice:
chats = value.Chats
case *tg.MessagesDialogsNotModified:
return nil, false, errors.New("reconcile create unexpectedly returned dialogsNotModified")
default:
return nil, false, fmt.Errorf("reconcile create messages.getDialogs returned %T", dialogs)
}
matches := make([]*tg.Channel, 0, 1)
for _, chat := range chats {
channel, ok := chat.(*tg.Channel)
if !ok || channel.Title != group.Title || !channel.Megagroup {
continue
}
if channel.ID <= 0 || channel.AccessHash == 0 {
return nil, false, fmt.Errorf("reconciled channel %q has incomplete identity", group.Title)
}
matches = append(matches, channel)
}
if len(matches) > 1 {
return nil, false, fmt.Errorf("ambiguous create produced %d channels named %q", len(matches), group.Title)
}
if len(matches) == 0 {
return nil, false, nil
}
return matches[0], true, nil
}
func createdChannelFromUpdates(updates tg.UpdatesClass, title string) (*tg.Channel, error) {
var chats []tg.ChatClass
switch value := updates.(type) {
case *tg.Updates:
chats = value.Chats
case *tg.UpdatesCombined:
chats = value.Chats
default:
return nil, fmt.Errorf("channels.createChannel returned %T", updates)
}
for _, chat := range chats {
channel, ok := chat.(*tg.Channel)
if ok && channel.Title == title && channel.Megagroup && channel.ID > 0 && channel.AccessHash != 0 {
return channel, nil
}
}
return nil, fmt.Errorf("channels.createChannel response omitted supergroup %q", title)
}
func groupInvitees(group DatasetGroup) []int {
invitees := make([]int, 0, len(group.MemberAccounts)-1)
for _, account := range group.MemberAccounts {
if account != group.CreatorAccount {
invitees = append(invitees, account)
}
}
return invitees
}
func inviteAccounts(
ctx context.Context,
timeout time.Duration,
raw *tg.Client,
groupState DatasetSeedGroupState,
accounts []int,
targets []SessionRecord,
) error {
if len(accounts) == 0 || len(accounts) > maxSeedInviteBatch {
return fmt.Errorf("invalid invite batch size %d", len(accounts))
}
users := make([]tg.InputUserClass, 0, len(accounts))
for _, account := range accounts {
target := targets[account]
users = append(users, &tg.InputUser{UserID: target.UserID, AccessHash: target.AccessHash})
}
result, err := rpcWithFloodWaitRetry(ctx, timeout, func(rpcCtx context.Context) (*tg.MessagesInvitedUsers, error) {
return raw.ChannelsInviteToChannel(rpcCtx, &tg.ChannelsInviteToChannelRequest{
Channel: &tg.InputChannel{ChannelID: groupState.ChannelID, AccessHash: groupState.AccessHash},
Users: users,
})
})
if err != nil {
return err
}
if len(result.MissingInvitees) != 0 {
return fmt.Errorf("server reported %d missing_invitees", len(result.MissingInvitees))
}
return nil
}
func reconcilePendingInvites(
ctx context.Context,
timeout time.Duration,
raw *tg.Client,
groupState DatasetSeedGroupState,
invitees []int,
targets []SessionRecord,
) error {
pending := invitees[groupState.InviteCursor:groupState.InvitePendingEnd]
missing := make([]int, 0, len(pending))
for _, account := range pending {
target := targets[account]
rpcCtx, cancel := context.WithTimeout(ctx, timeout)
_, err := raw.ChannelsGetParticipant(rpcCtx, &tg.ChannelsGetParticipantRequest{
Channel: &tg.InputChannel{ChannelID: groupState.ChannelID, AccessHash: groupState.AccessHash},
Participant: &tg.InputPeerUser{UserID: target.UserID, AccessHash: target.AccessHash},
})
cancel()
switch {
case err == nil:
case tgerr.Is(err, "USER_NOT_PARTICIPANT"):
missing = append(missing, account)
default:
return fmt.Errorf("channels.getParticipant account %d: %w", account, err)
}
}
if len(missing) == 0 {
return nil
}
return inviteAccounts(ctx, timeout, raw, groupState, missing, targets)
}
type datasetHistoryTask struct {
GroupPosition int
MessageIndex int
}
func datasetHistoryTasks(dataset *Dataset) [][]datasetHistoryTask {
tasks := make([][]datasetHistoryTask, dataset.Config.Accounts)
for position, group := range dataset.Groups {
for message := 0; message < group.HistoryMessages; message++ {
account := group.MemberAccounts[message%len(group.MemberAccounts)]
tasks[account] = append(tasks[account], datasetHistoryTask{GroupPosition: position, MessageIndex: message})
}
}
return tasks
}
func seedGroupHistory(
ctx context.Context,
cfg SeedConfig,
manifest *Manifest,
dataset *Dataset,
journal *seedJournal,
targets []SessionRecord,
key [32]byte,
publicKey *rsa.PublicKey,
account int,
tasks []datasetHistoryTask,
) error {
cursor := journal.historyCursor(account)
if cursor == len(tasks) {
return nil
}
return withAuthorizedSeedSession(ctx, cfg, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
for i := cursor; i < len(tasks); i++ {
task := tasks[i]
group := dataset.Groups[task.GroupPosition]
groupState := journal.group(task.GroupPosition)
if groupState.ChannelID == 0 || groupState.InviteCursor != len(group.MemberAccounts)-1 {
return fmt.Errorf("group %d is not fully seeded", group.Index)
}
_, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
return raw.MessagesSendMessage(rpcCtx, &tg.MessagesSendMessageRequest{
Peer: &tg.InputPeerChannel{ChannelID: groupState.ChannelID, AccessHash: groupState.AccessHash},
Message: fmt.Sprintf("[%s group %04d message %04d sender %04d]", dataset.RunID, group.Index, task.MessageIndex+1, account),
RandomID: stableDatasetID(dataset.Config.Seed, "group-history", dataset.Config.Accounts, group.Index, task.MessageIndex),
})
})
if err != nil {
return fmt.Errorf("messages.sendMessage group %d message %d: %w", group.Index, task.MessageIndex, err)
}
}
return journal.setHistoryCursor(account, len(tasks))
})
}
func withAuthorizedSeedSession(
ctx context.Context,
cfg SeedConfig,
manifest *Manifest,
record SessionRecord,
key [32]byte,
publicKey *rsa.PublicKey,
work func(context.Context, *tg.Client) error,
) error {
storage := &EncryptedFileStorage{Path: resolveSessionPath(cfg.ManifestPath, record), Key: key}
client, err := newClient(manifest.Endpoint, publicKey, storage, clientHooks{})
if err != nil {
return err
}
return client.Run(ctx, func(ctx context.Context) error {
statusCtx, cancel := context.WithTimeout(ctx, cfg.OperationTimeout)
status, err := client.Auth().Status(statusCtx)
cancel()
if err != nil {
return fmt.Errorf("authorization status: %w", err)
}
if !status.Authorized || status.User == nil || status.User.ID != record.UserID {
return fmt.Errorf("session account %d is not authorized as expected user", record.AccountIndex)
}
return work(ctx, tg.NewClient(client))
})
}
type seedAccountResult struct {
account int
err error
}
func runSeedAccountPhase(
ctx context.Context,
phase string,
accounts []int,
concurrency int,
progress func(SeedEvent),
work func(context.Context, int) error,
) error {
phaseCtx, cancel := context.WithCancel(ctx)
defer cancel()
jobs := make(chan int)
results := make(chan seedAccountResult, len(accounts))
workers := min(concurrency, len(accounts))
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for account := range jobs {
err := work(phaseCtx, account)
results <- seedAccountResult{account: account, err: err}
if err != nil {
return
}
}
}()
}
go func() {
defer close(jobs)
for _, account := range accounts {
select {
case jobs <- account:
case <-phaseCtx.Done():
return
}
}
}()
go func() {
wg.Wait()
close(results)
}()
completed := 0
var firstErr error
for result := range results {
if result.err == nil {
completed++
} else if firstErr == nil {
firstErr = fmt.Errorf("%s account %d: %w", phase, result.account, result.err)
cancel()
}
if progress != nil {
progress(SeedEvent{Phase: phase, Completed: completed, Total: len(accounts), Account: result.account, Err: result.err})
}
}
if firstErr != nil {
return firstErr
}
if completed != len(accounts) {
if ctx.Err() != nil {
return ctx.Err()
}
return fmt.Errorf("%s completed %d/%d accounts", phase, completed, len(accounts))
}
return nil
}
type seedJournal struct {
mu sync.Mutex
path string
dataset *Dataset
state *DatasetSeedState
}
func (j *seedJournal) persist() error {
j.mu.Lock()
defer j.mu.Unlock()
return WriteDatasetSeedState(j.path, j.dataset, j.state)
}
func (j *seedJournal) privateCursor(account int) int {
j.mu.Lock()
defer j.mu.Unlock()
return j.state.PrivateSentByAccount[account]
}
func (j *seedJournal) historyCursor(account int) int {
j.mu.Lock()
defer j.mu.Unlock()
return j.state.HistorySentByAccount[account]
}
func (j *seedJournal) richStateComplete(account int) bool {
j.mu.Lock()
defer j.mu.Unlock()
return len(j.state.RichStateByAccount) == j.dataset.Config.Accounts && j.state.RichStateByAccount[account]
}
func (j *seedJournal) enableRichState() error {
j.mu.Lock()
defer j.mu.Unlock()
if len(j.state.RichStateByAccount) == j.dataset.Config.Accounts {
return nil
}
if len(j.state.RichStateByAccount) != 0 {
return errors.New("seed rich-state journal has invalid dimensions")
}
j.state.RichStateByAccount = make([]bool, j.dataset.Config.Accounts)
return WriteDatasetSeedState(j.path, j.dataset, j.state)
}
func (j *seedJournal) group(position int) DatasetSeedGroupState {
j.mu.Lock()
defer j.mu.Unlock()
return j.state.Groups[position]
}
func (j *seedJournal) setPrivateCursor(account, cursor int) error {
j.mu.Lock()
defer j.mu.Unlock()
old := j.state.PrivateSentByAccount[account]
j.state.PrivateSentByAccount[account] = cursor
if err := WriteDatasetSeedState(j.path, j.dataset, j.state); err != nil {
j.state.PrivateSentByAccount[account] = old
return err
}
return nil
}
func (j *seedJournal) setHistoryCursor(account, cursor int) error {
j.mu.Lock()
defer j.mu.Unlock()
old := j.state.HistorySentByAccount[account]
j.state.HistorySentByAccount[account] = cursor
if err := WriteDatasetSeedState(j.path, j.dataset, j.state); err != nil {
j.state.HistorySentByAccount[account] = old
return err
}
return nil
}
func (j *seedJournal) setRichStateComplete(account int) error {
j.mu.Lock()
defer j.mu.Unlock()
if len(j.state.RichStateByAccount) != j.dataset.Config.Accounts {
return errors.New("seed rich-state journal is not enabled")
}
old := j.state.RichStateByAccount[account]
j.state.RichStateByAccount[account] = true
if err := WriteDatasetSeedState(j.path, j.dataset, j.state); err != nil {
j.state.RichStateByAccount[account] = old
return err
}
return nil
}
func (j *seedJournal) updateGroup(position int, update func(*DatasetSeedGroupState)) error {
j.mu.Lock()
defer j.mu.Unlock()
old := j.state.Groups[position]
update(&j.state.Groups[position])
if err := WriteDatasetSeedState(j.path, j.dataset, j.state); err != nil {
j.state.Groups[position] = old
return err
}
return nil
}
func (j *seedJournal) beginCreate(position int) error {
return j.updateGroup(position, func(state *DatasetSeedGroupState) { state.CreatePending = true })
}
func (j *seedJournal) clearCreatePending(position int) error {
return j.updateGroup(position, func(state *DatasetSeedGroupState) { state.CreatePending = false })
}
func (j *seedJournal) commitChannel(position int, channelID, accessHash int64) error {
if channelID <= 0 || accessHash == 0 {
return errors.New("cannot commit incomplete channel identity")
}
return j.updateGroup(position, func(state *DatasetSeedGroupState) {
state.ChannelID = channelID
state.AccessHash = accessHash
state.CreatePending = false
})
}
func (j *seedJournal) beginInvite(position, end int) error {
return j.updateGroup(position, func(state *DatasetSeedGroupState) { state.InvitePendingEnd = end })
}
func (j *seedJournal) commitInvite(position, end int) error {
return j.updateGroup(position, func(state *DatasetSeedGroupState) {
state.InviteCursor = end
state.InvitePendingEnd = end
})
}
func (j *seedJournal) assertComplete() error {
j.mu.Lock()
defer j.mu.Unlock()
if err := j.state.Validate(j.dataset); err != nil {
return err
}
historyCounts := datasetHistoryTaskCounts(j.dataset)
for account := 0; account < j.dataset.Config.Accounts; account++ {
if j.state.PrivateSentByAccount[account] != j.dataset.Config.PrivateFanout || j.state.HistorySentByAccount[account] != historyCounts[account] {
return fmt.Errorf("account %d seed is incomplete", account)
}
if len(j.state.RichStateByAccount) != 0 && !j.state.RichStateByAccount[account] {
return fmt.Errorf("account %d rich state is incomplete", account)
}
}
for position, group := range j.dataset.Groups {
state := j.state.Groups[position]
if state.ChannelID == 0 || state.CreatePending || state.InviteCursor != len(group.MemberAccounts)-1 || state.InvitePendingEnd != state.InviteCursor {
return fmt.Errorf("group %d seed is incomplete", group.Index)
}
}
return nil
}

View file

@ -0,0 +1,177 @@
package loadharness
import (
"context"
"errors"
"path/filepath"
"reflect"
"testing"
"github.com/iamxvbaba/td/tg"
)
func TestSeedPrimaryTargetsSelectsPrimaryAndRejectsGap(t *testing.T) {
manifest := &Manifest{Sessions: []SessionRecord{
{Index: 2, AccountIndex: 0, DeviceIndex: 1, SessionFile: "extra", UserID: 10, AccessHash: 100},
{Index: 0, AccountIndex: 0, DeviceIndex: 0, SessionFile: "primary-0", UserID: 10, AccessHash: 100},
{Index: 1, AccountIndex: 1, DeviceIndex: 0, SessionFile: "primary-1", UserID: 11, AccessHash: 101},
}}
targets, err := seedPrimaryTargets(manifest, 2)
if err != nil {
t.Fatal(err)
}
if targets[0].SessionFile != "primary-0" || targets[1].SessionFile != "primary-1" {
t.Fatalf("primary targets = %+v", targets)
}
manifest.Sessions = append(manifest.Sessions, SessionRecord{
Index: 3, AccountIndex: 2, DeviceIndex: 0, SessionFile: "primary-2", UserID: 12, AccessHash: 102,
})
if targets, err := seedPrimaryTargets(manifest, 2); err != nil || len(targets) != 2 {
t.Fatalf("manifest superset targets=%d err=%v", len(targets), err)
}
manifest.Sessions = manifest.Sessions[:2]
if _, err := seedPrimaryTargets(manifest, 2); err == nil {
t.Fatal("manifest account gap passed validation")
}
}
func TestPlanRichAccountStateUsesDeterministicPrivateAndGroupPeers(t *testing.T) {
dataset, _, _ := snapshotFixture(t)
plan, err := planRichAccountState(dataset, 0)
if err != nil {
t.Fatal(err)
}
if plan.PinnedPeerAccount != 1 || plan.ReadPeerAccount != 3 || plan.ReadGroupPosition != 0 || plan.DraftMarker == "" {
t.Fatalf("rich plan = %+v", plan)
}
}
func TestValidateSeededRichDialogs(t *testing.T) {
dataset, seedState, targets := snapshotFixture(t)
seedState.RichStateByAccount = make([]bool, dataset.Config.Accounts)
seedState.RichStateByAccount[0] = true
plan, err := planRichAccountState(dataset, 0)
if err != nil {
t.Fatal(err)
}
dialogs := []ClientDialogState{
{PeerType: "user", PeerID: targets[plan.PinnedPeerAccount].UserID, Pinned: true, HasDraft: true, DraftText: plan.DraftMarker},
{PeerType: "user", PeerID: targets[plan.ReadPeerAccount].UserID, TopMessage: 8, ReadInboxMaxID: 8},
{PeerType: "channel", PeerID: seedState.Groups[plan.ReadGroupPosition].ChannelID, TopMessage: 9, ReadInboxMaxID: 9},
}
if err := validateSeededRichDialogs(dataset, seedState, targets, 0, dialogs, true); err != nil {
t.Fatal(err)
}
dialogs[0].DraftText = "wrong"
if err := validateSeededRichDialogs(dataset, seedState, targets, 0, dialogs, true); err == nil {
t.Fatal("wrong draft marker passed rich-state validation")
}
}
func TestCreatedChannelFromUpdates(t *testing.T) {
channel := &tg.Channel{ID: 41, AccessHash: 42, Title: "target", Megagroup: true}
got, err := createdChannelFromUpdates(&tg.Updates{Chats: []tg.ChatClass{
&tg.Chat{ID: 1, Title: "other"}, channel,
}}, "target")
if err != nil {
t.Fatal(err)
}
if got != channel {
t.Fatalf("created channel = %#v, want target", got)
}
if _, err := createdChannelFromUpdates(&tg.UpdateShort{}, "target"); err == nil {
t.Fatal("unexpected create response passed validation")
}
if _, err := createdChannelFromUpdates(&tg.UpdatesCombined{Chats: []tg.ChatClass{
&tg.Channel{ID: 41, AccessHash: 42, Title: "target"},
}}, "target"); err == nil {
t.Fatal("broadcast/non-megagroup response passed validation")
}
}
func TestDatasetHistoryTasksRotateSenders(t *testing.T) {
cfg := DefaultDatasetConfig(20)
cfg.HotGroups, cfg.MediumGroups, cfg.HeavyGroups = 0, 0, 0
cfg.SmallGroups, cfg.SmallMembers, cfg.SmallHistory = 1, 4, 9
dataset, err := PlanDataset(cfg)
if err != nil {
t.Fatal(err)
}
tasks := datasetHistoryTasks(dataset)
wantCounts := datasetHistoryTaskCounts(dataset)
gotCounts := make([]int, len(tasks))
for account := range tasks {
gotCounts[account] = len(tasks[account])
for _, task := range tasks[account] {
group := dataset.Groups[task.GroupPosition]
if got := group.MemberAccounts[task.MessageIndex%len(group.MemberAccounts)]; got != account {
t.Fatalf("message %d sender = %d, task account %d", task.MessageIndex, got, account)
}
}
}
if !reflect.DeepEqual(gotCounts, wantCounts) {
t.Fatalf("history task counts = %v, want %v", gotCounts, wantCounts)
}
}
func TestSeedJournalPersistsPendingBoundaries(t *testing.T) {
dataset, err := PlanDataset(DefaultDatasetConfig(20))
if err != nil {
t.Fatal(err)
}
state, err := NewDatasetSeedState(dataset)
if err != nil {
t.Fatal(err)
}
path := filepath.Join(t.TempDir(), "seed-state.json")
journal := &seedJournal{path: path, dataset: dataset, state: state}
if err := journal.persist(); err != nil {
t.Fatal(err)
}
if err := journal.beginCreate(0); err != nil {
t.Fatal(err)
}
loaded, err := LoadDatasetSeedState(path, dataset)
if err != nil {
t.Fatal(err)
}
if !loaded.Groups[0].CreatePending || loaded.Groups[0].ChannelID != 0 {
t.Fatalf("pending create state = %+v", loaded.Groups[0])
}
if err := journal.commitChannel(0, 101, 202); err != nil {
t.Fatal(err)
}
if err := journal.beginInvite(0, 7); err != nil {
t.Fatal(err)
}
loaded, err = LoadDatasetSeedState(path, dataset)
if err != nil {
t.Fatal(err)
}
if loaded.Groups[0].InviteCursor != 0 || loaded.Groups[0].InvitePendingEnd != 7 {
t.Fatalf("pending invite state = %+v", loaded.Groups[0])
}
if err := journal.commitInvite(0, 7); err != nil {
t.Fatal(err)
}
loaded, err = LoadDatasetSeedState(path, dataset)
if err != nil {
t.Fatal(err)
}
if loaded.Groups[0].InviteCursor != 7 || loaded.Groups[0].InvitePendingEnd != 7 {
t.Fatalf("committed invite state = %+v", loaded.Groups[0])
}
}
func TestRunSeedAccountPhaseStopsAfterFailure(t *testing.T) {
want := errors.New("stop")
err := runSeedAccountPhase(context.Background(), "test", []int{0, 1, 2}, 1, nil, func(_ context.Context, account int) error {
if account == 1 {
return want
}
return nil
})
if !errors.Is(err, want) {
t.Fatalf("phase error = %v, want wrapped stop", err)
}
}

View file

@ -16,43 +16,88 @@ import (
const maxServerMetricsBytes = 4 << 20
var selectedServerMetrics = map[string]struct{}{
"telesrv_mtproto_raw_connections": {},
"telesrv_mtproto_sessions": {},
"telesrv_mtproto_logical_sessions": {},
"telesrv_mtproto_logical_outbox_frames": {},
"telesrv_mtproto_logical_outbox_bytes": {},
"telesrv_mtproto_logical_outbox_acked_frames_total": {},
"telesrv_mtproto_logical_outbox_acked_bytes_total": {},
"telesrv_mtproto_logical_outbox_retained_seconds_count": {},
"telesrv_mtproto_logical_outbox_retained_seconds_sum": {},
"telesrv_mtproto_pending_push_bytes": {},
"telesrv_mtproto_inbound_rpc_tasks": {},
"telesrv_mtproto_inbound_rpc_bytes": {},
"telesrv_mtproto_inbound_frame_bytes": {},
"telesrv_mtproto_outbound_tracked_bytes": {},
"telesrv_mtproto_outbound_write_bytes": {},
"telesrv_mtproto_rpc_execution_owners": {},
"telesrv_mtproto_rpc_execution_reserved_entries": {},
"telesrv_mtproto_rpc_execution_receipts": {},
"telesrv_mtproto_rpc_execution_receipt_budget_bytes": {},
"telesrv_mtproto_rpc_execution_subscribers": {},
"telesrv_mtproto_rpc_result_inner_bytes_total": {},
"telesrv_mtproto_rpc_result_wire_bytes_total": {},
"telesrv_mtproto_rpc_result_delivered_bytes_total": {},
"telesrv_go_goroutines": {},
"telesrv_go_heap_alloc_bytes": {},
"telesrv_go_heap_inuse_bytes": {},
"telesrv_go_heap_objects": {},
"telesrv_go_sys_bytes": {},
"telesrv_postgres_pool_connections": {},
"telesrv_postgres_pool_acquire_wait_seconds": {},
"telesrv_postgres_pool_empty_acquire_count": {},
"telesrv_postgres_pool_canceled_acquire_count": {},
"telesrv_redis_pool_connections": {},
"telesrv_redis_pool_pending_requests": {},
"telesrv_redis_pool_timeouts": {},
"telesrv_redis_pool_wait_seconds": {},
"telesrv_metrics_dropped_observations_total": {},
"telesrv_mtproto_raw_connections": {},
"telesrv_mtproto_connections_active": {},
"telesrv_mtproto_sessions": {},
"telesrv_mtproto_logical_sessions": {},
"telesrv_mtproto_logical_outbox_frames": {},
"telesrv_mtproto_logical_outbox_bytes": {},
"telesrv_mtproto_logical_outbox_acked_frames_total": {},
"telesrv_mtproto_logical_outbox_acked_bytes_total": {},
"telesrv_mtproto_logical_outbox_retained_seconds_count": {},
"telesrv_mtproto_logical_outbox_retained_seconds_sum": {},
"telesrv_mtproto_pending_push_bytes": {},
"telesrv_mtproto_inbound_rpc_tasks": {},
"telesrv_mtproto_inbound_rpc_bytes": {},
"telesrv_mtproto_rpc_delivery_hook_workers": {},
"telesrv_mtproto_rpc_delivery_hook_capacity": {},
"telesrv_mtproto_rpc_delivery_hook_reserved": {},
"telesrv_mtproto_rpc_delivery_hook_queued": {},
"telesrv_mtproto_rpc_delivery_hook_running": {},
"telesrv_mtproto_rpc_delivery_hook_completed_total": {},
"telesrv_mtproto_rpc_delivery_hook_rejected_total": {},
"telesrv_mtproto_rpc_delivery_hook_panics_total": {},
"telesrv_mtproto_rpc_delivery_hook_duration_seconds_total": {},
"telesrv_mtproto_inbound_frame_bytes": {},
"telesrv_mtproto_outbound_tracked_bytes": {},
"telesrv_mtproto_outbound_write_bytes": {},
"telesrv_mtproto_rpc_execution_owners": {},
"telesrv_mtproto_rpc_execution_reserved_entries": {},
"telesrv_mtproto_rpc_execution_receipts": {},
"telesrv_mtproto_rpc_execution_receipt_budget_bytes": {},
"telesrv_mtproto_rpc_execution_subscribers": {},
"telesrv_mtproto_rpc_result_inner_bytes_total": {},
"telesrv_mtproto_rpc_result_wire_bytes_total": {},
"telesrv_mtproto_rpc_result_delivered_total": {},
"telesrv_mtproto_rpc_result_delivered_bytes_total": {},
"telesrv_go_goroutines": {},
"telesrv_process_cpu_seconds": {},
"telesrv_go_scheduler_busy_seconds": {},
"telesrv_go_gc_cycles": {},
"telesrv_go_gc_pause_seconds": {},
"telesrv_go_heap_alloc_bytes": {},
"telesrv_go_heap_inuse_bytes": {},
"telesrv_go_heap_objects": {},
"telesrv_go_stack_inuse_bytes": {},
"telesrv_go_sys_bytes": {},
"telesrv_postgres_pool_connections": {},
"telesrv_postgres_pool_acquire_count": {},
"telesrv_postgres_pool_acquire_wait_seconds": {},
"telesrv_postgres_pool_empty_acquire_count": {},
"telesrv_postgres_pool_canceled_acquire_count": {},
"telesrv_postgres_pool_max_connections": {},
"telesrv_redis_pool_connections": {},
"telesrv_redis_pool_hits": {},
"telesrv_redis_pool_misses": {},
"telesrv_redis_pool_pending_requests": {},
"telesrv_redis_pool_timeouts": {},
"telesrv_redis_pool_wait_count": {},
"telesrv_redis_pool_wait_seconds": {},
"telesrv_rpc_db_queries_total": {},
"telesrv_rpc_db_errors_total": {},
"telesrv_rpc_db_time_seconds_sum": {},
"telesrv_rpc_db_time_seconds_count": {},
"telesrv_channel_difference_cache_entries": {},
"telesrv_channel_difference_cache_weight_bytes": {},
"telesrv_channel_difference_cache_hits": {},
"telesrv_channel_difference_cache_misses": {},
"telesrv_channel_difference_cache_loads": {},
"telesrv_channel_difference_cache_load_errors": {},
"telesrv_bootstrap_ready_batches_total": {},
"telesrv_bootstrap_ready_selectors_total": {},
"telesrv_bootstrap_ready_pending": {},
"telesrv_active_channel_ids_cache_total": {},
"telesrv_active_channel_ids_batches_total": {},
"telesrv_active_channel_ids_selectors_total": {},
"telesrv_active_channel_ids_rows_total": {},
"telesrv_active_channel_ids_pending": {},
"telesrv_presence_last_seen_batches_total": {},
"telesrv_presence_last_seen_updates_total": {},
"telesrv_presence_last_seen_submitted_total": {},
"telesrv_presence_last_seen_pending": {},
"telesrv_presence_last_seen_overflow_total": {},
"telesrv_presence_last_seen_drain_dropped_total": {},
"telesrv_metrics_dropped_observations_total": {},
}
type serverMetricsClient struct {
@ -113,10 +158,31 @@ func (c *serverMetricsClient) scrape(ctx context.Context) (map[string]float64, e
}
// Reports need bounded, comparable capacity signals, not an unbounded copy
// of Prometheus label series. Aggregate every selected family into one
// key so method/encoding cardinality can never starve later gauges (the
// endpoint orders counters before gauges). The source /metrics endpoint
// retains full labels for detailed diagnosis.
// key. Response-byte and DB-work families additionally retain only their
// code-owned method label; bounded pool/session families retain their state
// label. This supports attribution without copying auth/session/user
// cardinality.
values[name] += value
if isPerMethodOutcomeServerMetric(name) {
method, methodOK := prometheusLabelValue(fields[0], "method")
outcome, outcomeOK := prometheusLabelValue(fields[0], "outcome")
if methodOK && outcomeOK {
values[name+`{method="`+method+`",outcome="`+outcome+`"}`] += value
}
} else if isPerMethodServerMetric(name) {
if method, ok := prometheusLabelValue(fields[0], "method"); ok {
values[name+`{method="`+method+`"}`] += value
}
} else if isOutcomeServerMetric(name) {
if outcome, ok := prometheusLabelValue(fields[0], "outcome"); ok {
values[name+`{outcome="`+outcome+`"}`] += value
}
}
if isStateServerMetric(name) {
if state, ok := prometheusLabelValue(fields[0], "state"); ok {
values[name+`{state="`+state+`"}`] += value
}
}
}
if err := reader.Err(); err != nil {
c.errors.Add(1)
@ -126,6 +192,112 @@ func (c *serverMetricsClient) scrape(ctx context.Context) (map[string]float64, e
return values, nil
}
// waitForPresenceLastSeenSettlement waits until every expected lifecycle event
// has reached the server-owned batch queue and all accepted work has drained.
// It is report-only synchronization: it does not participate in RPC success or
// alter the server's presence semantics.
func (c *serverMetricsClient) waitForPresenceLastSeenSettlement(
ctx context.Context,
baselineSubmitted float64,
expectedSubmitted uint64,
timeout time.Duration,
) (map[string]float64, error) {
if c == nil {
return nil, nil
}
if timeout <= 0 {
timeout = 15 * time.Second
}
deadline := time.NewTimer(timeout)
defer deadline.Stop()
poll := time.NewTicker(100 * time.Millisecond)
defer poll.Stop()
var last map[string]float64
for {
sample, err := c.scrape(ctx)
if err != nil {
return last, err
}
last = sample
submitted := metricValue(sample, "telesrv_presence_last_seen_submitted_total") - baselineSubmitted
pending := metricValue(sample, "telesrv_presence_last_seen_pending")
bootstrapPending := metricValue(sample, "telesrv_bootstrap_ready_pending")
activeChannelIDsPending := metricValue(sample, "telesrv_active_channel_ids_pending")
if submitted >= float64(expectedSubmitted) && pending == 0 && bootstrapPending == 0 && activeChannelIDsPending == 0 {
return sample, nil
}
select {
case <-ctx.Done():
return last, ctx.Err()
case <-deadline.C:
return last, fmt.Errorf("startup settlement timeout: presence submitted=%.0f expected=%d pending=%.0f bootstrap_pending=%.0f active_channel_ids_pending=%.0f", submitted, expectedSubmitted, pending, bootstrapPending, activeChannelIDsPending)
case <-poll.C:
}
}
}
func isOutcomeServerMetric(name string) bool {
switch name {
case "telesrv_presence_last_seen_batches_total", "telesrv_presence_last_seen_updates_total",
"telesrv_bootstrap_ready_batches_total", "telesrv_bootstrap_ready_selectors_total",
"telesrv_active_channel_ids_cache_total", "telesrv_active_channel_ids_batches_total",
"telesrv_active_channel_ids_selectors_total":
return true
default:
return false
}
}
func isPerMethodServerMetric(name string) bool {
switch name {
case "telesrv_mtproto_rpc_result_inner_bytes_total",
"telesrv_mtproto_rpc_result_wire_bytes_total",
"telesrv_rpc_db_queries_total",
"telesrv_rpc_db_errors_total",
"telesrv_rpc_db_time_seconds_sum",
"telesrv_rpc_db_time_seconds_count":
return true
default:
return false
}
}
func isPerMethodOutcomeServerMetric(name string) bool {
switch name {
case "telesrv_mtproto_rpc_result_delivered_total", "telesrv_mtproto_rpc_result_delivered_bytes_total":
return true
default:
return false
}
}
func isStateServerMetric(name string) bool {
switch name {
case "telesrv_mtproto_sessions", "telesrv_mtproto_logical_sessions",
"telesrv_postgres_pool_connections", "telesrv_redis_pool_connections":
return true
default:
return false
}
}
func prometheusLabelValue(series, label string) (string, bool) {
needle := label + `="`
start := strings.Index(series, needle)
if start < 0 {
return "", false
}
start += len(needle)
end := start
for end < len(series) {
if series[end] == '"' && (end == start || series[end-1] != '\\') {
return series[start:end], true
}
end++
}
return "", false
}
func (c *serverMetricsClient) successes() uint64 {
if c == nil {
return 0

View file

@ -5,7 +5,9 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
)
func TestServerMetricsScrapeSelectsBoundedCapacitySignals(t *testing.T) {
@ -16,6 +18,24 @@ func TestServerMetricsScrapeSelectsBoundedCapacitySignals(t *testing.T) {
for i := 0; i < 256; i++ {
fmt.Fprintf(w, "telesrv_mtproto_rpc_result_wire_bytes_total{method=%q} 1\n", fmt.Sprintf("method-%d", i))
}
fmt.Fprintln(w, `telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="ok"} 7`)
fmt.Fprintln(w, `telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="edge_overload"} 3`)
fmt.Fprintln(w, `telesrv_mtproto_rpc_result_delivered_bytes_total{method="users.getUsers",outcome="ok"} 700`)
fmt.Fprintln(w, `telesrv_mtproto_rpc_result_delivered_bytes_total{method="users.getUsers",outcome="edge_overload"} 300`)
fmt.Fprintln(w, `telesrv_presence_last_seen_batches_total{outcome="ok"} 17`)
fmt.Fprintln(w, `telesrv_presence_last_seen_batches_total{outcome="error"} 2`)
fmt.Fprintln(w, `telesrv_presence_last_seen_updates_total{outcome="ok"} 900`)
fmt.Fprintln(w, `telesrv_presence_last_seen_updates_total{outcome="error"} 23`)
fmt.Fprintln(w, `telesrv_presence_last_seen_submitted_total 923`)
fmt.Fprintln(w, `telesrv_presence_last_seen_pending 11`)
fmt.Fprintln(w, `telesrv_presence_last_seen_overflow_total 1`)
fmt.Fprintln(w, `telesrv_presence_last_seen_drain_dropped_total 4`)
fmt.Fprintln(w, `telesrv_bootstrap_ready_batches_total{outcome="ok"} 10`)
fmt.Fprintln(w, `telesrv_bootstrap_ready_batches_total{outcome="error"} 1`)
fmt.Fprintln(w, `telesrv_bootstrap_ready_selectors_total{outcome="matched"} 2`)
fmt.Fprintln(w, `telesrv_bootstrap_ready_selectors_total{outcome="miss"} 800`)
fmt.Fprintln(w, `telesrv_bootstrap_ready_selectors_total{outcome="error"} 3`)
fmt.Fprintln(w, `telesrv_bootstrap_ready_pending 1`)
fmt.Fprintln(w, `unrelated_high_cardinality{user_id="secret"} 1`)
}))
defer server.Close()
@ -27,7 +47,89 @@ func TestServerMetricsScrapeSelectsBoundedCapacitySignals(t *testing.T) {
if values["telesrv_mtproto_raw_connections"] != 500 || values["telesrv_mtproto_sessions"] != 500 || values["telesrv_mtproto_rpc_result_wire_bytes_total"] != 256 {
t.Fatalf("values = %#v", values)
}
if len(values) != 3 || client.successes() != 1 || client.failures() != 0 {
if values[`telesrv_mtproto_rpc_result_wire_bytes_total{method="method-17"}`] != 1 {
t.Fatalf("method response bytes = %#v", values)
}
if values[`telesrv_mtproto_sessions{state="active"}`] != 499 || values[`telesrv_mtproto_sessions{state="provisional"}`] != 1 {
t.Fatalf("session state values = %#v", values)
}
if values[`telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="ok"}`] != 7 ||
values[`telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="edge_overload"}`] != 3 {
t.Fatalf("delivery outcomes = %#v", values)
}
if values[`telesrv_presence_last_seen_batches_total{outcome="ok"}`] != 17 ||
values[`telesrv_presence_last_seen_batches_total{outcome="error"}`] != 2 ||
values[`telesrv_presence_last_seen_updates_total{outcome="ok"}`] != 900 ||
values[`telesrv_presence_last_seen_updates_total{outcome="error"}`] != 23 ||
values["telesrv_presence_last_seen_submitted_total"] != 923 ||
values["telesrv_presence_last_seen_pending"] != 11 ||
values["telesrv_presence_last_seen_overflow_total"] != 1 ||
values["telesrv_presence_last_seen_drain_dropped_total"] != 4 {
t.Fatalf("presence batch values = %#v", values)
}
if values[`telesrv_bootstrap_ready_batches_total{outcome="ok"}`] != 10 ||
values[`telesrv_bootstrap_ready_batches_total{outcome="error"}`] != 1 ||
values[`telesrv_bootstrap_ready_selectors_total{outcome="matched"}`] != 2 ||
values[`telesrv_bootstrap_ready_selectors_total{outcome="miss"}`] != 800 ||
values[`telesrv_bootstrap_ready_selectors_total{outcome="error"}`] != 3 ||
values["telesrv_bootstrap_ready_pending"] != 1 {
t.Fatalf("bootstrap readiness values = %#v", values)
}
if len(values) != 285 || client.successes() != 1 || client.failures() != 0 {
t.Fatalf("bounded values/scrapes = %#v, %d/%d", values, client.successes(), client.failures())
}
}
func TestPrometheusLabelValue(t *testing.T) {
value, ok := prometheusLabelValue(`metric{encoding="gzip",method="messages.getDialogs",outcome="ok"}`, "method")
if !ok || value != "messages.getDialogs" {
t.Fatalf("method = %q, %v", value, ok)
}
if _, ok := prometheusLabelValue(`metric{encoding="gzip"}`, "method"); ok {
t.Fatal("missing method label was accepted")
}
}
func TestServerMetricsWaitsForPresenceLastSeenSettlement(t *testing.T) {
var calls atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
call := calls.Add(1)
if call < 3 {
fmt.Fprintln(w, `telesrv_presence_last_seen_submitted_total 1`)
fmt.Fprintln(w, `telesrv_presence_last_seen_pending 1`)
fmt.Fprintln(w, `telesrv_bootstrap_ready_pending 1`)
return
}
fmt.Fprintln(w, `telesrv_presence_last_seen_submitted_total 2`)
fmt.Fprintln(w, `telesrv_presence_last_seen_pending 0`)
fmt.Fprintln(w, `telesrv_bootstrap_ready_pending 0`)
}))
defer server.Close()
client := newServerMetricsClient(server.URL)
values, err := client.waitForPresenceLastSeenSettlement(context.Background(), 0, 2, time.Second)
if err != nil {
t.Fatalf("wait: %v", err)
}
if values["telesrv_presence_last_seen_submitted_total"] != 2 || values["telesrv_presence_last_seen_pending"] != 0 ||
values["telesrv_bootstrap_ready_pending"] != 0 {
t.Fatalf("settled values = %#v", values)
}
if calls.Load() != 3 {
t.Fatalf("scrape calls = %d, want 3", calls.Load())
}
}
func TestMetricValueDoesNotDoubleCountAggregateAndStateBreakdown(t *testing.T) {
values := map[string]float64{
"telesrv_mtproto_logical_sessions": 254,
`telesrv_mtproto_logical_sessions{state="retained"}`: 254,
`telesrv_mtproto_logical_sessions{state="offline"}`: 0,
}
if got := metricValue(values, "telesrv_mtproto_logical_sessions"); got != 254 {
t.Fatalf("logical sessions = %v, want aggregate 254", got)
}
delete(values, "telesrv_mtproto_logical_sessions")
if got := metricValue(values, "telesrv_mtproto_logical_sessions"); got != 254 {
t.Fatalf("legacy labeled-only logical sessions = %v, want 254", got)
}
}

View file

@ -0,0 +1,723 @@
package loadharness
import (
"context"
"crypto/rsa"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/iamxvbaba/td/tg"
)
const ClientStateVersion = 1
type ClientUpdateState struct {
Pts int `json:"pts"`
Qts int `json:"qts"`
Date int `json:"date"`
Seq int `json:"seq"`
UnreadCount int `json:"unread_count"`
}
type ClientDialogState struct {
PeerType string `json:"peer_type"`
PeerID int64 `json:"peer_id"`
AccessHash int64 `json:"access_hash"`
TopMessage int `json:"top_message"`
TopMessageDate int `json:"top_message_date"`
Pts int `json:"pts,omitempty"`
HasPts bool `json:"has_pts,omitempty"`
ReadInboxMaxID int `json:"read_inbox_max_id"`
ReadOutboxMaxID int `json:"read_outbox_max_id"`
UnreadCount int `json:"unread_count"`
UnreadMentions int `json:"unread_mentions"`
UnreadReactions int `json:"unread_reactions"`
Pinned bool `json:"pinned,omitempty"`
HasDraft bool `json:"has_draft,omitempty"`
DraftText string `json:"draft_text,omitempty"`
DatasetExpected bool `json:"dataset_expected,omitempty"`
}
type ClientAccountState struct {
AccountIndex int `json:"account_index"`
UserID int64 `json:"user_id"`
State ClientUpdateState `json:"state"`
Dialogs []ClientDialogState `json:"dialogs"`
}
type ClientState struct {
Version int `json:"version"`
DatasetSHA256 string `json:"dataset_sha256"`
SeedIdentitySHA string `json:"seed_identity_sha256"`
CreatedAt time.Time `json:"created_at"`
Accounts []ClientAccountState `json:"accounts"`
}
type SnapshotConfig struct {
ManifestPath string
SessionKeyPath string
RSAKeyOverride string
DatasetPath string
SeedStatePath string
ClientStatePath string
Concurrency int
OperationTimeout time.Duration
}
type SnapshotEvent struct {
Completed int
Total int
Account int
Resumed bool
Err error
}
type SnapshotResult struct {
Accounts int
Dialogs int
Channels int
}
func (c SnapshotConfig) validate() error {
if c.ManifestPath == "" || c.SessionKeyPath == "" || c.DatasetPath == "" || c.SeedStatePath == "" || c.ClientStatePath == "" {
return errors.New("manifest, session-key, dataset, seed-state and client-state paths are required")
}
if c.Concurrency <= 0 || c.Concurrency > 64 {
return errors.New("snapshot concurrency must be between 1 and 64")
}
if c.OperationTimeout <= 0 {
return errors.New("snapshot operation timeout must be positive")
}
return nil
}
// SnapshotClientState establishes the old client cursors that later offline
// mutations must advance from. Dialogs are read through public paginated RPCs;
// per-account part files make a 1,000-account snapshot safely resumable without
// rewriting the growing aggregate after every account.
func SnapshotClientState(ctx context.Context, cfg SnapshotConfig, progress func(SnapshotEvent)) (*SnapshotResult, error) {
if err := cfg.validate(); err != nil {
return nil, err
}
manifest, err := LoadManifest(cfg.ManifestPath)
if err != nil {
return nil, err
}
dataset, err := LoadDataset(cfg.DatasetPath)
if err != nil {
return nil, err
}
targets, err := seedPrimaryTargets(manifest, dataset.Config.Accounts)
if err != nil {
return nil, err
}
seedState, err := LoadDatasetSeedState(cfg.SeedStatePath, dataset)
if err != nil {
return nil, err
}
seedJournal := &seedJournal{dataset: dataset, state: seedState}
if err := seedJournal.assertComplete(); err != nil {
return nil, fmt.Errorf("snapshot requires a complete seed: %w", err)
}
seedIdentity, err := seedIdentitySHA256(seedState)
if err != nil {
return nil, err
}
if existing, loadErr := LoadClientState(cfg.ClientStatePath); loadErr == nil {
if err := existing.Validate(dataset, seedState, targets); err != nil {
return nil, fmt.Errorf("existing client state does not match requested dataset: %w", err)
}
return clientStateResult(existing), nil
} else if !os.IsNotExist(loadErr) {
return nil, loadErr
}
key, err := LoadSessionKey(cfg.SessionKeyPath)
if err != nil {
return nil, err
}
publicKey, err := loadManifestPublicKey(cfg.ManifestPath, manifest.Endpoint, cfg.RSAKeyOverride)
if err != nil {
return nil, err
}
accounts := make([]int, dataset.Config.Accounts)
for account := range accounts {
accounts[account] = account
}
resumed := make([]bool, len(accounts))
for _, account := range accounts {
if _, err := loadClientStatePart(clientStatePartPath(cfg.ClientStatePath, account), dataset, seedState, targets, account); err == nil {
resumed[account] = true
} else if !os.IsNotExist(err) {
return nil, err
}
}
completed := 0
if err := runSeedAccountPhase(ctx, "snapshot", accounts, cfg.Concurrency, func(event SeedEvent) {
if event.Err == nil {
completed++
}
if progress != nil {
progress(SnapshotEvent{Completed: completed, Total: len(accounts), Account: event.Account, Resumed: resumed[event.Account], Err: event.Err})
}
}, func(ctx context.Context, account int) error {
if resumed[account] {
return nil
}
state, err := snapshotAccount(ctx, cfg, manifest, dataset, seedState, targets, key, publicKey, account)
if err != nil {
return err
}
return writeClientStatePart(clientStatePartPath(cfg.ClientStatePath, account), dataset.PlanSHA256, seedIdentity, state)
}); err != nil {
return nil, err
}
clientState := &ClientState{
Version: ClientStateVersion, DatasetSHA256: dataset.PlanSHA256, SeedIdentitySHA: seedIdentity, CreatedAt: time.Now().UTC(),
Accounts: make([]ClientAccountState, 0, len(accounts)),
}
for _, account := range accounts {
state, err := loadClientStatePart(clientStatePartPath(cfg.ClientStatePath, account), dataset, seedState, targets, account)
if err != nil {
return nil, err
}
clientState.Accounts = append(clientState.Accounts, *state)
}
if err := clientState.Validate(dataset, seedState, targets); err != nil {
return nil, err
}
if err := WriteClientState(cfg.ClientStatePath, clientState); err != nil {
return nil, err
}
return clientStateResult(clientState), nil
}
func snapshotAccount(
ctx context.Context,
cfg SnapshotConfig,
manifest *Manifest,
dataset *Dataset,
seedState *DatasetSeedState,
targets []SessionRecord,
key [32]byte,
publicKey *rsa.PublicKey,
account int,
) (*ClientAccountState, error) {
var result *ClientAccountState
err := withAuthorizedSeedSession(ctx, SeedConfig{
ManifestPath: cfg.ManifestPath, OperationTimeout: cfg.OperationTimeout,
}, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
dialogs, err := snapshotDialogs(ctx, cfg.OperationTimeout, raw)
if err != nil {
return err
}
expected := expectedDatasetPeers(dataset, seedState, targets, account)
for i := range dialogs {
_, dialogs[i].DatasetExpected = expected[clientPeerKey{typ: dialogs[i].PeerType, id: dialogs[i].PeerID}]
if dialogs[i].DatasetExpected {
delete(expected, clientPeerKey{typ: dialogs[i].PeerType, id: dialogs[i].PeerID})
}
}
if len(expected) != 0 {
missing := make([]string, 0, min(len(expected), 5))
for peer := range expected {
missing = append(missing, fmt.Sprintf("%s:%d", peer.typ, peer.id))
if len(missing) == 5 {
break
}
}
sort.Strings(missing)
return fmt.Errorf("messages.getDialogs omitted %d expected dataset peers (sample %s)", len(expected), strings.Join(missing, ","))
}
if err := validateSeededRichDialogs(dataset, seedState, targets, account, dialogs, true); err != nil {
return fmt.Errorf("rich dialog state: %w", err)
}
stateCtx, cancel := context.WithTimeout(ctx, cfg.OperationTimeout)
state, err := raw.UpdatesGetState(stateCtx)
cancel()
if err != nil {
return fmt.Errorf("updates.getState: %w", err)
}
result = &ClientAccountState{
AccountIndex: account, UserID: targets[account].UserID,
State: ClientUpdateState{Pts: state.Pts, Qts: state.Qts, Date: state.Date, Seq: state.Seq, UnreadCount: state.UnreadCount},
Dialogs: dialogs,
}
return nil
})
if err == nil && result == nil {
return nil, errors.New("snapshot session ended without producing account state")
}
return result, err
}
type clientPeerKey struct {
typ string
id int64
}
func snapshotDialogs(ctx context.Context, timeout time.Duration, raw *tg.Client) ([]ClientDialogState, error) {
dialogs, _, err := snapshotDialogsObserved(ctx, timeout, raw, snapshotPaginationProfile, nil)
return dialogs, err
}
func snapshotDialogsObserved(
ctx context.Context,
timeout time.Duration,
raw *tg.Client,
profile dialogPaginationProfile,
observe func(string, time.Time, error),
) ([]ClientDialogState, StartupDialogsCounts, error) {
if profile.FirstLimit <= 0 || profile.SubsequentLimit <= 0 {
return nil, StartupDialogsCounts{}, errors.New("invalid dialogs pagination profile")
}
dialogsByPeer := make(map[clientPeerKey]ClientDialogState)
counts := StartupDialogsCounts{}
start := time.Now()
pinnedCtx, cancel := context.WithTimeout(ctx, timeout)
pinned, err := raw.MessagesGetPinnedDialogs(pinnedCtx, 0)
cancel()
if observe != nil {
observe("messages.getPinnedDialogs", start, err)
}
counts.PinnedCalls++
if err != nil {
return nil, counts, fmt.Errorf("messages.getPinnedDialogs: %w", err)
}
if _, err := mergeDialogPage(dialogsByPeer, pinned.Dialogs, pinned.Messages, pinned.Chats, pinned.Users, true); err != nil {
return nil, counts, err
}
pinnedPeers := make(map[clientPeerKey]struct{}, len(dialogsByPeer))
for peer := range dialogsByPeer {
pinnedPeers[peer] = struct{}{}
}
request := &tg.MessagesGetDialogsRequest{ExcludePinned: true, OffsetPeer: &tg.InputPeerEmpty{}, Limit: profile.limit(0)}
for page := 0; page < 100; page++ {
request.Limit = profile.limit(page)
start := time.Now()
rpcCtx, cancel := context.WithTimeout(ctx, timeout)
response, err := raw.MessagesGetDialogs(rpcCtx, request)
cancel()
if observe != nil {
observe("messages.getDialogs", start, err)
if page == 0 {
observe("messages.getDialogs.first", start, err)
} else {
observe("messages.getDialogs.next", start, err)
}
}
counts.Calls++
if err != nil {
return nil, counts, fmt.Errorf("messages.getDialogs page %d: %w", page+1, err)
}
var pageDialogs []tg.DialogClass
var messages []tg.MessageClass
var chats []tg.ChatClass
var users []tg.UserClass
complete := false
switch value := response.(type) {
case *tg.MessagesDialogs:
if observe != nil {
observe("messages.getDialogs.full", start, nil)
}
counts.Full++
pageDialogs, messages, chats, users, complete = value.Dialogs, value.Messages, value.Chats, value.Users, true
case *tg.MessagesDialogsSlice:
if observe != nil {
observe("messages.getDialogs.slice", start, nil)
}
counts.Slice++
pageDialogs, messages, chats, users = value.Dialogs, value.Messages, value.Chats, value.Users
case *tg.MessagesDialogsNotModified:
return nil, counts, errors.New("messages.getDialogs with hash=0 returned dialogsNotModified")
default:
return nil, counts, fmt.Errorf("messages.getDialogs returned %T", response)
}
last, overlap, err := mergeDialogPageKnownOverlap(dialogsByPeer, pageDialogs, messages, chats, users, false, pinnedPeers)
counts.PinnedOverlap += overlap
if err != nil {
return nil, counts, fmt.Errorf("messages.getDialogs page %d: %w", page+1, err)
}
// A dialogsSlice is explicitly non-final. The server may enforce a
// smaller per-page cap than the client-requested limit (TDesktop asks
// for 500 after its first page while telesrv currently returns at most
// 100). Only the full constructor or an empty slice proves completion;
// treating a short slice as EOF truncates the real TDesktop workload.
if dialogsPaginationDone(complete, len(pageDialogs)) {
break
}
if last.TopMessage <= 0 || last.TopMessageDate <= 0 {
return nil, counts, fmt.Errorf("messages.getDialogs page %d has no usable offset", page+1)
}
request.OffsetDate = last.TopMessageDate
request.OffsetID = last.TopMessage
request.OffsetPeer = clientDialogInputPeer(last)
if request.OffsetPeer == nil {
return nil, counts, fmt.Errorf("messages.getDialogs page %d has invalid offset peer", page+1)
}
if page == 99 {
return nil, counts, errors.New("messages.getDialogs exceeded 100 pages")
}
}
result := make([]ClientDialogState, 0, len(dialogsByPeer))
for _, dialog := range dialogsByPeer {
result = append(result, dialog)
}
sort.Slice(result, func(i, j int) bool {
if result[i].PeerType != result[j].PeerType {
return result[i].PeerType < result[j].PeerType
}
return result[i].PeerID < result[j].PeerID
})
counts.Dialogs = len(result)
return result, counts, nil
}
func dialogsPaginationDone(complete bool, pageSize int) bool {
return complete || pageSize == 0
}
func mergeDialogPage(
destination map[clientPeerKey]ClientDialogState,
dialogClasses []tg.DialogClass,
messages []tg.MessageClass,
chats []tg.ChatClass,
users []tg.UserClass,
pinnedPage bool,
) (ClientDialogState, error) {
last, _, err := mergeDialogPageKnownOverlap(destination, dialogClasses, messages, chats, users, pinnedPage, nil)
return last, err
}
func mergeDialogPageKnownOverlap(
destination map[clientPeerKey]ClientDialogState,
dialogClasses []tg.DialogClass,
messages []tg.MessageClass,
chats []tg.ChatClass,
users []tg.UserClass,
pinnedPage bool,
allowedOverlap map[clientPeerKey]struct{},
) (ClientDialogState, int, error) {
accessHashes := make(map[clientPeerKey]int64, len(chats)+len(users))
for _, chat := range chats {
channel, ok := chat.(*tg.Channel)
if !ok {
continue
}
hash, ok := channel.GetAccessHash()
if ok {
accessHashes[clientPeerKey{typ: "channel", id: channel.ID}] = hash
}
}
for _, userClass := range users {
user, ok := userClass.(*tg.User)
if !ok {
continue
}
hash, ok := user.GetAccessHash()
if ok {
accessHashes[clientPeerKey{typ: "user", id: user.ID}] = hash
}
}
messageDates := make(map[string]int, len(messages))
for _, messageClass := range messages {
message, ok := messageClass.AsNotEmpty()
if !ok {
continue
}
peer, ok := clientPeerFromTG(message.GetPeerID())
if !ok {
continue
}
messageDates[clientMessageKey(peer, message.GetID())] = message.GetDate()
}
var last ClientDialogState
overlaps := 0
for _, dialogClass := range dialogClasses {
dialog, ok := dialogClass.(*tg.Dialog)
if !ok {
continue
}
peer, ok := clientPeerFromTG(dialog.Peer)
if !ok {
continue
}
hash := accessHashes[peer]
if hash == 0 {
return ClientDialogState{}, overlaps, fmt.Errorf("dialog %s:%d omitted access hash", peer.typ, peer.id)
}
pts, hasPts := dialog.GetPts()
if peer.typ == "channel" && (!hasPts || pts <= 0) {
return ClientDialogState{}, overlaps, fmt.Errorf("channel dialog %d omitted pts", peer.id)
}
date := messageDates[clientMessageKey(peer, dialog.TopMessage)]
if dialog.TopMessage <= 0 || date <= 0 {
return ClientDialogState{}, overlaps, fmt.Errorf("dialog %s:%d omitted top message payload", peer.typ, peer.id)
}
state := ClientDialogState{
PeerType: peer.typ, PeerID: peer.id, AccessHash: hash,
TopMessage: dialog.TopMessage, TopMessageDate: date, Pts: pts, HasPts: hasPts,
ReadInboxMaxID: dialog.ReadInboxMaxID, ReadOutboxMaxID: dialog.ReadOutboxMaxID,
UnreadCount: dialog.UnreadCount, UnreadMentions: dialog.UnreadMentionsCount,
UnreadReactions: dialog.UnreadReactionsCount, Pinned: pinnedPage || dialog.Pinned,
}
if draftClass, ok := dialog.GetDraft(); ok {
if draft, ok := draftClass.(*tg.DraftMessage); ok && draft.Message != "" {
state.HasDraft, state.DraftText = true, draft.Message
}
}
if existing, exists := destination[peer]; exists {
if _, allowed := allowedOverlap[peer]; !allowed || !existing.Pinned || !state.Pinned || existing.TopMessage != state.TopMessage || existing.AccessHash != state.AccessHash {
return ClientDialogState{}, overlaps, fmt.Errorf("duplicate dialog %s:%d", peer.typ, peer.id)
}
delete(allowedOverlap, peer)
if !state.HasDraft && existing.HasDraft {
state.HasDraft, state.DraftText = existing.HasDraft, existing.DraftText
}
overlaps++
}
destination[peer] = state
last = state
}
return last, overlaps, nil
}
func clientPeerFromTG(peer tg.PeerClass) (clientPeerKey, bool) {
switch value := peer.(type) {
case *tg.PeerUser:
return clientPeerKey{typ: "user", id: value.UserID}, value.UserID > 0
case *tg.PeerChannel:
return clientPeerKey{typ: "channel", id: value.ChannelID}, value.ChannelID > 0
default:
return clientPeerKey{}, false
}
}
func clientMessageKey(peer clientPeerKey, messageID int) string {
return fmt.Sprintf("%s:%d:%d", peer.typ, peer.id, messageID)
}
func clientDialogInputPeer(dialog ClientDialogState) tg.InputPeerClass {
switch dialog.PeerType {
case "user":
return &tg.InputPeerUser{UserID: dialog.PeerID, AccessHash: dialog.AccessHash}
case "channel":
return &tg.InputPeerChannel{ChannelID: dialog.PeerID, AccessHash: dialog.AccessHash}
default:
return nil
}
}
func expectedDatasetPeers(dataset *Dataset, seedState *DatasetSeedState, targets []SessionRecord, account int) map[clientPeerKey]struct{} {
expected := make(map[clientPeerKey]struct{})
for _, edge := range dataset.PrivateEdges {
switch account {
case edge.SenderAccount:
expected[clientPeerKey{typ: "user", id: targets[edge.RecipientAccount].UserID}] = struct{}{}
case edge.RecipientAccount:
expected[clientPeerKey{typ: "user", id: targets[edge.SenderAccount].UserID}] = struct{}{}
}
}
for position, group := range dataset.Groups {
memberIndex := sort.SearchInts(group.MemberAccounts, account)
if memberIndex < len(group.MemberAccounts) && group.MemberAccounts[memberIndex] == account {
expected[clientPeerKey{typ: "channel", id: seedState.Groups[position].ChannelID}] = struct{}{}
}
}
return expected
}
func clientStatePartPath(clientStatePath string, account int) string {
return filepath.Join(clientStatePath+".parts", fmt.Sprintf("account-%04d.json", account))
}
type clientStatePart struct {
Version int `json:"version"`
DatasetSHA256 string `json:"dataset_sha256"`
SeedIdentitySHA string `json:"seed_identity_sha256"`
Account ClientAccountState `json:"account"`
}
func writeClientStatePart(path, datasetSHA, seedIdentity string, account *ClientAccountState) error {
if account == nil {
return errors.New("cannot write nil client account state")
}
part := clientStatePart{Version: ClientStateVersion, DatasetSHA256: datasetSHA, SeedIdentitySHA: seedIdentity, Account: *account}
data, err := json.MarshalIndent(part, "", " ")
if err != nil {
return err
}
return writeFileAtomic(path, append(data, '\n'), 0o600)
}
func loadClientStatePart(path string, dataset *Dataset, seedState *DatasetSeedState, targets []SessionRecord, account int) (*ClientAccountState, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var part clientStatePart
decoder := json.NewDecoder(strings.NewReader(string(data)))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&part); err != nil {
return nil, fmt.Errorf("decode client state part: %w", err)
}
seedIdentity, err := seedIdentitySHA256(seedState)
if err != nil {
return nil, err
}
target := targets[account]
if part.Version != ClientStateVersion || part.DatasetSHA256 != dataset.PlanSHA256 || part.SeedIdentitySHA != seedIdentity || part.Account.AccountIndex != target.AccountIndex || part.Account.UserID != target.UserID {
return nil, errors.New("client state part does not match dataset or account")
}
if err := validateClientAccountState(&part.Account); err != nil {
return nil, err
}
if err := validateExpectedDatasetPeers(dataset, seedState, targets, &part.Account); err != nil {
return nil, err
}
return &part.Account, nil
}
func (s *ClientState) Validate(dataset *Dataset, seedState *DatasetSeedState, targets []SessionRecord) error {
if s == nil || s.Version != ClientStateVersion || dataset == nil || s.DatasetSHA256 != dataset.PlanSHA256 {
return errors.New("client state does not match dataset")
}
seedIdentity, err := seedIdentitySHA256(seedState)
if err != nil {
return err
}
if s.SeedIdentitySHA != seedIdentity {
return errors.New("client state does not match seeded channel identities")
}
if len(s.Accounts) != dataset.Config.Accounts || len(targets) != dataset.Config.Accounts {
return errors.New("client state account count does not match dataset")
}
for account := range s.Accounts {
if s.Accounts[account].AccountIndex != account || s.Accounts[account].UserID != targets[account].UserID {
return fmt.Errorf("client state account %d has wrong identity", account)
}
if err := validateClientAccountState(&s.Accounts[account]); err != nil {
return fmt.Errorf("client state account %d: %w", account, err)
}
if err := validateExpectedDatasetPeers(dataset, seedState, targets, &s.Accounts[account]); err != nil {
return fmt.Errorf("client state account %d: %w", account, err)
}
}
return nil
}
func validateExpectedDatasetPeers(dataset *Dataset, seedState *DatasetSeedState, targets []SessionRecord, account *ClientAccountState) error {
expected := expectedDatasetPeers(dataset, seedState, targets, account.AccountIndex)
for _, dialog := range account.Dialogs {
peer := clientPeerKey{typ: dialog.PeerType, id: dialog.PeerID}
_, shouldBeExpected := expected[peer]
if dialog.DatasetExpected != shouldBeExpected {
return fmt.Errorf("dialog %s:%d has incorrect dataset_expected marker", peer.typ, peer.id)
}
if shouldBeExpected {
delete(expected, peer)
}
}
if len(expected) != 0 {
return fmt.Errorf("client state omits %d expected dataset peers", len(expected))
}
return nil
}
func seedIdentitySHA256(state *DatasetSeedState) (string, error) {
if state == nil {
return "", errors.New("nil seed state")
}
type identity struct {
GroupIndex int `json:"group_index"`
ChannelID int64 `json:"channel_id"`
AccessHash int64 `json:"access_hash"`
}
identities := make([]identity, len(state.Groups))
for i, group := range state.Groups {
if group.ChannelID <= 0 || group.AccessHash == 0 {
return "", fmt.Errorf("group %d has incomplete identity", group.GroupIndex)
}
identities[i] = identity{GroupIndex: group.GroupIndex, ChannelID: group.ChannelID, AccessHash: group.AccessHash}
}
data, err := json.Marshal(identities)
if err != nil {
return "", err
}
sum := sha256.Sum256(data)
return fmt.Sprintf("%x", sum[:]), nil
}
func validateClientAccountState(account *ClientAccountState) error {
if account == nil || account.AccountIndex < 0 || account.UserID <= 0 || account.State.Pts < 0 || account.State.Qts < 0 || account.State.Date <= 0 || account.State.Seq < 0 {
return errors.New("invalid account state")
}
seen := make(map[clientPeerKey]struct{}, len(account.Dialogs))
for _, dialog := range account.Dialogs {
peer := clientPeerKey{typ: dialog.PeerType, id: dialog.PeerID}
if (peer.typ != "user" && peer.typ != "channel") || peer.id <= 0 || dialog.AccessHash == 0 || dialog.TopMessage <= 0 || dialog.TopMessageDate <= 0 {
return fmt.Errorf("invalid dialog %s:%d", peer.typ, peer.id)
}
if peer.typ == "channel" && (!dialog.HasPts || dialog.Pts <= 0) {
return fmt.Errorf("invalid channel pts for %d", peer.id)
}
if dialog.HasDraft != (dialog.DraftText != "") {
return fmt.Errorf("invalid draft state for %s:%d", peer.typ, peer.id)
}
if _, exists := seen[peer]; exists {
return fmt.Errorf("duplicate dialog %s:%d", peer.typ, peer.id)
}
seen[peer] = struct{}{}
}
return nil
}
func WriteClientState(path string, state *ClientState) error {
if state == nil || state.Version != ClientStateVersion || state.DatasetSHA256 == "" || state.SeedIdentitySHA == "" {
return errors.New("invalid client state")
}
data, err := json.MarshalIndent(state, "", " ")
if err != nil {
return err
}
return writeFileAtomic(path, append(data, '\n'), 0o600)
}
func LoadClientState(path string) (*ClientState, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var state ClientState
decoder := json.NewDecoder(strings.NewReader(string(data)))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&state); err != nil {
return nil, fmt.Errorf("decode client state: %w", err)
}
if state.Version != ClientStateVersion || state.DatasetSHA256 == "" || state.SeedIdentitySHA == "" {
return nil, errors.New("invalid client state")
}
return &state, nil
}
func clientStateResult(state *ClientState) *SnapshotResult {
result := &SnapshotResult{Accounts: len(state.Accounts)}
for _, account := range state.Accounts {
result.Dialogs += len(account.Dialogs)
for _, dialog := range account.Dialogs {
if dialog.PeerType == "channel" {
result.Channels++
}
}
}
return result
}

View file

@ -0,0 +1,189 @@
package loadharness
import (
"os"
"path/filepath"
"testing"
"github.com/iamxvbaba/td/tg"
)
func TestMergeDialogPageCapturesChannelCursorAndOffset(t *testing.T) {
user := &tg.User{ID: 11}
user.SetAccessHash(111)
channel := &tg.Channel{ID: 22, Title: "group", Megagroup: true}
channel.SetAccessHash(222)
userDialog := &tg.Dialog{Peer: &tg.PeerUser{UserID: 11}, TopMessage: 7}
channelDialog := &tg.Dialog{Peer: &tg.PeerChannel{ChannelID: 22}, TopMessage: 8}
channelDialog.SetPts(12)
destination := make(map[clientPeerKey]ClientDialogState)
last, err := mergeDialogPage(destination,
[]tg.DialogClass{userDialog, channelDialog},
[]tg.MessageClass{
&tg.Message{ID: 7, PeerID: &tg.PeerUser{UserID: 11}, Date: 101},
&tg.Message{ID: 8, PeerID: &tg.PeerChannel{ChannelID: 22}, Date: 102},
},
[]tg.ChatClass{channel}, []tg.UserClass{user}, false,
)
if err != nil {
t.Fatal(err)
}
if len(destination) != 2 || last.PeerType != "channel" || last.PeerID != 22 || last.Pts != 12 || !last.HasPts || last.TopMessageDate != 102 {
t.Fatalf("merged dialogs = %+v, last = %+v", destination, last)
}
if _, err := mergeDialogPage(destination,
[]tg.DialogClass{channelDialog},
[]tg.MessageClass{&tg.Message{ID: 8, PeerID: &tg.PeerChannel{ChannelID: 22}, Date: 102}},
[]tg.ChatClass{channel}, nil, false,
); err == nil {
t.Fatal("duplicate dialog page passed validation")
}
overlapDestination := make(map[clientPeerKey]ClientDialogState)
pinnedDialog := *channelDialog
pinnedDialog.Pinned = true
if _, err := mergeDialogPage(overlapDestination,
[]tg.DialogClass{&pinnedDialog},
[]tg.MessageClass{&tg.Message{ID: 8, PeerID: &tg.PeerChannel{ChannelID: 22}, Date: 102}},
[]tg.ChatClass{channel}, nil, true,
); err != nil {
t.Fatal(err)
}
allowed := map[clientPeerKey]struct{}{{typ: "channel", id: 22}: {}}
if _, overlaps, err := mergeDialogPageKnownOverlap(overlapDestination,
[]tg.DialogClass{&pinnedDialog},
[]tg.MessageClass{&tg.Message{ID: 8, PeerID: &tg.PeerChannel{ChannelID: 22}, Date: 102}},
[]tg.ChatClass{channel}, nil, false, allowed,
); err != nil || overlaps != 1 {
t.Fatalf("known pinned overlap count=%d err=%v", overlaps, err)
}
if _, _, err := mergeDialogPageKnownOverlap(overlapDestination,
[]tg.DialogClass{&pinnedDialog},
[]tg.MessageClass{&tg.Message{ID: 8, PeerID: &tg.PeerChannel{ChannelID: 22}, Date: 102}},
[]tg.ChatClass{channel}, nil, false, allowed,
); err == nil {
t.Fatal("second copy of a consumed pinned overlap passed validation")
}
channelWithoutPts := &tg.Dialog{Peer: &tg.PeerChannel{ChannelID: 22}, TopMessage: 8}
if _, err := mergeDialogPage(make(map[clientPeerKey]ClientDialogState),
[]tg.DialogClass{channelWithoutPts},
[]tg.MessageClass{&tg.Message{ID: 8, PeerID: &tg.PeerChannel{ChannelID: 22}, Date: 102}},
[]tg.ChatClass{channel}, nil, false,
); err == nil {
t.Fatal("channel dialog without pts passed validation")
}
}
func TestClientStateRoundTripLocksSeededChannelIdentity(t *testing.T) {
dataset, seedState, targets := snapshotFixture(t)
seedIdentity, err := seedIdentitySHA256(seedState)
if err != nil {
t.Fatal(err)
}
state := &ClientState{
Version: ClientStateVersion, DatasetSHA256: dataset.PlanSHA256, SeedIdentitySHA: seedIdentity,
Accounts: make([]ClientAccountState, dataset.Config.Accounts),
}
for account := range state.Accounts {
state.Accounts[account] = ClientAccountState{
AccountIndex: account, UserID: targets[account].UserID,
State: ClientUpdateState{Pts: account + 1, Date: 100},
}
expected := expectedDatasetPeers(dataset, seedState, targets, account)
for peer := range expected {
dialog := ClientDialogState{
PeerType: peer.typ, PeerID: peer.id, AccessHash: 99,
TopMessage: 1, TopMessageDate: 100, DatasetExpected: true,
}
if peer.typ == "channel" {
dialog.HasPts, dialog.Pts = true, 5
}
state.Accounts[account].Dialogs = append(state.Accounts[account].Dialogs, dialog)
}
}
if err := state.Validate(dataset, seedState, targets); err != nil {
t.Fatal(err)
}
path := filepath.Join(t.TempDir(), "client-state.json")
if err := WriteClientState(path, state); err != nil {
t.Fatal(err)
}
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0o600 {
t.Fatalf("client state mode = %o, want 600", info.Mode().Perm())
}
loaded, err := LoadClientState(path)
if err != nil {
t.Fatal(err)
}
if err := loaded.Validate(dataset, seedState, targets); err != nil {
t.Fatal(err)
}
seedState.Groups[0].ChannelID++
if err := loaded.Validate(dataset, seedState, targets); err == nil {
t.Fatal("client state accepted different seeded channel identity")
}
}
func TestClientStateRejectsMissingExpectedPeer(t *testing.T) {
dataset, seedState, targets := snapshotFixture(t)
expected := expectedDatasetPeers(dataset, seedState, targets, 0)
account := ClientAccountState{AccountIndex: 0, UserID: targets[0].UserID, State: ClientUpdateState{Date: 100}}
for peer := range expected {
dialog := ClientDialogState{PeerType: peer.typ, PeerID: peer.id, AccessHash: 1, TopMessage: 1, TopMessageDate: 1, DatasetExpected: true}
if peer.typ == "channel" {
dialog.HasPts, dialog.Pts = true, 1
}
account.Dialogs = append(account.Dialogs, dialog)
}
account.Dialogs = account.Dialogs[1:]
if err := validateExpectedDatasetPeers(dataset, seedState, targets, &account); err == nil {
t.Fatal("account with missing expected peer passed validation")
}
}
func TestDialogsPaginationOnlyFinishesOnFullOrEmptyResponse(t *testing.T) {
if dialogsPaginationDone(false, 100) {
t.Fatal("non-empty dialogsSlice was treated as final when the client requested a larger page")
}
if !dialogsPaginationDone(false, 0) {
t.Fatal("empty dialogsSlice did not finish pagination")
}
if !dialogsPaginationDone(true, 100) {
t.Fatal("messages.dialogs full constructor did not finish pagination")
}
}
func snapshotFixture(t *testing.T) (*Dataset, *DatasetSeedState, []SessionRecord) {
t.Helper()
cfg := DatasetConfig{
Accounts: 4, Seed: 7, PrivateFanout: 1,
HotGroups: 1, HotMembers: 4, HotHistory: 1,
}
dataset, err := PlanDataset(cfg)
if err != nil {
t.Fatal(err)
}
seedState, err := NewDatasetSeedState(dataset)
if err != nil {
t.Fatal(err)
}
// This fixture models a pre-rich-state journal unless an individual test
// explicitly opts into the newer phase.
seedState.RichStateByAccount = nil
for account := 0; account < cfg.Accounts; account++ {
seedState.PrivateSentByAccount[account] = cfg.PrivateFanout
}
seedState.Groups[0].ChannelID = 500
seedState.Groups[0].AccessHash = 600
seedState.Groups[0].InviteCursor = 3
seedState.Groups[0].InvitePendingEnd = 3
seedState.HistorySentByAccount[dataset.Groups[0].MemberAccounts[0]] = 1
targets := make([]SessionRecord, cfg.Accounts)
for account := range targets {
targets[account] = SessionRecord{AccountIndex: account, UserID: int64(100 + account), AccessHash: int64(200 + account), SessionFile: "session"}
}
return dataset, seedState, targets
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,67 @@
package loadharness
import (
"fmt"
"strings"
"time"
"github.com/iamxvbaba/td/telegram"
)
const (
StartupProfileTDesktopReturningV1 = "tdesktop-cold-returning-v1"
StartupProfileTDLibReturningV1 = "tdlib-returning-v1"
)
type dialogPaginationProfile struct {
FirstLimit int
SubsequentLimit int
}
type startupWorkloadProfile struct {
Name string
GetStateBeforeDifference bool
AccountDifference bool
Dialogs dialogPaginationProfile
ForceChannelDifference bool
}
func resolveStartupProfile(name string) (startupWorkloadProfile, error) {
switch strings.TrimSpace(name) {
case "", StartupProfileTDesktopReturningV1:
return startupWorkloadProfile{
Name: StartupProfileTDesktopReturningV1, GetStateBeforeDifference: true,
Dialogs: dialogPaginationProfile{FirstLimit: 20, SubsequentLimit: 500},
ForceChannelDifference: true,
}, nil
case StartupProfileTDLibReturningV1:
return startupWorkloadProfile{
Name: StartupProfileTDLibReturningV1,
AccountDifference: true,
Dialogs: dialogPaginationProfile{FirstLimit: 100, SubsequentLimit: 100},
ForceChannelDifference: true,
}, nil
default:
return startupWorkloadProfile{}, fmt.Errorf("unknown startup profile %q", name)
}
}
func (p dialogPaginationProfile) limit(page int) int {
if page == 0 {
return p.FirstLimit
}
return p.SubsequentLimit
}
func (p startupWorkloadProfile) device() telegram.DeviceConfig {
if p.Name == StartupProfileTDLibReturningV1 {
return telegram.DeviceConfig{
DeviceModel: "telesrv-flutter TDLib", SystemVersion: "Android SDK 36", AppVersion: "load-profile-v1",
SystemLangCode: "en-US", LangPack: "android", LangCode: "en",
Params: telegram.TimezoneParams(time.Local),
}
}
return telegram.DeviceTDesktopWindows()
}
var snapshotPaginationProfile = dialogPaginationProfile{FirstLimit: 100, SubsequentLimit: 100}

View file

@ -0,0 +1,197 @@
package loadharness
import (
"os"
"path/filepath"
"slices"
"testing"
"time"
"github.com/iamxvbaba/td/tg"
)
func TestStartupRampDelay(t *testing.T) {
ramp := 30 * time.Second
if got := startupRampDelay(ramp, 0, 4); got != 0 {
t.Fatalf("first delay = %s", got)
}
if got := startupRampDelay(ramp, 1, 4); got != 10*time.Second {
t.Fatalf("second delay = %s", got)
}
if got := startupRampDelay(ramp, 3, 4); got != ramp {
t.Fatalf("last delay = %s", got)
}
}
func TestStartupAccountOrder(t *testing.T) {
sequential := startupAccountOrder(6, StartupOrderAccountIndex, 7)
if !slices.Equal(sequential, []int{0, 1, 2, 3, 4, 5}) {
t.Fatalf("sequential order = %v", sequential)
}
first := startupAccountOrder(100, StartupOrderShuffled, 20260827)
second := startupAccountOrder(100, StartupOrderShuffled, 20260827)
if !slices.Equal(first, second) {
t.Fatal("shuffled startup order is not deterministic")
}
if slices.Equal(first, startupAccountOrder(100, StartupOrderShuffled, 20260828)) {
t.Fatal("different shuffled seeds produced identical order")
}
seen := make(map[int]bool, len(first))
for _, account := range first {
if account < 0 || account >= len(first) || seen[account] {
t.Fatalf("invalid shuffled account %d in %v", account, first)
}
seen[account] = true
}
}
func TestResolveStartupProfiles(t *testing.T) {
tdesktop, err := resolveStartupProfile(StartupProfileTDesktopReturningV1)
if err != nil {
t.Fatal(err)
}
if !tdesktop.GetStateBeforeDifference || tdesktop.AccountDifference || tdesktop.Dialogs.limit(0) != 20 || tdesktop.Dialogs.limit(1) != 500 || !tdesktop.ForceChannelDifference {
t.Fatalf("tdesktop profile = %+v", tdesktop)
}
tdlib, err := resolveStartupProfile(StartupProfileTDLibReturningV1)
if err != nil {
t.Fatal(err)
}
if tdlib.GetStateBeforeDifference || !tdlib.AccountDifference || tdlib.Dialogs.limit(0) != 100 || tdlib.Dialogs.limit(1) != 100 {
t.Fatalf("tdlib profile = %+v", tdlib)
}
if device := tdlib.device(); device.LangPack != "android" || device.SystemVersion != "Android SDK 36" {
t.Fatalf("tdlib device = %+v", device)
}
if _, err := resolveStartupProfile("unknown"); err == nil {
t.Fatal("unknown startup profile passed validation")
}
}
func TestRequireExactMarkers(t *testing.T) {
if err := requireExactMarkers(map[string]int{"a": 1, "b": 1}, []string{"a", "b"}); err != nil {
t.Fatal(err)
}
if err := requireExactMarkers(map[string]int{"a": 2, "b": 1}, []string{"a", "b"}); err == nil {
t.Fatal("duplicate marker passed validation")
}
if err := requireExactMarkers(map[string]int{"a": 1, "b": 1, "wrong": 1}, []string{"a", "b"}); err == nil {
t.Fatal("wrong-account marker passed validation")
}
}
func TestCollectChannelMarkersIncludesEdits(t *testing.T) {
runID := "run"
markers := make(map[string]int)
collectChannelMarkers(markers,
[]tg.MessageClass{&tg.Message{Message: "[run offline channel 0000 message 0001]"}},
[]tg.UpdateClass{&tg.UpdateEditChannelMessage{Message: &tg.Message{Message: "[run offline channel 0000 message 0001] edited"}}},
runID,
)
if markers["[run offline channel 0000 message 0001]"] != 1 || markers["[run offline channel 0000 message 0001] edited"] != 1 {
t.Fatalf("collected markers = %v", markers)
}
}
func TestValidateStartupDialogsRequiresCurrentDirtyChannelPts(t *testing.T) {
dataset, seedState, targets := snapshotFixture(t)
plan := planOfflineMutation(dataset)
mutation := &OfflineMutationState{
PrivateMessageIDs: []int{1, 1, 1, 1},
Channels: []OfflineMutationChannelState{{LatestPts: 50}},
}
expected := expectedDatasetPeers(dataset, seedState, targets, 0)
dialogs := make([]ClientDialogState, 0, len(expected))
for peer := range expected {
dialog := ClientDialogState{PeerType: peer.typ, PeerID: peer.id, AccessHash: 1, TopMessage: 1, TopMessageDate: 1}
if peer.typ == "channel" {
dialog.HasPts, dialog.Pts = true, 50
}
dialogs = append(dialogs, dialog)
}
if err := validateStartupDialogs(dataset, seedState, plan, mutation, targets, 0, dialogs); err != nil {
t.Fatal(err)
}
for i := range dialogs {
if dialogs[i].PeerType == "channel" {
dialogs[i].Pts = 49
}
}
if err := validateStartupDialogs(dataset, seedState, plan, mutation, targets, 0, dialogs); err == nil {
t.Fatal("stale current channel pts passed validation")
}
}
func TestWriteStartupReportOwnerOnly(t *testing.T) {
path := filepath.Join(t.TempDir(), "startup-report.json")
report := &StartupRunReport{Version: StartupReportVersion, DatasetSHA256: "plan", ExpectedAccounts: 1, BusinessReady: 1, Pass: true}
if err := WriteStartupReport(path, report); err != nil {
t.Fatal(err)
}
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0o600 {
t.Fatalf("startup report mode = %o, want 600", info.Mode().Perm())
}
}
func TestStartupResponseBytesUsesPerMethodCounterDeltas(t *testing.T) {
baseline := map[string]float64{
`telesrv_mtproto_rpc_result_inner_bytes_total{method="messages.getDialogs"}`: 100,
`telesrv_mtproto_rpc_result_wire_bytes_total{method="messages.getDialogs"}`: 50,
}
final := map[string]float64{
`telesrv_mtproto_rpc_result_inner_bytes_total{method="messages.getDialogs"}`: 900,
`telesrv_mtproto_rpc_result_wire_bytes_total{method="messages.getDialogs"}`: 250,
`telesrv_mtproto_rpc_result_delivered_bytes_total{method="messages.getDialogs",outcome="ok"}`: 200,
`telesrv_mtproto_rpc_result_delivered_bytes_total{method="messages.getDialogs",outcome="edge_overload"}`: 40,
}
bytes := startupResponseBytes(baseline, final)["messages.getDialogs"]
if bytes.Inner != 800 || bytes.Wire != 200 || bytes.Delivered != 200 {
t.Fatalf("response bytes = %+v", bytes)
}
}
func TestStartupDatabaseWorkUsesPerMethodCounterDeltas(t *testing.T) {
baseline := map[string]float64{
`telesrv_rpc_db_queries_total{method="messages.getDialogs"}`: 10,
`telesrv_rpc_db_time_seconds_sum{method="messages.getDialogs"}`: 0.5,
`telesrv_rpc_db_time_seconds_count{method="messages.getDialogs"}`: 1,
`telesrv_rpc_db_errors_total{method="messages.getDialogs"}`: 1,
}
final := map[string]float64{
`telesrv_rpc_db_queries_total{method="messages.getDialogs"}`: 210,
`telesrv_rpc_db_time_seconds_sum{method="messages.getDialogs"}`: 2.75,
`telesrv_rpc_db_time_seconds_count{method="messages.getDialogs"}`: 11,
`telesrv_rpc_db_errors_total{method="messages.getDialogs"}`: 1,
}
work := startupDatabaseWork(baseline, final)["messages.getDialogs"]
if work.Queries != 200 || work.RPCs != 10 || work.Errors != 0 || work.DurationSeconds != 2.25 {
t.Fatalf("database work = %+v", work)
}
}
func TestStartupRPCDeliveryOutcomesUsesBoundedMethodAndOutcomeDeltas(t *testing.T) {
baseline := map[string]float64{
`telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="ok"}`: 5,
}
final := map[string]float64{
`telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="ok"}`: 12,
`telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="edge_overload"}`: 3,
}
outcomes := startupRPCDeliveryOutcomes(baseline, final)
if outcomes["users.getUsers"]["ok"] != 7 || outcomes["users.getUsers"]["edge_overload"] != 3 {
t.Fatalf("delivery outcomes = %#v", outcomes)
}
}
func TestUpdateMetricPeaks(t *testing.T) {
peaks := map[string]float64{"heap": 10}
updateMetricPeaks(peaks, map[string]float64{"heap": 9, "connections": 5})
updateMetricPeaks(peaks, map[string]float64{"heap": 12, "connections": 2})
if peaks["heap"] != 12 || peaks["connections"] != 5 {
t.Fatalf("peaks = %v", peaks)
}
}