merged from gramsrv upstream
This commit is contained in:
parent
79c64ee916
commit
21a0856587
651 changed files with 54774 additions and 4590 deletions
|
|
@ -12,8 +12,12 @@ import (
|
|||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/iamxvbaba/td/exchange"
|
||||
"github.com/iamxvbaba/td/mtproto"
|
||||
"github.com/iamxvbaba/td/proto"
|
||||
"github.com/iamxvbaba/td/telegram"
|
||||
"github.com/iamxvbaba/td/telegram/dcs"
|
||||
"github.com/iamxvbaba/td/tg"
|
||||
|
|
@ -24,6 +28,40 @@ type clientHooks struct {
|
|||
Update telegram.UpdateHandler
|
||||
ConnectionState func(telegram.ConnectionState)
|
||||
Dead func(error)
|
||||
Device *telegram.DeviceConfig
|
||||
}
|
||||
|
||||
// loadMessageIDSource keeps the load generator on the same MTProto message-id
|
||||
// rules as a production client even when the host clock lands exactly on an
|
||||
// integral second. The underlying gotd generator can emit a client id whose
|
||||
// lower 32 bits are zero in that narrow window; Telegram explicitly forbids
|
||||
// that value as replay protection. Retrying also fences any encoded duplicate
|
||||
// caused by a very low-resolution clock without weakening the DUT validator.
|
||||
type loadMessageIDSource struct {
|
||||
mu sync.Mutex
|
||||
source mtproto.MessageIDSource
|
||||
last int64
|
||||
}
|
||||
|
||||
func newLoadMessageIDSource(now func() time.Time) *loadMessageIDSource {
|
||||
return &loadMessageIDSource{source: proto.NewMessageIDGen(now)}
|
||||
}
|
||||
|
||||
func (s *loadMessageIDSource) New(messageType proto.MessageType) int64 {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
for {
|
||||
messageID := s.source.New(messageType)
|
||||
if messageID <= s.last {
|
||||
continue
|
||||
}
|
||||
if messageType == proto.MessageFromClient && uint32(messageID) == 0 {
|
||||
continue
|
||||
}
|
||||
s.last = messageID
|
||||
return messageID
|
||||
}
|
||||
}
|
||||
|
||||
func newClient(endpoint Endpoint, publicKey *rsa.PublicKey, storage telegram.SessionStorage, hooks clientHooks) (*telegram.Client, error) {
|
||||
|
|
@ -44,6 +82,10 @@ func newClient(endpoint Endpoint, publicKey *rsa.PublicKey, storage telegram.Ses
|
|||
if updateHandler == nil {
|
||||
updateHandler = telegram.UpdateHandlerFunc(func(context.Context, tg.UpdatesClass) error { return nil })
|
||||
}
|
||||
device := telegram.DeviceTDesktopWindows()
|
||||
if hooks.Device != nil {
|
||||
device = *hooks.Device
|
||||
}
|
||||
return telegram.NewClient(endpoint.APIID, endpoint.APIHash, telegram.Options{
|
||||
PublicKeys: []exchange.PublicKey{{RSA: publicKey}},
|
||||
DC: endpoint.DC,
|
||||
|
|
@ -55,7 +97,8 @@ func newClient(endpoint Endpoint, publicKey *rsa.PublicKey, storage telegram.Ses
|
|||
UpdateHandler: updateHandler,
|
||||
EnablePFS: endpoint.PFS,
|
||||
TempKeyTTL: endpoint.TempKeyTTL,
|
||||
Device: telegram.DeviceTDesktopWindows(),
|
||||
Device: device,
|
||||
MessageID: newLoadMessageIDSource(time.Now),
|
||||
OnConnectionState: hooks.ConnectionState,
|
||||
OnDead: hooks.Dead,
|
||||
}), nil
|
||||
|
|
|
|||
60
internal/loadharness/client_test.go
Normal file
60
internal/loadharness/client_test.go
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/iamxvbaba/td/proto"
|
||||
)
|
||||
|
||||
func TestLoadMessageIDSourceSkipsEmptyClientFraction(t *testing.T) {
|
||||
second := time.Unix(1_800_000_000, 0)
|
||||
source := newLoadMessageIDSource(func() time.Time { return second })
|
||||
|
||||
first := source.New(proto.MessageFromClient)
|
||||
secondID := source.New(proto.MessageFromClient)
|
||||
for index, messageID := range []int64{first, secondID} {
|
||||
if uint32(messageID) == 0 {
|
||||
t.Fatalf("message id %d lower 32 bits are empty", index)
|
||||
}
|
||||
if proto.MessageID(messageID).Type() != proto.MessageFromClient {
|
||||
t.Fatalf("message id %d type = %v, want client", index, proto.MessageID(messageID).Type())
|
||||
}
|
||||
}
|
||||
if secondID <= first {
|
||||
t.Fatalf("message ids are not strictly increasing: %d then %d", first, secondID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadMessageIDSourceConcurrentMonotonicUnique(t *testing.T) {
|
||||
second := time.Unix(1_800_000_000, 0)
|
||||
source := newLoadMessageIDSource(func() time.Time { return second })
|
||||
|
||||
const calls = 1_000
|
||||
ids := make(chan int64, calls)
|
||||
var workers sync.WaitGroup
|
||||
workers.Add(calls)
|
||||
for range calls {
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
ids <- source.New(proto.MessageFromClient)
|
||||
}()
|
||||
}
|
||||
workers.Wait()
|
||||
close(ids)
|
||||
|
||||
seen := make(map[int64]struct{}, calls)
|
||||
for messageID := range ids {
|
||||
if uint32(messageID) == 0 || proto.MessageID(messageID).Type() != proto.MessageFromClient {
|
||||
t.Fatalf("invalid client message id %d", messageID)
|
||||
}
|
||||
if _, duplicate := seen[messageID]; duplicate {
|
||||
t.Fatalf("duplicate client message id %d", messageID)
|
||||
}
|
||||
seen[messageID] = struct{}{}
|
||||
}
|
||||
if len(seen) != calls {
|
||||
t.Fatalf("unique message ids = %d, want %d", len(seen), calls)
|
||||
}
|
||||
}
|
||||
446
internal/loadharness/dataset.go
Normal file
446
internal/loadharness/dataset.go
Normal file
|
|
@ -0,0 +1,446 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
DatasetVersion = 1
|
||||
maxDatasetGroups = 10000
|
||||
maxDatasetMembership = 2_000_000
|
||||
maxDatasetMessages = 2_000_000
|
||||
)
|
||||
|
||||
// DatasetConfig describes a deterministic social graph whose durable facts are
|
||||
// later materialized exclusively through real MTProto RPCs. Planning does not
|
||||
// contact the server and never embeds auth/session material.
|
||||
type DatasetConfig struct {
|
||||
Accounts int `json:"accounts"`
|
||||
Seed int64 `json:"seed"`
|
||||
PrivateFanout int `json:"private_fanout"`
|
||||
HotGroups int `json:"hot_groups"`
|
||||
HotMembers int `json:"hot_members"`
|
||||
HotHistory int `json:"hot_history"`
|
||||
MediumGroups int `json:"medium_groups"`
|
||||
MediumMembers int `json:"medium_members"`
|
||||
MediumHistory int `json:"medium_history"`
|
||||
SmallGroups int `json:"small_groups"`
|
||||
SmallMembers int `json:"small_members"`
|
||||
SmallHistory int `json:"small_history"`
|
||||
HeavyGroups int `json:"heavy_groups"`
|
||||
HeavyAccounts int `json:"heavy_accounts"`
|
||||
HeavyHistory int `json:"heavy_history"`
|
||||
}
|
||||
|
||||
func DefaultDatasetConfig(accounts int) DatasetConfig {
|
||||
return DatasetConfig{
|
||||
Accounts: accounts, Seed: 20260827, PrivateFanout: min(10, max(accounts-1, 0)),
|
||||
HotGroups: 10, HotMembers: accounts, HotHistory: 100,
|
||||
MediumGroups: 100, MediumMembers: min(100, accounts), MediumHistory: 30,
|
||||
SmallGroups: 200, SmallMembers: min(20, accounts), SmallHistory: 10,
|
||||
HeavyGroups: 200, HeavyAccounts: min(100, accounts), HeavyHistory: 30,
|
||||
}
|
||||
}
|
||||
|
||||
type DatasetPrivateEdge struct {
|
||||
SenderAccount int `json:"sender_account"`
|
||||
RecipientAccount int `json:"recipient_account"`
|
||||
RandomID int64 `json:"random_id"`
|
||||
Marker string `json:"marker"`
|
||||
}
|
||||
|
||||
type DatasetGroup struct {
|
||||
Index int `json:"index"`
|
||||
Tier string `json:"tier"`
|
||||
Title string `json:"title"`
|
||||
About string `json:"about"`
|
||||
CreatorAccount int `json:"creator_account"`
|
||||
MemberAccounts []int `json:"member_accounts"`
|
||||
HistoryMessages int `json:"history_messages"`
|
||||
}
|
||||
|
||||
// Dataset is the immutable plan. Real server identities and resumable progress
|
||||
// live in the separate compact DatasetSeedState so a large plan is not rewritten
|
||||
// after every reconciled RPC batch.
|
||||
type Dataset struct {
|
||||
Version int `json:"version"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
RunID string `json:"run_id"`
|
||||
Config DatasetConfig `json:"config"`
|
||||
PlanSHA256 string `json:"plan_sha256"`
|
||||
PrivateEdges []DatasetPrivateEdge `json:"private_edges"`
|
||||
Groups []DatasetGroup `json:"groups"`
|
||||
}
|
||||
|
||||
type DatasetSeedGroupState struct {
|
||||
GroupIndex int `json:"group_index"`
|
||||
ChannelID int64 `json:"channel_id,omitempty"`
|
||||
AccessHash int64 `json:"access_hash,omitempty"`
|
||||
CreatePending bool `json:"create_pending,omitempty"`
|
||||
InviteCursor int `json:"invite_cursor,omitempty"`
|
||||
InvitePendingEnd int `json:"invite_pending_end,omitempty"`
|
||||
}
|
||||
|
||||
type DatasetSeedState struct {
|
||||
Version int `json:"version"`
|
||||
PlanSHA256 string `json:"plan_sha256"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
PrivateSentByAccount []int `json:"private_sent_by_account"`
|
||||
HistorySentByAccount []int `json:"history_sent_by_account"`
|
||||
RichStateByAccount []bool `json:"rich_state_by_account,omitempty"`
|
||||
Groups []DatasetSeedGroupState `json:"groups"`
|
||||
}
|
||||
|
||||
func PlanDataset(cfg DatasetConfig) (*Dataset, error) {
|
||||
if err := cfg.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
runID := fmt.Sprintf("rpc-startup-%d-%016x", cfg.Accounts, uint64(cfg.Seed))
|
||||
dataset := &Dataset{
|
||||
Version: DatasetVersion, CreatedAt: time.Now().UTC(),
|
||||
RunID: runID, Config: cfg,
|
||||
PrivateEdges: make([]DatasetPrivateEdge, 0, cfg.Accounts*cfg.PrivateFanout),
|
||||
}
|
||||
for account := 0; account < cfg.Accounts; account++ {
|
||||
for offset := 1; offset <= cfg.PrivateFanout; offset++ {
|
||||
recipient := (account + offset) % cfg.Accounts
|
||||
dataset.PrivateEdges = append(dataset.PrivateEdges, DatasetPrivateEdge{
|
||||
SenderAccount: account, RecipientAccount: recipient,
|
||||
RandomID: stableDatasetID(cfg.Seed, "private", cfg.Accounts, account, offset),
|
||||
Marker: fmt.Sprintf("[%s private %04d/%02d]", runID, account, offset),
|
||||
})
|
||||
}
|
||||
}
|
||||
groupIndex := 0
|
||||
appendTier := func(tier string, count, members, history int, memberSet func(int) []int) {
|
||||
for i := 0; i < count; i++ {
|
||||
groupMembers := memberSet(i)
|
||||
creator := groupMembers[i%len(groupMembers)]
|
||||
dataset.Groups = append(dataset.Groups, DatasetGroup{
|
||||
Index: groupIndex, Tier: tier,
|
||||
Title: fmt.Sprintf("%s %s %04d", runID, tier, i+1),
|
||||
About: fmt.Sprintf("telesrv real-RPC load dataset %s group %d", tier, i+1),
|
||||
CreatorAccount: creator, MemberAccounts: groupMembers, HistoryMessages: history,
|
||||
})
|
||||
groupIndex++
|
||||
}
|
||||
}
|
||||
appendTier("hot", cfg.HotGroups, cfg.HotMembers, cfg.HotHistory, func(i int) []int {
|
||||
return cyclicMembers(cfg.Accounts, i*max(cfg.HotMembers, 1), cfg.HotMembers)
|
||||
})
|
||||
appendTier("medium", cfg.MediumGroups, cfg.MediumMembers, cfg.MediumHistory, func(i int) []int {
|
||||
return cyclicMembers(cfg.Accounts, i*max(cfg.MediumMembers, 1), cfg.MediumMembers)
|
||||
})
|
||||
appendTier("small", cfg.SmallGroups, cfg.SmallMembers, cfg.SmallHistory, func(i int) []int {
|
||||
return cyclicMembers(cfg.Accounts, i*max(cfg.SmallMembers, 1), cfg.SmallMembers)
|
||||
})
|
||||
appendTier("heavy", cfg.HeavyGroups, cfg.HeavyAccounts, cfg.HeavyHistory, func(int) []int {
|
||||
members := make([]int, cfg.HeavyAccounts)
|
||||
for i := range members {
|
||||
members[i] = i
|
||||
}
|
||||
return members
|
||||
})
|
||||
planHash, err := dataset.planHash()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dataset.PlanSHA256 = planHash
|
||||
if err := dataset.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dataset, nil
|
||||
}
|
||||
|
||||
func (c DatasetConfig) validate() error {
|
||||
if c.Accounts < 2 || c.Accounts > 100000 {
|
||||
return errors.New("dataset accounts must be between 2 and 100000")
|
||||
}
|
||||
if c.PrivateFanout < 0 || c.PrivateFanout >= c.Accounts {
|
||||
return errors.New("private fanout must be non-negative and smaller than accounts")
|
||||
}
|
||||
groups := c.HotGroups + c.MediumGroups + c.SmallGroups + c.HeavyGroups
|
||||
if groups <= 0 || groups > maxDatasetGroups {
|
||||
return fmt.Errorf("dataset group count must be between 1 and %d", maxDatasetGroups)
|
||||
}
|
||||
tiers := []struct {
|
||||
name string
|
||||
groups, members, history int
|
||||
}{
|
||||
{"hot", c.HotGroups, c.HotMembers, c.HotHistory},
|
||||
{"medium", c.MediumGroups, c.MediumMembers, c.MediumHistory},
|
||||
{"small", c.SmallGroups, c.SmallMembers, c.SmallHistory},
|
||||
{"heavy", c.HeavyGroups, c.HeavyAccounts, c.HeavyHistory},
|
||||
}
|
||||
memberships := 0
|
||||
messages := c.Accounts * c.PrivateFanout
|
||||
for _, tier := range tiers {
|
||||
if tier.groups < 0 || tier.members < 0 || tier.members > c.Accounts || tier.history < 0 {
|
||||
return fmt.Errorf("invalid %s dataset tier", tier.name)
|
||||
}
|
||||
if tier.groups > 0 && tier.members == 0 {
|
||||
return fmt.Errorf("%s dataset tier has groups without members", tier.name)
|
||||
}
|
||||
memberships += tier.groups * tier.members
|
||||
messages += tier.groups * tier.history
|
||||
}
|
||||
if memberships > maxDatasetMembership {
|
||||
return fmt.Errorf("dataset memberships %d exceed hard limit %d", memberships, maxDatasetMembership)
|
||||
}
|
||||
if messages > maxDatasetMessages {
|
||||
return fmt.Errorf("dataset messages %d exceed hard limit %d", messages, maxDatasetMessages)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dataset) Validate() error {
|
||||
if d == nil || d.Version != DatasetVersion {
|
||||
return errors.New("invalid dataset version")
|
||||
}
|
||||
if strings.TrimSpace(d.RunID) == "" || strings.TrimSpace(d.PlanSHA256) == "" {
|
||||
return errors.New("dataset is missing run id or plan hash")
|
||||
}
|
||||
if err := d.Config.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(d.PrivateEdges) != d.Config.Accounts*d.Config.PrivateFanout {
|
||||
return errors.New("dataset private graph size does not match config")
|
||||
}
|
||||
seenGroups := make(map[int]struct{}, len(d.Groups))
|
||||
for _, group := range d.Groups {
|
||||
if group.Index < 0 || group.CreatorAccount < 0 || group.CreatorAccount >= d.Config.Accounts || group.HistoryMessages < 0 {
|
||||
return fmt.Errorf("invalid group %d", group.Index)
|
||||
}
|
||||
if _, exists := seenGroups[group.Index]; exists {
|
||||
return fmt.Errorf("duplicate group index %d", group.Index)
|
||||
}
|
||||
seenGroups[group.Index] = struct{}{}
|
||||
members := make(map[int]struct{}, len(group.MemberAccounts))
|
||||
for _, account := range group.MemberAccounts {
|
||||
if account < 0 || account >= d.Config.Accounts {
|
||||
return fmt.Errorf("group %d has invalid account %d", group.Index, account)
|
||||
}
|
||||
if _, exists := members[account]; exists {
|
||||
return fmt.Errorf("group %d has duplicate account %d", group.Index, account)
|
||||
}
|
||||
members[account] = struct{}{}
|
||||
}
|
||||
if _, ok := members[group.CreatorAccount]; !ok {
|
||||
return fmt.Errorf("group %d creator is not a member", group.Index)
|
||||
}
|
||||
}
|
||||
wantHash, err := d.planHash()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if wantHash != d.PlanSHA256 {
|
||||
return errors.New("dataset immutable plan hash mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func WriteDataset(path string, dataset *Dataset) error {
|
||||
if err := dataset.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(dataset, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode dataset: %w", err)
|
||||
}
|
||||
return writeFileAtomic(path, append(data, '\n'), 0o600)
|
||||
}
|
||||
|
||||
func NewDatasetSeedState(dataset *Dataset) (*DatasetSeedState, error) {
|
||||
if err := dataset.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
state := &DatasetSeedState{
|
||||
Version: DatasetVersion, PlanSHA256: dataset.PlanSHA256, UpdatedAt: time.Now().UTC(),
|
||||
PrivateSentByAccount: make([]int, dataset.Config.Accounts),
|
||||
HistorySentByAccount: make([]int, dataset.Config.Accounts),
|
||||
RichStateByAccount: make([]bool, dataset.Config.Accounts),
|
||||
Groups: make([]DatasetSeedGroupState, len(dataset.Groups)),
|
||||
}
|
||||
for i, group := range dataset.Groups {
|
||||
state.Groups[i].GroupIndex = group.Index
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func (s *DatasetSeedState) Validate(dataset *Dataset) error {
|
||||
if s == nil || s.Version != DatasetVersion || dataset == nil || s.PlanSHA256 != dataset.PlanSHA256 {
|
||||
return errors.New("seed state does not match dataset plan")
|
||||
}
|
||||
if len(s.PrivateSentByAccount) != dataset.Config.Accounts || len(s.HistorySentByAccount) != dataset.Config.Accounts || len(s.Groups) != len(dataset.Groups) {
|
||||
return errors.New("seed state dimensions do not match dataset plan")
|
||||
}
|
||||
// A nil rich-state vector is accepted only for seed journals created before
|
||||
// the comprehensive startup workload added this phase. New journals always
|
||||
// allocate the full vector and therefore cannot silently skip it.
|
||||
if len(s.RichStateByAccount) != 0 && len(s.RichStateByAccount) != dataset.Config.Accounts {
|
||||
return errors.New("seed rich-state dimensions do not match dataset plan")
|
||||
}
|
||||
historyTasks := datasetHistoryTaskCounts(dataset)
|
||||
for account := 0; account < dataset.Config.Accounts; account++ {
|
||||
if s.PrivateSentByAccount[account] < 0 || s.PrivateSentByAccount[account] > dataset.Config.PrivateFanout {
|
||||
return fmt.Errorf("invalid private seed progress for account %d", account)
|
||||
}
|
||||
if s.HistorySentByAccount[account] < 0 || s.HistorySentByAccount[account] > historyTasks[account] {
|
||||
return fmt.Errorf("invalid history seed progress for account %d", account)
|
||||
}
|
||||
}
|
||||
for i, groupState := range s.Groups {
|
||||
group := dataset.Groups[i]
|
||||
invitees := len(group.MemberAccounts) - 1
|
||||
if groupState.GroupIndex != group.Index || groupState.InviteCursor < 0 || groupState.InviteCursor > invitees {
|
||||
return fmt.Errorf("invalid seed state for group %d", group.Index)
|
||||
}
|
||||
if (groupState.ChannelID == 0) != (groupState.AccessHash == 0) {
|
||||
return fmt.Errorf("group %d has partial channel identity", group.Index)
|
||||
}
|
||||
if groupState.ChannelID != 0 && groupState.CreatePending {
|
||||
return fmt.Errorf("group %d has both channel identity and pending create", group.Index)
|
||||
}
|
||||
if groupState.InvitePendingEnd < groupState.InviteCursor || groupState.InvitePendingEnd > invitees {
|
||||
return fmt.Errorf("group %d has invalid pending invite range", group.Index)
|
||||
}
|
||||
if groupState.ChannelID == 0 && (groupState.InviteCursor != 0 || groupState.InvitePendingEnd != 0) {
|
||||
return fmt.Errorf("group %d has invite progress without a channel identity", group.Index)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func WriteDatasetSeedState(path string, dataset *Dataset, state *DatasetSeedState) error {
|
||||
if err := state.Validate(dataset); err != nil {
|
||||
return err
|
||||
}
|
||||
state.UpdatedAt = time.Now().UTC()
|
||||
data, err := json.MarshalIndent(state, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode dataset seed state: %w", err)
|
||||
}
|
||||
return writeFileAtomic(path, append(data, '\n'), 0o600)
|
||||
}
|
||||
|
||||
func LoadDatasetSeedState(path string, dataset *Dataset) (*DatasetSeedState, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if os.IsNotExist(err) {
|
||||
return NewDatasetSeedState(dataset)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read dataset seed state: %w", err)
|
||||
}
|
||||
var state DatasetSeedState
|
||||
decoder := json.NewDecoder(strings.NewReader(string(data)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&state); err != nil {
|
||||
return nil, fmt.Errorf("decode dataset seed state: %w", err)
|
||||
}
|
||||
if err := state.Validate(dataset); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &state, nil
|
||||
}
|
||||
|
||||
func datasetHistoryTaskCounts(dataset *Dataset) []int {
|
||||
counts := make([]int, dataset.Config.Accounts)
|
||||
for _, group := range dataset.Groups {
|
||||
for message := 0; message < group.HistoryMessages; message++ {
|
||||
counts[group.MemberAccounts[message%len(group.MemberAccounts)]]++
|
||||
}
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
func LoadDataset(path string) (*Dataset, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read dataset: %w", err)
|
||||
}
|
||||
var dataset Dataset
|
||||
decoder := json.NewDecoder(strings.NewReader(string(data)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&dataset); err != nil {
|
||||
return nil, fmt.Errorf("decode dataset: %w", err)
|
||||
}
|
||||
if err := dataset.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dataset, nil
|
||||
}
|
||||
|
||||
func (d *Dataset) planHash() (string, error) {
|
||||
type immutableDataset struct {
|
||||
Version int `json:"version"`
|
||||
RunID string `json:"run_id"`
|
||||
Config DatasetConfig `json:"config"`
|
||||
PrivateEdges []DatasetPrivateEdge `json:"private_edges"`
|
||||
Groups []struct {
|
||||
Index int `json:"index"`
|
||||
Tier string `json:"tier"`
|
||||
Title string `json:"title"`
|
||||
About string `json:"about"`
|
||||
CreatorAccount int `json:"creator_account"`
|
||||
MemberAccounts []int `json:"member_accounts"`
|
||||
HistoryMessages int `json:"history_messages"`
|
||||
} `json:"groups"`
|
||||
}
|
||||
immutable := immutableDataset{Version: d.Version, RunID: d.RunID, Config: d.Config, PrivateEdges: d.PrivateEdges}
|
||||
for _, group := range d.Groups {
|
||||
immutable.Groups = append(immutable.Groups, struct {
|
||||
Index int `json:"index"`
|
||||
Tier string `json:"tier"`
|
||||
Title string `json:"title"`
|
||||
About string `json:"about"`
|
||||
CreatorAccount int `json:"creator_account"`
|
||||
MemberAccounts []int `json:"member_accounts"`
|
||||
HistoryMessages int `json:"history_messages"`
|
||||
}{group.Index, group.Tier, group.Title, group.About, group.CreatorAccount, group.MemberAccounts, group.HistoryMessages})
|
||||
}
|
||||
data, err := json.Marshal(immutable)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func cyclicMembers(accounts, start, count int) []int {
|
||||
if count <= 0 {
|
||||
return nil
|
||||
}
|
||||
members := make([]int, count)
|
||||
for i := range members {
|
||||
members[i] = (start + i) % accounts
|
||||
}
|
||||
sort.Ints(members)
|
||||
return members
|
||||
}
|
||||
|
||||
func stableDatasetID(seed int64, namespace string, values ...int) int64 {
|
||||
h := sha256.New()
|
||||
var buf [8]byte
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(seed))
|
||||
_, _ = h.Write(buf[:])
|
||||
_, _ = h.Write([]byte(namespace))
|
||||
for _, value := range values {
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(value))
|
||||
_, _ = h.Write(buf[:])
|
||||
}
|
||||
sum := h.Sum(nil)
|
||||
id := int64(binary.LittleEndian.Uint64(sum[:8]) & ^(uint64(1) << 63))
|
||||
if id == 0 {
|
||||
return 1
|
||||
}
|
||||
return id
|
||||
}
|
||||
179
internal/loadharness/dataset_test.go
Normal file
179
internal/loadharness/dataset_test.go
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPlanDatasetDefaultTopology(t *testing.T) {
|
||||
dataset, err := PlanDataset(DefaultDatasetConfig(1000))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := len(dataset.PrivateEdges), 10000; got != want {
|
||||
t.Fatalf("private edges = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := len(dataset.Groups), 510; got != want {
|
||||
t.Fatalf("groups = %d, want %d", got, want)
|
||||
}
|
||||
memberships := make([]int, dataset.Config.Accounts)
|
||||
totalMemberships := 0
|
||||
for _, group := range dataset.Groups {
|
||||
totalMemberships += len(group.MemberAccounts)
|
||||
for _, account := range group.MemberAccounts {
|
||||
memberships[account]++
|
||||
}
|
||||
}
|
||||
if totalMemberships != 44000 {
|
||||
t.Fatalf("memberships = %d, want 44000", totalMemberships)
|
||||
}
|
||||
if got, want := memberships[999]+20, 44; got != want {
|
||||
t.Fatalf("regular account dialogs = %d, want %d", got, want)
|
||||
}
|
||||
if got, want := memberships[0]+20, 244; got != want {
|
||||
t.Fatalf("heavy account dialogs = %d, want %d", got, want)
|
||||
}
|
||||
if dataset.PlanSHA256 == "" || dataset.PrivateEdges[0].RandomID == 0 {
|
||||
t.Fatal("dataset is missing stable identity")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanDatasetIsDeterministic(t *testing.T) {
|
||||
cfg := DefaultDatasetConfig(100)
|
||||
first, err := PlanDataset(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := PlanDataset(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.PlanSHA256 != second.PlanSHA256 {
|
||||
t.Fatalf("plan hash changed: %s != %s", first.PlanSHA256, second.PlanSHA256)
|
||||
}
|
||||
if first.PrivateEdges[37] != second.PrivateEdges[37] {
|
||||
t.Fatal("private edge plan changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetRandomIDsAreNamespacedByScale(t *testing.T) {
|
||||
ten, err := PlanDataset(DefaultDatasetConfig(10))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hundred, err := PlanDataset(DefaultDatasetConfig(100))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ten.PrivateEdges[0].RandomID == hundred.PrivateEdges[0].RandomID {
|
||||
t.Fatal("private random_id collided across dataset scales")
|
||||
}
|
||||
if stableDatasetID(7, "offline", 10, 0) == stableDatasetID(7, "offline", 100, 0) {
|
||||
t.Fatal("offline random_id collided across dataset scales")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetRoundTripAndImmutableHash(t *testing.T) {
|
||||
dataset, err := PlanDataset(DefaultDatasetConfig(20))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "dataset.json")
|
||||
if err := WriteDataset(path, dataset); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := LoadDataset(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loaded.PlanSHA256 != dataset.PlanSHA256 || len(loaded.Groups) != len(dataset.Groups) {
|
||||
t.Fatal("dataset round trip changed the plan")
|
||||
}
|
||||
loaded.Groups[0].Title += " changed"
|
||||
if err := loaded.Validate(); err == nil {
|
||||
t.Fatal("mutated immutable plan passed validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetSeedStateRoundTrip(t *testing.T) {
|
||||
dataset, err := PlanDataset(DefaultDatasetConfig(20))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err := NewDatasetSeedState(dataset)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state.PrivateSentByAccount[0] = 2
|
||||
state.Groups[0].ChannelID = 11
|
||||
state.Groups[0].AccessHash = 22
|
||||
state.Groups[0].InviteCursor = 3
|
||||
state.Groups[0].InvitePendingEnd = 7
|
||||
path := filepath.Join(t.TempDir(), "seed-state.json")
|
||||
if err := WriteDatasetSeedState(path, dataset, state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := LoadDatasetSeedState(path, dataset)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loaded.PrivateSentByAccount[0] != 2 || loaded.Groups[0].ChannelID != 11 || loaded.Groups[0].InviteCursor != 3 || loaded.Groups[0].InvitePendingEnd != 7 {
|
||||
t.Fatal("seed state round trip changed progress")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetSeedStateAcceptsLegacyMissingRichVector(t *testing.T) {
|
||||
dataset, err := PlanDataset(DefaultDatasetConfig(20))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err := NewDatasetSeedState(dataset)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state.RichStateByAccount = nil
|
||||
if err := state.Validate(dataset); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state.RichStateByAccount = []bool{true}
|
||||
if err := state.Validate(dataset); err == nil {
|
||||
t.Fatal("partial rich-state vector passed validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetSeedStateRejectsImpossiblePendingOperation(t *testing.T) {
|
||||
dataset, err := PlanDataset(DefaultDatasetConfig(20))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err := NewDatasetSeedState(dataset)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state.Groups[0].CreatePending = true
|
||||
state.Groups[0].ChannelID = 11
|
||||
state.Groups[0].AccessHash = 22
|
||||
if err := state.Validate(dataset); err == nil {
|
||||
t.Fatal("channel identity plus pending create passed validation")
|
||||
}
|
||||
state.Groups[0].CreatePending = false
|
||||
state.Groups[0].ChannelID = 0
|
||||
state.Groups[0].AccessHash = 0
|
||||
state.Groups[0].InvitePendingEnd = 1
|
||||
if err := state.Validate(dataset); err == nil {
|
||||
t.Fatal("invite progress without channel identity passed validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetConfigRejectsUnsafeScale(t *testing.T) {
|
||||
cfg := DefaultDatasetConfig(1000)
|
||||
cfg.HotGroups = maxDatasetGroups + 1
|
||||
if _, err := PlanDataset(cfg); err == nil {
|
||||
t.Fatal("oversized dataset passed validation")
|
||||
}
|
||||
cfg = DefaultDatasetConfig(10)
|
||||
cfg.PrivateFanout = 10
|
||||
if _, err := PlanDataset(cfg); err == nil {
|
||||
t.Fatal("self-wrapping private fanout passed validation")
|
||||
}
|
||||
}
|
||||
230
internal/loadharness/delivery.go
Normal file
230
internal/loadharness/delivery.go
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/iamxvbaba/td/tg"
|
||||
)
|
||||
|
||||
const deliveryMarkerPrefix = "telesrv-load-v3"
|
||||
|
||||
type deliverySource uint8
|
||||
|
||||
const (
|
||||
deliveryLive deliverySource = 1 << iota
|
||||
deliveryDifference
|
||||
)
|
||||
|
||||
type deliveryExpectation struct {
|
||||
senderUserID int64
|
||||
targetUserID int64
|
||||
startedAt time.Time
|
||||
committed bool
|
||||
}
|
||||
|
||||
type deliveryObservation struct {
|
||||
sources deliverySource
|
||||
repeats uint64
|
||||
firstAt time.Time
|
||||
}
|
||||
|
||||
type deliveryTracker struct {
|
||||
mu sync.Mutex
|
||||
runID string
|
||||
expected map[string]deliveryExpectation
|
||||
observations map[string]map[int64]deliveryObservation
|
||||
}
|
||||
|
||||
type DeliveryReport struct {
|
||||
RunID string `json:"run_id"`
|
||||
Expected uint64 `json:"expected"`
|
||||
Delivered uint64 `json:"delivered"`
|
||||
Missing uint64 `json:"missing"`
|
||||
LiveDelivered uint64 `json:"live_delivered"`
|
||||
DifferenceRecovered uint64 `json:"difference_recovered"`
|
||||
DuplicateObservations uint64 `json:"duplicate_observations"`
|
||||
WrongAccountObserved uint64 `json:"wrong_account_observed"`
|
||||
UnmatchedMarkers uint64 `json:"unmatched_markers"`
|
||||
E2EP50MS float64 `json:"e2e_p50_ms"`
|
||||
E2EP95MS float64 `json:"e2e_p95_ms"`
|
||||
E2EP99MS float64 `json:"e2e_p99_ms"`
|
||||
E2EMaxMS float64 `json:"e2e_max_ms"`
|
||||
}
|
||||
|
||||
func newDeliveryTracker(runID string) *deliveryTracker {
|
||||
return &deliveryTracker{
|
||||
runID: runID,
|
||||
expected: make(map[string]deliveryExpectation),
|
||||
observations: make(map[string]map[int64]deliveryObservation),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *deliveryTracker) marker(senderIndex int, sequence uint64) string {
|
||||
return fmt.Sprintf("%s/%s/%d/%d", deliveryMarkerPrefix, t.runID, senderIndex, sequence)
|
||||
}
|
||||
|
||||
func (t *deliveryTracker) expect(marker string, senderUserID, targetUserID int64) {
|
||||
t.begin(marker, senderUserID, targetUserID, time.Now())
|
||||
t.finish(marker, true)
|
||||
}
|
||||
|
||||
func (t *deliveryTracker) begin(marker string, senderUserID, targetUserID int64, startedAt time.Time) {
|
||||
if t == nil || !t.matches(marker) {
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.expected[marker] = deliveryExpectation{senderUserID: senderUserID, targetUserID: targetUserID, startedAt: startedAt}
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
func (t *deliveryTracker) finish(marker string, success bool) {
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
expectation, ok := t.expected[marker]
|
||||
if ok && success {
|
||||
expectation.committed = true
|
||||
t.expected[marker] = expectation
|
||||
} else if ok {
|
||||
delete(t.expected, marker)
|
||||
}
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
func (t *deliveryTracker) observe(marker string, accountUserID int64, source deliverySource) {
|
||||
if t == nil || accountUserID <= 0 || !t.matches(marker) {
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
byAccount := t.observations[marker]
|
||||
if byAccount == nil {
|
||||
byAccount = make(map[int64]deliveryObservation)
|
||||
t.observations[marker] = byAccount
|
||||
}
|
||||
observation := byAccount[accountUserID]
|
||||
if observation.firstAt.IsZero() {
|
||||
observation.firstAt = time.Now()
|
||||
}
|
||||
if observation.sources&source != 0 {
|
||||
observation.repeats++
|
||||
}
|
||||
observation.sources |= source
|
||||
byAccount[accountUserID] = observation
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
func (t *deliveryTracker) matches(marker string) bool {
|
||||
parts := strings.Split(marker, "/")
|
||||
if len(parts) != 4 || parts[0] != deliveryMarkerPrefix || parts[1] != t.runID {
|
||||
return false
|
||||
}
|
||||
if _, err := strconv.Atoi(parts[2]); err != nil {
|
||||
return false
|
||||
}
|
||||
sequence, err := strconv.ParseUint(parts[3], 10, 64)
|
||||
return err == nil && sequence > 0
|
||||
}
|
||||
|
||||
func (t *deliveryTracker) report() DeliveryReport {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
report := DeliveryReport{RunID: t.runID}
|
||||
latencies := make([]time.Duration, 0, len(t.expected))
|
||||
for marker, expectation := range t.expected {
|
||||
if !expectation.committed {
|
||||
continue
|
||||
}
|
||||
report.Expected++
|
||||
byAccount := t.observations[marker]
|
||||
observation, delivered := byAccount[expectation.targetUserID]
|
||||
if delivered {
|
||||
report.Delivered++
|
||||
switch {
|
||||
case observation.sources&deliveryLive != 0:
|
||||
report.LiveDelivered++
|
||||
case observation.sources&deliveryDifference != 0:
|
||||
report.DifferenceRecovered++
|
||||
}
|
||||
report.DuplicateObservations += observation.repeats
|
||||
if !expectation.startedAt.IsZero() && !observation.firstAt.Before(expectation.startedAt) {
|
||||
latencies = append(latencies, observation.firstAt.Sub(expectation.startedAt))
|
||||
}
|
||||
} else {
|
||||
report.Missing++
|
||||
}
|
||||
for accountUserID, other := range byAccount {
|
||||
if accountUserID == expectation.targetUserID || accountUserID == expectation.senderUserID {
|
||||
continue
|
||||
}
|
||||
report.WrongAccountObserved++
|
||||
report.DuplicateObservations += other.repeats
|
||||
}
|
||||
}
|
||||
for marker := range t.observations {
|
||||
if expectation, ok := t.expected[marker]; !ok || !expectation.committed {
|
||||
report.UnmatchedMarkers++
|
||||
}
|
||||
}
|
||||
if len(latencies) > 0 {
|
||||
report.E2EP50MS = deliveryQuantileMS(latencies, 0.50)
|
||||
report.E2EP95MS = deliveryQuantileMS(latencies, 0.95)
|
||||
report.E2EP99MS = deliveryQuantileMS(latencies, 0.99)
|
||||
report.E2EMaxMS = deliveryQuantileMS(latencies, 1)
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
func deliveryQuantileMS(values []time.Duration, quantile float64) float64 {
|
||||
sorted := append([]time.Duration(nil), values...)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
|
||||
index := int(math.Ceil(float64(len(sorted))*quantile)) - 1
|
||||
index = max(0, min(index, len(sorted)-1))
|
||||
return durationMS(sorted[index])
|
||||
}
|
||||
|
||||
func observeUpdatesClass(tracker *deliveryTracker, accountUserID int64, updates tg.UpdatesClass, source deliverySource) {
|
||||
switch value := updates.(type) {
|
||||
case *tg.Updates:
|
||||
observeUpdateClasses(tracker, accountUserID, value.Updates, source)
|
||||
case *tg.UpdatesCombined:
|
||||
observeUpdateClasses(tracker, accountUserID, value.Updates, source)
|
||||
case *tg.UpdateShort:
|
||||
observeUpdateClass(tracker, accountUserID, value.Update, source)
|
||||
case *tg.UpdateShortMessage:
|
||||
tracker.observe(value.Message, accountUserID, source)
|
||||
}
|
||||
}
|
||||
|
||||
func observeUpdateClasses(tracker *deliveryTracker, accountUserID int64, updates []tg.UpdateClass, source deliverySource) {
|
||||
for _, update := range updates {
|
||||
observeUpdateClass(tracker, accountUserID, update, source)
|
||||
}
|
||||
}
|
||||
|
||||
func observeUpdateClass(tracker *deliveryTracker, accountUserID int64, update tg.UpdateClass, source deliverySource) {
|
||||
switch value := update.(type) {
|
||||
case *tg.UpdateNewMessage:
|
||||
observeMessageClass(tracker, accountUserID, value.Message, source)
|
||||
case *tg.UpdateNewChannelMessage:
|
||||
observeMessageClass(tracker, accountUserID, value.Message, source)
|
||||
}
|
||||
}
|
||||
|
||||
func observeMessageClasses(tracker *deliveryTracker, accountUserID int64, messages []tg.MessageClass, source deliverySource) {
|
||||
for _, message := range messages {
|
||||
observeMessageClass(tracker, accountUserID, message, source)
|
||||
}
|
||||
}
|
||||
|
||||
func observeMessageClass(tracker *deliveryTracker, accountUserID int64, message tg.MessageClass, source deliverySource) {
|
||||
if value, ok := message.(*tg.Message); ok {
|
||||
tracker.observe(value.Message, accountUserID, source)
|
||||
}
|
||||
}
|
||||
83
internal/loadharness/delivery_test.go
Normal file
83
internal/loadharness/delivery_test.go
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/iamxvbaba/td/tg"
|
||||
)
|
||||
|
||||
func TestDeliveryTrackerReconcilesObservationBeforeSendReturn(t *testing.T) {
|
||||
tracker := newDeliveryTracker("run")
|
||||
marker := tracker.marker(7, 9)
|
||||
|
||||
tracker.observe(marker, 200, deliveryLive)
|
||||
tracker.expect(marker, 100, 200)
|
||||
|
||||
report := tracker.report()
|
||||
if report.Expected != 1 || report.Delivered != 1 || report.LiveDelivered != 1 || report.Missing != 0 {
|
||||
t.Fatalf("unexpected report: %+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryTrackerMeasuresFromSendStart(t *testing.T) {
|
||||
tracker := newDeliveryTracker("run")
|
||||
marker := tracker.marker(1, 1)
|
||||
tracker.begin(marker, 100, 200, time.Now().Add(-25*time.Millisecond))
|
||||
tracker.observe(marker, 200, deliveryLive)
|
||||
tracker.finish(marker, true)
|
||||
|
||||
report := tracker.report()
|
||||
if report.E2EP50MS < 20 || report.E2EP99MS < report.E2EP50MS || report.E2EMaxMS < report.E2EP99MS {
|
||||
t.Fatalf("unexpected e2e latency report: %+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryTrackerSeparatesDifferenceRecoveryAndDuplicates(t *testing.T) {
|
||||
tracker := newDeliveryTracker("run")
|
||||
liveMarker := tracker.marker(1, 1)
|
||||
differenceMarker := tracker.marker(2, 1)
|
||||
tracker.expect(liveMarker, 100, 200)
|
||||
tracker.expect(differenceMarker, 200, 300)
|
||||
|
||||
tracker.observe(liveMarker, 200, deliveryLive)
|
||||
tracker.observe(liveMarker, 200, deliveryLive)
|
||||
tracker.observe(liveMarker, 200, deliveryDifference)
|
||||
tracker.observe(differenceMarker, 300, deliveryDifference)
|
||||
|
||||
report := tracker.report()
|
||||
if report.Delivered != 2 || report.LiveDelivered != 1 || report.DifferenceRecovered != 1 {
|
||||
t.Fatalf("unexpected delivery sources: %+v", report)
|
||||
}
|
||||
if report.DuplicateObservations != 1 {
|
||||
t.Fatalf("duplicate observations = %d, want 1", report.DuplicateObservations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserveUpdatesClassExtractsPrivateMessages(t *testing.T) {
|
||||
tracker := newDeliveryTracker("run")
|
||||
shortMarker := tracker.marker(1, 1)
|
||||
fullMarker := tracker.marker(2, 1)
|
||||
tracker.expect(shortMarker, 100, 200)
|
||||
tracker.expect(fullMarker, 300, 200)
|
||||
|
||||
observeUpdatesClass(tracker, 200, &tg.UpdateShortMessage{Message: shortMarker}, deliveryLive)
|
||||
observeUpdatesClass(tracker, 200, &tg.Updates{Updates: []tg.UpdateClass{
|
||||
&tg.UpdateNewMessage{Message: &tg.Message{Message: fullMarker}},
|
||||
}}, deliveryLive)
|
||||
|
||||
report := tracker.report()
|
||||
if report.Delivered != 2 || report.Missing != 0 {
|
||||
t.Fatalf("unexpected report: %+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryTrackerRejectsForeignAndMalformedMarkers(t *testing.T) {
|
||||
tracker := newDeliveryTracker("run")
|
||||
tracker.observe("telesrv-load-v3/other/1/1", 200, deliveryLive)
|
||||
tracker.observe("telesrv-load-v3/run/not-an-index/1", 200, deliveryLive)
|
||||
|
||||
if report := tracker.report(); report.UnmatchedMarkers != 0 {
|
||||
t.Fatalf("foreign marker entered report: %+v", report)
|
||||
}
|
||||
}
|
||||
778
internal/loadharness/mutate.go
Normal file
778
internal/loadharness/mutate.go
Normal file
|
|
@ -0,0 +1,778 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/iamxvbaba/td/tg"
|
||||
"github.com/iamxvbaba/td/tgerr"
|
||||
)
|
||||
|
||||
const OfflineMutationVersion = 1
|
||||
|
||||
type OfflineMutationChannelPlan struct {
|
||||
GroupPosition int
|
||||
Messages int
|
||||
}
|
||||
|
||||
type OfflineMutationChannelState struct {
|
||||
GroupIndex int `json:"group_index"`
|
||||
MessageIDs []int `json:"message_ids"`
|
||||
LatestPts int `json:"latest_pts,omitempty"`
|
||||
EditPending bool `json:"edit_pending,omitempty"`
|
||||
EditDone bool `json:"edit_done,omitempty"`
|
||||
DeletePending bool `json:"delete_pending,omitempty"`
|
||||
DeleteDone bool `json:"delete_done,omitempty"`
|
||||
PinPending bool `json:"pin_pending,omitempty"`
|
||||
PinDone bool `json:"pin_done,omitempty"`
|
||||
}
|
||||
|
||||
type OfflineMutationState struct {
|
||||
Version int `json:"version"`
|
||||
DatasetSHA256 string `json:"dataset_sha256"`
|
||||
SeedIdentitySHA string `json:"seed_identity_sha256"`
|
||||
BaselineStateSHA string `json:"baseline_state_sha256"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
PrivateMessageIDs []int `json:"private_message_ids"`
|
||||
AccountObservedPts []int `json:"account_observed_pts"`
|
||||
Channels []OfflineMutationChannelState `json:"channels"`
|
||||
}
|
||||
|
||||
type MutateOfflineConfig struct {
|
||||
ManifestPath string
|
||||
SessionKeyPath string
|
||||
RSAKeyOverride string
|
||||
DatasetPath string
|
||||
SeedStatePath string
|
||||
ClientStatePath string
|
||||
MutationStatePath string
|
||||
Concurrency int
|
||||
OperationTimeout time.Duration
|
||||
}
|
||||
|
||||
type MutationEvent struct {
|
||||
Phase string
|
||||
Completed int
|
||||
Total int
|
||||
Account int
|
||||
Err error
|
||||
}
|
||||
|
||||
type MutationResult struct {
|
||||
PrivateMessages int
|
||||
ChannelMessages int
|
||||
DirtyChannels int
|
||||
Edited int
|
||||
Deleted int
|
||||
Pinned int
|
||||
}
|
||||
|
||||
func (c MutateOfflineConfig) validate() error {
|
||||
if c.ManifestPath == "" || c.SessionKeyPath == "" || c.DatasetPath == "" || c.SeedStatePath == "" || c.ClientStatePath == "" || c.MutationStatePath == "" {
|
||||
return errors.New("manifest, session-key, dataset, seed-state, client-state and mutation-state paths are required")
|
||||
}
|
||||
if c.Concurrency <= 0 || c.Concurrency > 64 {
|
||||
return errors.New("offline mutation concurrency must be between 1 and 64")
|
||||
}
|
||||
if c.OperationTimeout <= 0 {
|
||||
return errors.New("offline mutation operation timeout must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MutateOffline creates gaps only after a complete baseline has been locked.
|
||||
// Message sends use stable random_id values. Mutable channel operations use a
|
||||
// pending journal and public read-back reconciliation before a resumed run can
|
||||
// declare them complete.
|
||||
func MutateOffline(ctx context.Context, cfg MutateOfflineConfig, progress func(MutationEvent)) (*MutationResult, error) {
|
||||
if err := cfg.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
manifest, err := LoadManifest(cfg.ManifestPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dataset, err := LoadDataset(cfg.DatasetPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targets, err := seedPrimaryTargets(manifest, dataset.Config.Accounts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seedState, err := LoadDatasetSeedState(cfg.SeedStatePath, dataset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seedJournal := &seedJournal{dataset: dataset, state: seedState}
|
||||
if err := seedJournal.assertComplete(); err != nil {
|
||||
return nil, fmt.Errorf("offline mutation requires a complete seed: %w", err)
|
||||
}
|
||||
clientState, err := LoadClientState(cfg.ClientStatePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := clientState.Validate(dataset, seedState, targets); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
baselineSHA, err := fileSHA256(cfg.ClientStatePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seedIdentity, err := seedIdentitySHA256(seedState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plan := planOfflineMutation(dataset)
|
||||
state, err := loadOrCreateOfflineMutationState(cfg.MutationStatePath, dataset, seedIdentity, baselineSHA, plan)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
journal := &mutationJournal{path: cfg.MutationStatePath, dataset: dataset, plan: plan, state: state}
|
||||
if err := journal.persist(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := LoadSessionKey(cfg.SessionKeyPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
publicKey, err := loadManifestPublicKey(cfg.ManifestPath, manifest.Endpoint, cfg.RSAKeyOverride)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
accounts := make([]int, dataset.Config.Accounts)
|
||||
for account := range accounts {
|
||||
accounts[account] = account
|
||||
}
|
||||
|
||||
if err := runSeedAccountPhase(ctx, "mutate-private", accounts, cfg.Concurrency, mutationProgressAdapter("private", progress), func(ctx context.Context, account int) error {
|
||||
if journal.privateMessageID(account) != 0 {
|
||||
return nil
|
||||
}
|
||||
return withAuthorizedSeedSession(ctx, SeedConfig{ManifestPath: cfg.ManifestPath, OperationTimeout: cfg.OperationTimeout}, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
|
||||
recipient := (account + 1) % dataset.Config.Accounts
|
||||
marker := offlinePrivateMarker(dataset, account, recipient)
|
||||
updates, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
|
||||
return raw.MessagesSendMessage(rpcCtx, &tg.MessagesSendMessageRequest{
|
||||
Peer: &tg.InputPeerUser{UserID: targets[recipient].UserID, AccessHash: targets[recipient].AccessHash},
|
||||
Message: marker, RandomID: stableDatasetID(dataset.Config.Seed, "offline-private", dataset.Config.Accounts, account, recipient),
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("messages.sendMessage: %w", err)
|
||||
}
|
||||
observation, err := sentMessageObservation(updates, clientPeerKey{typ: "user", id: targets[recipient].UserID}, marker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return journal.commitPrivate(account, observation.ID, observation.Pts)
|
||||
})
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
channelTasks := offlineChannelTasks(dataset, plan)
|
||||
channelAccounts := make([]int, 0, len(channelTasks))
|
||||
for account := range channelTasks {
|
||||
channelAccounts = append(channelAccounts, account)
|
||||
}
|
||||
sort.Ints(channelAccounts)
|
||||
if err := runSeedAccountPhase(ctx, "mutate-channel", channelAccounts, cfg.Concurrency, mutationProgressAdapter("channel", progress), func(ctx context.Context, account int) error {
|
||||
return withAuthorizedSeedSession(ctx, SeedConfig{ManifestPath: cfg.ManifestPath, OperationTimeout: cfg.OperationTimeout}, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
|
||||
for _, task := range channelTasks[account] {
|
||||
if journal.channelMessageID(task.PlanPosition, task.MessageIndex) != 0 {
|
||||
continue
|
||||
}
|
||||
channelPlan := plan[task.PlanPosition]
|
||||
group := dataset.Groups[channelPlan.GroupPosition]
|
||||
channel := seedState.Groups[channelPlan.GroupPosition]
|
||||
marker := offlineChannelMarker(dataset, group, task.MessageIndex)
|
||||
updates, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
|
||||
return raw.MessagesSendMessage(rpcCtx, &tg.MessagesSendMessageRequest{
|
||||
Peer: &tg.InputPeerChannel{ChannelID: channel.ChannelID, AccessHash: channel.AccessHash},
|
||||
Message: marker, RandomID: stableDatasetID(dataset.Config.Seed, "offline-channel", dataset.Config.Accounts, group.Index, task.MessageIndex),
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("group %d message %d: %w", group.Index, task.MessageIndex, err)
|
||||
}
|
||||
observation, err := sentMessageObservation(updates, clientPeerKey{typ: "channel", id: channel.ChannelID}, marker)
|
||||
if err != nil {
|
||||
return fmt.Errorf("group %d message %d: %w", group.Index, task.MessageIndex, err)
|
||||
}
|
||||
if err := journal.commitChannelMessage(task.PlanPosition, task.MessageIndex, observation.ID, observation.Pts); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The first dirty channel deliberately exceeds the page limit and owns the
|
||||
// edit, delete and pin events. The creator authors the edited message and can
|
||||
// administratively delete/pin the two newest messages in one session.
|
||||
if len(plan) == 0 || plan[0].Messages < 120 {
|
||||
return nil, errors.New("offline mutation plan has no multi-page channel")
|
||||
}
|
||||
actionGroup := dataset.Groups[plan[0].GroupPosition]
|
||||
if err := runSeedAccountPhase(ctx, "mutate-actions", []int{actionGroup.CreatorAccount}, 1, mutationProgressAdapter("actions", progress), func(ctx context.Context, account int) error {
|
||||
return withAuthorizedSeedSession(ctx, SeedConfig{ManifestPath: cfg.ManifestPath, OperationTimeout: cfg.OperationTimeout}, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
|
||||
return applyOfflineChannelActions(ctx, cfg.OperationTimeout, dataset, seedState, plan, journal, raw)
|
||||
})
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := journal.assertComplete(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return offlineMutationResult(plan, state), nil
|
||||
}
|
||||
|
||||
func planOfflineMutation(dataset *Dataset) []OfflineMutationChannelPlan {
|
||||
plan := make([]OfflineMutationChannelPlan, 0, 40)
|
||||
counts := map[string]int{"hot": 10, "medium": 10, "small": 10, "heavy": 10}
|
||||
messages := map[string]int{"hot": 3, "medium": 2, "small": 1, "heavy": 3}
|
||||
seen := make(map[string]int)
|
||||
for position, group := range dataset.Groups {
|
||||
if seen[group.Tier] >= counts[group.Tier] {
|
||||
continue
|
||||
}
|
||||
count := messages[group.Tier]
|
||||
switch len(plan) {
|
||||
case 0:
|
||||
count = 120
|
||||
case 1:
|
||||
// Exactly one full page complements the first channel's >limit
|
||||
// channelDifferenceTooLong snapshot path.
|
||||
count = 100
|
||||
}
|
||||
plan = append(plan, OfflineMutationChannelPlan{GroupPosition: position, Messages: count})
|
||||
seen[group.Tier]++
|
||||
}
|
||||
if len(plan) == 0 && len(dataset.Groups) != 0 {
|
||||
plan = append(plan, OfflineMutationChannelPlan{GroupPosition: 0, Messages: 120})
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
type offlineChannelTask struct {
|
||||
PlanPosition int
|
||||
MessageIndex int
|
||||
}
|
||||
|
||||
func offlineChannelTasks(dataset *Dataset, plan []OfflineMutationChannelPlan) map[int][]offlineChannelTask {
|
||||
tasks := make(map[int][]offlineChannelTask)
|
||||
for planPosition, channelPlan := range plan {
|
||||
group := dataset.Groups[channelPlan.GroupPosition]
|
||||
for message := 0; message < channelPlan.Messages; message++ {
|
||||
account := offlineMutationSender(group, message)
|
||||
tasks[account] = append(tasks[account], offlineChannelTask{PlanPosition: planPosition, MessageIndex: message})
|
||||
}
|
||||
}
|
||||
return tasks
|
||||
}
|
||||
|
||||
func offlineMutationSender(group DatasetGroup, message int) int {
|
||||
if message < 3 {
|
||||
return group.CreatorAccount
|
||||
}
|
||||
return group.MemberAccounts[message%len(group.MemberAccounts)]
|
||||
}
|
||||
|
||||
func offlinePrivateMarker(dataset *Dataset, sender, recipient int) string {
|
||||
return fmt.Sprintf("[%s offline private %04d->%04d]", dataset.RunID, sender, recipient)
|
||||
}
|
||||
|
||||
func offlineChannelMarker(dataset *Dataset, group DatasetGroup, message int) string {
|
||||
return fmt.Sprintf("[%s offline channel %04d message %04d]", dataset.RunID, group.Index, message+1)
|
||||
}
|
||||
|
||||
type messageObservation struct {
|
||||
ID int
|
||||
Pts int
|
||||
}
|
||||
|
||||
func sentMessageObservation(updates tg.UpdatesClass, peer clientPeerKey, marker string) (messageObservation, error) {
|
||||
switch value := updates.(type) {
|
||||
case *tg.UpdateShortSentMessage:
|
||||
if value.ID > 0 {
|
||||
return messageObservation{ID: value.ID, Pts: value.Pts}, nil
|
||||
}
|
||||
case *tg.UpdateShortMessage:
|
||||
if value.ID > 0 && value.Message == marker {
|
||||
return messageObservation{ID: value.ID, Pts: value.Pts}, nil
|
||||
}
|
||||
case *tg.Updates:
|
||||
return sentMessageObservationFromUpdates(value.Updates, peer, marker)
|
||||
case *tg.UpdatesCombined:
|
||||
return sentMessageObservationFromUpdates(value.Updates, peer, marker)
|
||||
}
|
||||
return messageObservation{}, fmt.Errorf("messages.sendMessage returned %T without marker", updates)
|
||||
}
|
||||
|
||||
func sentMessageObservationFromUpdates(updates []tg.UpdateClass, peer clientPeerKey, marker string) (messageObservation, error) {
|
||||
for _, update := range updates {
|
||||
var message tg.MessageClass
|
||||
pts := 0
|
||||
switch value := update.(type) {
|
||||
case *tg.UpdateNewMessage:
|
||||
message, pts = value.Message, value.Pts
|
||||
case *tg.UpdateNewChannelMessage:
|
||||
message, pts = value.Message, value.Pts
|
||||
default:
|
||||
continue
|
||||
}
|
||||
full, ok := message.(*tg.Message)
|
||||
if !ok || full.Message != marker {
|
||||
continue
|
||||
}
|
||||
messagePeer, ok := clientPeerFromTG(full.PeerID)
|
||||
if ok && messagePeer == peer && full.ID > 0 {
|
||||
return messageObservation{ID: full.ID, Pts: pts}, nil
|
||||
}
|
||||
}
|
||||
return messageObservation{}, errors.New("messages.sendMessage updates omitted expected marker")
|
||||
}
|
||||
|
||||
func applyOfflineChannelActions(
|
||||
ctx context.Context,
|
||||
timeout time.Duration,
|
||||
dataset *Dataset,
|
||||
seedState *DatasetSeedState,
|
||||
plan []OfflineMutationChannelPlan,
|
||||
journal *mutationJournal,
|
||||
raw *tg.Client,
|
||||
) error {
|
||||
channelPlan := plan[0]
|
||||
group := dataset.Groups[channelPlan.GroupPosition]
|
||||
channel := seedState.Groups[channelPlan.GroupPosition]
|
||||
state := journal.channel(0)
|
||||
deleteIndex, pinIndex := channelPlan.Messages-2, channelPlan.Messages-1
|
||||
if state.MessageIDs[0] == 0 || state.MessageIDs[deleteIndex] == 0 || state.MessageIDs[pinIndex] == 0 {
|
||||
return errors.New("channel action messages are incomplete")
|
||||
}
|
||||
peer := &tg.InputPeerChannel{ChannelID: channel.ChannelID, AccessHash: channel.AccessHash}
|
||||
inputChannel := &tg.InputChannel{ChannelID: channel.ChannelID, AccessHash: channel.AccessHash}
|
||||
|
||||
if !state.EditDone {
|
||||
if !state.EditPending {
|
||||
if err := journal.beginAction(0, "edit"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
editedMarker := offlineChannelMarker(dataset, group, 0) + " edited"
|
||||
updates, err := rpcWithFloodWaitRetry(ctx, timeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
|
||||
return raw.MessagesEditMessage(rpcCtx, &tg.MessagesEditMessageRequest{Peer: peer, ID: state.MessageIDs[0], Message: editedMarker})
|
||||
})
|
||||
pts := maxPtsFromUpdates(updates)
|
||||
if err != nil {
|
||||
if !tgerr.Is(err, "MESSAGE_NOT_MODIFIED") {
|
||||
return fmt.Errorf("messages.editMessage pending reconciliation: %w", err)
|
||||
}
|
||||
matches, verifyErr := channelMessageMatches(ctx, timeout, raw, inputChannel, state.MessageIDs[0], editedMarker)
|
||||
if verifyErr != nil || !matches {
|
||||
return fmt.Errorf("reconcile messages.editMessage: matched=%v err=%w", matches, verifyErr)
|
||||
}
|
||||
}
|
||||
if err := journal.commitAction(0, "edit", pts); err != nil {
|
||||
return err
|
||||
}
|
||||
state = journal.channel(0)
|
||||
}
|
||||
if !state.DeleteDone {
|
||||
if !state.DeletePending {
|
||||
if err := journal.beginAction(0, "delete"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
affected, err := rpcWithFloodWaitRetry(ctx, timeout, func(rpcCtx context.Context) (*tg.MessagesAffectedMessages, error) {
|
||||
return raw.ChannelsDeleteMessages(rpcCtx, &tg.ChannelsDeleteMessagesRequest{Channel: inputChannel, ID: []int{state.MessageIDs[deleteIndex]}})
|
||||
})
|
||||
pts := 0
|
||||
if affected != nil {
|
||||
pts = affected.Pts
|
||||
}
|
||||
if err != nil {
|
||||
deleted, verifyErr := channelMessageDeleted(ctx, timeout, raw, inputChannel, state.MessageIDs[deleteIndex])
|
||||
if verifyErr != nil || !deleted {
|
||||
return fmt.Errorf("reconcile channels.deleteMessages: deleted=%v err=%w (rpc %v)", deleted, verifyErr, err)
|
||||
}
|
||||
}
|
||||
if err := journal.commitAction(0, "delete", pts); err != nil {
|
||||
return err
|
||||
}
|
||||
state = journal.channel(0)
|
||||
}
|
||||
if !state.PinDone {
|
||||
if !state.PinPending {
|
||||
if err := journal.beginAction(0, "pin"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
updates, err := rpcWithFloodWaitRetry(ctx, timeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
|
||||
return raw.MessagesUpdatePinnedMessage(rpcCtx, &tg.MessagesUpdatePinnedMessageRequest{Silent: true, Peer: peer, ID: state.MessageIDs[pinIndex]})
|
||||
})
|
||||
pts := maxPtsFromUpdates(updates)
|
||||
if err != nil {
|
||||
pinned, verifyErr := channelMessagePinned(ctx, timeout, raw, inputChannel, state.MessageIDs[pinIndex])
|
||||
if verifyErr != nil || !pinned {
|
||||
return fmt.Errorf("reconcile messages.updatePinnedMessage: pinned=%v err=%w (rpc %v)", pinned, verifyErr, err)
|
||||
}
|
||||
}
|
||||
if err := journal.commitAction(0, "pin", pts); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func channelMessageMatches(ctx context.Context, timeout time.Duration, raw *tg.Client, channel *tg.InputChannel, messageID int, text string) (bool, error) {
|
||||
messages, err := getChannelMessages(ctx, timeout, raw, channel, messageID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, message := range messages {
|
||||
if full, ok := message.(*tg.Message); ok && full.ID == messageID {
|
||||
return full.Message == text, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func channelMessageDeleted(ctx context.Context, timeout time.Duration, raw *tg.Client, channel *tg.InputChannel, messageID int) (bool, error) {
|
||||
messages, err := getChannelMessages(ctx, timeout, raw, channel, messageID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, message := range messages {
|
||||
if full, ok := message.(*tg.Message); ok && full.ID == messageID {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func getChannelMessages(ctx context.Context, timeout time.Duration, raw *tg.Client, channel *tg.InputChannel, messageID int) ([]tg.MessageClass, error) {
|
||||
rpcCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
response, err := raw.ChannelsGetMessages(rpcCtx, &tg.ChannelsGetMessagesRequest{
|
||||
Channel: channel, ID: []tg.InputMessageClass{&tg.InputMessageID{ID: messageID}},
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
modified, ok := response.AsModified()
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("channels.getMessages returned %T", response)
|
||||
}
|
||||
return modified.GetMessages(), nil
|
||||
}
|
||||
|
||||
func channelMessagePinned(ctx context.Context, timeout time.Duration, raw *tg.Client, channel *tg.InputChannel, messageID int) (bool, error) {
|
||||
rpcCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
response, err := raw.ChannelsGetFullChannel(rpcCtx, channel)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
full, ok := response.FullChat.(*tg.ChannelFull)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("channels.getFullChannel returned %T", response.FullChat)
|
||||
}
|
||||
pinned, ok := full.GetPinnedMsgID()
|
||||
return ok && pinned == messageID, nil
|
||||
}
|
||||
|
||||
func maxPtsFromUpdates(updates tg.UpdatesClass) int {
|
||||
if updates == nil {
|
||||
return 0
|
||||
}
|
||||
maxPts := 0
|
||||
var classes []tg.UpdateClass
|
||||
switch value := updates.(type) {
|
||||
case *tg.UpdateShortSentMessage:
|
||||
return value.Pts
|
||||
case *tg.UpdateShortMessage:
|
||||
return value.Pts
|
||||
case *tg.Updates:
|
||||
classes = value.Updates
|
||||
case *tg.UpdatesCombined:
|
||||
classes = value.Updates
|
||||
}
|
||||
for _, class := range classes {
|
||||
switch value := class.(type) {
|
||||
case *tg.UpdateNewMessage:
|
||||
maxPts = max(maxPts, value.Pts)
|
||||
case *tg.UpdateNewChannelMessage:
|
||||
maxPts = max(maxPts, value.Pts)
|
||||
case *tg.UpdateEditChannelMessage:
|
||||
maxPts = max(maxPts, value.Pts)
|
||||
case *tg.UpdateDeleteChannelMessages:
|
||||
maxPts = max(maxPts, value.Pts)
|
||||
case *tg.UpdatePinnedChannelMessages:
|
||||
maxPts = max(maxPts, value.Pts)
|
||||
}
|
||||
}
|
||||
return maxPts
|
||||
}
|
||||
|
||||
func mutationProgressAdapter(phase string, progress func(MutationEvent)) func(SeedEvent) {
|
||||
if progress == nil {
|
||||
return nil
|
||||
}
|
||||
return func(event SeedEvent) {
|
||||
progress(MutationEvent{Phase: phase, Completed: event.Completed, Total: event.Total, Account: event.Account, Err: event.Err})
|
||||
}
|
||||
}
|
||||
|
||||
func fileSHA256(path string) (string, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return fmt.Sprintf("%x", sum[:]), nil
|
||||
}
|
||||
|
||||
func loadOrCreateOfflineMutationState(
|
||||
path string,
|
||||
dataset *Dataset,
|
||||
seedIdentity, baselineSHA string,
|
||||
plan []OfflineMutationChannelPlan,
|
||||
) (*OfflineMutationState, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if os.IsNotExist(err) {
|
||||
state := &OfflineMutationState{
|
||||
Version: OfflineMutationVersion, DatasetSHA256: dataset.PlanSHA256,
|
||||
SeedIdentitySHA: seedIdentity, BaselineStateSHA: baselineSHA,
|
||||
PrivateMessageIDs: make([]int, dataset.Config.Accounts), AccountObservedPts: make([]int, dataset.Config.Accounts),
|
||||
Channels: make([]OfflineMutationChannelState, len(plan)),
|
||||
}
|
||||
for i, channelPlan := range plan {
|
||||
state.Channels[i] = OfflineMutationChannelState{
|
||||
GroupIndex: dataset.Groups[channelPlan.GroupPosition].Index,
|
||||
MessageIDs: make([]int, channelPlan.Messages),
|
||||
}
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var state OfflineMutationState
|
||||
decoder := json.NewDecoder(strings.NewReader(string(data)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&state); err != nil {
|
||||
return nil, fmt.Errorf("decode offline mutation state: %w", err)
|
||||
}
|
||||
if err := state.Validate(dataset, seedIdentity, baselineSHA, plan); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &state, nil
|
||||
}
|
||||
|
||||
func (s *OfflineMutationState) Validate(dataset *Dataset, seedIdentity, baselineSHA string, plan []OfflineMutationChannelPlan) error {
|
||||
if s == nil || s.Version != OfflineMutationVersion || s.DatasetSHA256 != dataset.PlanSHA256 || s.SeedIdentitySHA != seedIdentity || s.BaselineStateSHA != baselineSHA {
|
||||
return errors.New("offline mutation state does not match baseline dataset")
|
||||
}
|
||||
if len(s.PrivateMessageIDs) != dataset.Config.Accounts || len(s.AccountObservedPts) != dataset.Config.Accounts || len(s.Channels) != len(plan) {
|
||||
return errors.New("offline mutation state dimensions do not match plan")
|
||||
}
|
||||
for account := range s.PrivateMessageIDs {
|
||||
if s.PrivateMessageIDs[account] < 0 || s.AccountObservedPts[account] < 0 {
|
||||
return fmt.Errorf("invalid offline private mutation account %d", account)
|
||||
}
|
||||
}
|
||||
for i, channel := range s.Channels {
|
||||
if channel.GroupIndex != dataset.Groups[plan[i].GroupPosition].Index || len(channel.MessageIDs) != plan[i].Messages || channel.LatestPts < 0 {
|
||||
return fmt.Errorf("invalid offline mutation channel %d", channel.GroupIndex)
|
||||
}
|
||||
for _, messageID := range channel.MessageIDs {
|
||||
if messageID < 0 {
|
||||
return fmt.Errorf("offline mutation channel %d has invalid message id", channel.GroupIndex)
|
||||
}
|
||||
}
|
||||
if (channel.EditDone && channel.EditPending) || (channel.DeleteDone && channel.DeletePending) || (channel.PinDone && channel.PinPending) {
|
||||
return fmt.Errorf("offline mutation channel %d has completed pending action", channel.GroupIndex)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type mutationJournal struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
dataset *Dataset
|
||||
plan []OfflineMutationChannelPlan
|
||||
state *OfflineMutationState
|
||||
}
|
||||
|
||||
func (j *mutationJournal) persistLocked() error {
|
||||
if err := j.state.Validate(j.dataset, j.state.SeedIdentitySHA, j.state.BaselineStateSHA, j.plan); err != nil {
|
||||
return err
|
||||
}
|
||||
j.state.UpdatedAt = time.Now().UTC()
|
||||
data, err := json.MarshalIndent(j.state, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeFileAtomic(j.path, append(data, '\n'), 0o600)
|
||||
}
|
||||
|
||||
func (j *mutationJournal) persist() error {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.persistLocked()
|
||||
}
|
||||
|
||||
func (j *mutationJournal) privateMessageID(account int) int {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.state.PrivateMessageIDs[account]
|
||||
}
|
||||
|
||||
func (j *mutationJournal) channelMessageID(planPosition, message int) int {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.state.Channels[planPosition].MessageIDs[message]
|
||||
}
|
||||
|
||||
func (j *mutationJournal) channel(planPosition int) OfflineMutationChannelState {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
state := j.state.Channels[planPosition]
|
||||
state.MessageIDs = append([]int(nil), state.MessageIDs...)
|
||||
return state
|
||||
}
|
||||
|
||||
func (j *mutationJournal) commitPrivate(account, messageID, pts int) error {
|
||||
if messageID <= 0 || pts < 0 {
|
||||
return errors.New("invalid private mutation observation")
|
||||
}
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
oldID, oldPts := j.state.PrivateMessageIDs[account], j.state.AccountObservedPts[account]
|
||||
j.state.PrivateMessageIDs[account] = messageID
|
||||
j.state.AccountObservedPts[account] = max(oldPts, pts)
|
||||
if err := j.persistLocked(); err != nil {
|
||||
j.state.PrivateMessageIDs[account], j.state.AccountObservedPts[account] = oldID, oldPts
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j *mutationJournal) commitChannelMessage(planPosition, message, messageID, pts int) error {
|
||||
if messageID <= 0 || pts <= 0 {
|
||||
return errors.New("invalid channel mutation observation")
|
||||
}
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
channel := &j.state.Channels[planPosition]
|
||||
oldID, oldPts := channel.MessageIDs[message], channel.LatestPts
|
||||
channel.MessageIDs[message] = messageID
|
||||
channel.LatestPts = max(channel.LatestPts, pts)
|
||||
if err := j.persistLocked(); err != nil {
|
||||
channel.MessageIDs[message], channel.LatestPts = oldID, oldPts
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j *mutationJournal) beginAction(planPosition int, action string) error {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
channel := &j.state.Channels[planPosition]
|
||||
old := *channel
|
||||
switch action {
|
||||
case "edit":
|
||||
channel.EditPending = true
|
||||
case "delete":
|
||||
channel.DeletePending = true
|
||||
case "pin":
|
||||
channel.PinPending = true
|
||||
default:
|
||||
return fmt.Errorf("unknown mutation action %q", action)
|
||||
}
|
||||
if err := j.persistLocked(); err != nil {
|
||||
*channel = old
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j *mutationJournal) commitAction(planPosition int, action string, pts int) error {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
channel := &j.state.Channels[planPosition]
|
||||
old := *channel
|
||||
switch action {
|
||||
case "edit":
|
||||
channel.EditPending, channel.EditDone = false, true
|
||||
case "delete":
|
||||
channel.DeletePending, channel.DeleteDone = false, true
|
||||
case "pin":
|
||||
channel.PinPending, channel.PinDone = false, true
|
||||
default:
|
||||
return fmt.Errorf("unknown mutation action %q", action)
|
||||
}
|
||||
channel.LatestPts = max(channel.LatestPts, pts)
|
||||
if err := j.persistLocked(); err != nil {
|
||||
*channel = old
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j *mutationJournal) assertComplete() error {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
if err := j.state.Validate(j.dataset, j.state.SeedIdentitySHA, j.state.BaselineStateSHA, j.plan); err != nil {
|
||||
return err
|
||||
}
|
||||
for account, messageID := range j.state.PrivateMessageIDs {
|
||||
if messageID <= 0 {
|
||||
return fmt.Errorf("offline private mutation account %d is incomplete", account)
|
||||
}
|
||||
}
|
||||
for i, channel := range j.state.Channels {
|
||||
for message, messageID := range channel.MessageIDs {
|
||||
if messageID <= 0 {
|
||||
return fmt.Errorf("offline channel %d message %d is incomplete", channel.GroupIndex, message)
|
||||
}
|
||||
}
|
||||
if channel.LatestPts <= 0 {
|
||||
return fmt.Errorf("offline channel %d has no observed pts", channel.GroupIndex)
|
||||
}
|
||||
if i == 0 && (!channel.EditDone || !channel.DeleteDone || !channel.PinDone) {
|
||||
return fmt.Errorf("offline channel %d actions are incomplete", channel.GroupIndex)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func offlineMutationResult(plan []OfflineMutationChannelPlan, state *OfflineMutationState) *MutationResult {
|
||||
result := &MutationResult{PrivateMessages: len(state.PrivateMessageIDs), DirtyChannels: len(plan)}
|
||||
for _, channel := range state.Channels {
|
||||
result.ChannelMessages += len(channel.MessageIDs)
|
||||
if channel.EditDone {
|
||||
result.Edited++
|
||||
}
|
||||
if channel.DeleteDone {
|
||||
result.Deleted++
|
||||
}
|
||||
if channel.PinDone {
|
||||
result.Pinned++
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
123
internal/loadharness/mutate_test.go
Normal file
123
internal/loadharness/mutate_test.go
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/iamxvbaba/td/tg"
|
||||
)
|
||||
|
||||
func TestPlanOfflineMutationDefaultTopology(t *testing.T) {
|
||||
dataset, err := PlanDataset(DefaultDatasetConfig(1000))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan := planOfflineMutation(dataset)
|
||||
if got, want := len(plan), 40; got != want {
|
||||
t.Fatalf("dirty channels = %d, want %d", got, want)
|
||||
}
|
||||
total := 0
|
||||
for _, channel := range plan {
|
||||
total += channel.Messages
|
||||
}
|
||||
if got, want := total, 304; got != want {
|
||||
t.Fatalf("channel mutations = %d, want %d", got, want)
|
||||
}
|
||||
if plan[0].Messages != 120 || dataset.Groups[plan[0].GroupPosition].Tier != "hot" {
|
||||
t.Fatalf("multi-page channel plan = %+v", plan[0])
|
||||
}
|
||||
if plan[1].Messages != 100 {
|
||||
t.Fatalf("full-page boundary channel plan = %+v", plan[1])
|
||||
}
|
||||
group := dataset.Groups[plan[0].GroupPosition]
|
||||
for message := 0; message < 3; message++ {
|
||||
if got := offlineMutationSender(group, message); got != group.CreatorAccount {
|
||||
t.Fatalf("action message %d sender = %d, want creator %d", message, got, group.CreatorAccount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSentMessageObservation(t *testing.T) {
|
||||
peer := clientPeerKey{typ: "channel", id: 22}
|
||||
marker := "marker"
|
||||
observation, err := sentMessageObservation(&tg.Updates{Updates: []tg.UpdateClass{
|
||||
&tg.UpdateNewChannelMessage{
|
||||
Message: &tg.Message{ID: 7, PeerID: &tg.PeerChannel{ChannelID: 22}, Message: marker},
|
||||
Pts: 12, PtsCount: 1,
|
||||
},
|
||||
}}, peer, marker)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if observation.ID != 7 || observation.Pts != 12 {
|
||||
t.Fatalf("observation = %+v", observation)
|
||||
}
|
||||
short, err := sentMessageObservation(&tg.UpdateShortSentMessage{ID: 8, Pts: 13}, clientPeerKey{typ: "user", id: 1}, "private")
|
||||
if err != nil || short.ID != 8 || short.Pts != 13 {
|
||||
t.Fatalf("short observation = %+v err=%v", short, err)
|
||||
}
|
||||
if _, err := sentMessageObservation(&tg.Updates{}, peer, marker); err == nil {
|
||||
t.Fatal("updates without marker passed validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflineMutationJournalPersistsMessagesAndActions(t *testing.T) {
|
||||
dataset, seedState, _ := snapshotFixture(t)
|
||||
seedIdentity, err := seedIdentitySHA256(seedState)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan := planOfflineMutation(dataset)
|
||||
path := filepath.Join(t.TempDir(), "mutation-state.json")
|
||||
state, err := loadOrCreateOfflineMutationState(path, dataset, seedIdentity, "baseline", plan)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
journal := &mutationJournal{path: path, dataset: dataset, plan: plan, state: state}
|
||||
if err := journal.persist(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := journal.commitPrivate(0, 10, 20); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := journal.commitChannelMessage(0, 0, 30, 40); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := journal.beginAction(0, "edit"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := journal.commitAction(0, "edit", 41); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := loadOrCreateOfflineMutationState(path, dataset, seedIdentity, "baseline", plan)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loaded.PrivateMessageIDs[0] != 10 || loaded.AccountObservedPts[0] != 20 || loaded.Channels[0].MessageIDs[0] != 30 || loaded.Channels[0].LatestPts != 41 || !loaded.Channels[0].EditDone || loaded.Channels[0].EditPending {
|
||||
t.Fatalf("persisted mutation state = %+v", loaded)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("mutation state mode = %o, want 600", info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflineMutationStateRejectsDifferentBaseline(t *testing.T) {
|
||||
dataset, seedState, _ := snapshotFixture(t)
|
||||
seedIdentity, err := seedIdentitySHA256(seedState)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan := planOfflineMutation(dataset)
|
||||
state, err := loadOrCreateOfflineMutationState(filepath.Join(t.TempDir(), "missing.json"), dataset, seedIdentity, "baseline-a", plan)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := state.Validate(dataset, seedIdentity, "baseline-b", plan); err == nil {
|
||||
t.Fatal("mutation state accepted different baseline snapshot")
|
||||
}
|
||||
}
|
||||
|
|
@ -29,6 +29,10 @@ type ProvisionConfig struct {
|
|||
FirstNamePrefix string
|
||||
}
|
||||
|
||||
// DefaultPhonePrefix plus the six-digit account index produces the repository's
|
||||
// structurally possible reserved NANP range (for example +1 555 000 0001).
|
||||
const DefaultPhonePrefix = "+15550"
|
||||
|
||||
type ProvisionEvent struct {
|
||||
Completed int
|
||||
Total int
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ package loadharness
|
|||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
func TestSessionDirectoryForManifestIsolatesNamedBundles(t *testing.T) {
|
||||
|
|
@ -23,6 +25,14 @@ func TestSessionDirectoryForManifestIsolatesNamedBundles(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestDefaultProvisionPhonePrefixProducesCanonicalPossibleNumber(t *testing.T) {
|
||||
cfg := ProvisionConfig{PhonePrefix: DefaultPhonePrefix, FirstNamePrefix: "Load"}
|
||||
record := desiredSessionRecord(0, 0, 0, cfg)
|
||||
if got, want := domain.NormalizePhone(record.Phone), "15550000001"; got != want {
|
||||
t.Fatalf("normalized default phone = %q, want %q (wire %q)", got, want, record.Phone)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDesiredSessionRecordUsesManifestNamespace(t *testing.T) {
|
||||
cfg := ProvisionConfig{ManifestPath: filepath.Join("data", "manifest-500.json"), PhonePrefix: "+155500", FirstNamePrefix: "Load"}
|
||||
record := desiredSessionRecord(12, 12, 1, cfg)
|
||||
|
|
|
|||
|
|
@ -111,8 +111,9 @@ func durationMS(d time.Duration) float64 {
|
|||
}
|
||||
|
||||
type metricSet struct {
|
||||
mu sync.RWMutex
|
||||
ops map[string]*operationMetrics
|
||||
mu sync.RWMutex
|
||||
ops map[string]*operationMetrics
|
||||
frozen bool
|
||||
}
|
||||
|
||||
func newMetricSet(names ...string) *metricSet {
|
||||
|
|
@ -124,29 +125,56 @@ func newMetricSet(names ...string) *metricSet {
|
|||
}
|
||||
|
||||
func (m *metricSet) observe(name string, start time.Time, err error) {
|
||||
debugOperationError(name, err)
|
||||
m.mu.RLock()
|
||||
if m.frozen {
|
||||
m.mu.RUnlock()
|
||||
return
|
||||
}
|
||||
op := m.ops[name]
|
||||
if op != nil {
|
||||
debugOperationError(name, err)
|
||||
op.observe(start, err)
|
||||
m.mu.RUnlock()
|
||||
return
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
if op == nil {
|
||||
{
|
||||
// Operation names are code-owned and finite, but retain a lock-protected
|
||||
// fallback for optional scenarios added by the harness.
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.frozen {
|
||||
return
|
||||
}
|
||||
op = m.ops[name]
|
||||
if op == nil && len(m.ops) < 32 {
|
||||
op = &operationMetrics{}
|
||||
m.ops[name] = op
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
if op != nil {
|
||||
op.observe(start, err)
|
||||
if op != nil {
|
||||
debugOperationError(name, err)
|
||||
op.observe(start, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *metricSet) report() map[string]OperationReport {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.reportLocked()
|
||||
}
|
||||
|
||||
// freeze returns the immutable pre-teardown operation cut. Holding the write
|
||||
// lock waits for any observer already publishing its complete metric tuple and
|
||||
// prevents later coordinated-cancel outcomes from entering the business report.
|
||||
func (m *metricSet) freeze() map[string]OperationReport {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.frozen = true
|
||||
return m.reportLocked()
|
||||
}
|
||||
|
||||
func (m *metricSet) reportLocked() map[string]OperationReport {
|
||||
out := make(map[string]OperationReport, len(m.ops))
|
||||
for name, op := range m.ops {
|
||||
out[name] = op.report()
|
||||
|
|
@ -155,31 +183,46 @@ func (m *metricSet) report() map[string]OperationReport {
|
|||
}
|
||||
|
||||
type RunReport struct {
|
||||
Version int `json:"version"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
LoadEndedAt time.Time `json:"load_ended_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
RequestedDuration string `json:"requested_duration"`
|
||||
RecoveryDuration string `json:"recovery_duration"`
|
||||
ExpectedSessions int `json:"expected_sessions"`
|
||||
PeakReadySessions int `json:"peak_ready_sessions"`
|
||||
FinalReadySessions int `json:"final_ready_sessions"`
|
||||
SteadySamples int `json:"steady_samples"`
|
||||
SteadyReadyRatio float64 `json:"steady_ready_ratio"`
|
||||
MinSteadyReadySessions int `json:"min_steady_ready_sessions"`
|
||||
ConnectionAttempts uint64 `json:"connection_attempts"`
|
||||
Reconnects uint64 `json:"reconnects"`
|
||||
Disconnects uint64 `json:"disconnects"`
|
||||
UpdatesReceived uint64 `json:"updates_received"`
|
||||
DownloadedBytes uint64 `json:"downloaded_bytes"`
|
||||
WorkerFatalErrors uint64 `json:"worker_fatal_errors"`
|
||||
Operations map[string]OperationReport `json:"operations"`
|
||||
BaselineServerMetrics map[string]float64 `json:"baseline_server_metrics,omitempty"`
|
||||
FinalServerMetrics map[string]float64 `json:"final_server_metrics,omitempty"`
|
||||
ServerMetricsScrapes uint64 `json:"server_metrics_scrapes"`
|
||||
ServerMetricsErrors uint64 `json:"server_metrics_errors"`
|
||||
Pass bool `json:"pass"`
|
||||
Failures []string `json:"failures,omitempty"`
|
||||
Version int `json:"version"`
|
||||
StartOrder string `json:"start_order,omitempty"`
|
||||
StartOrderSeed int64 `json:"start_order_seed,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
LoadEndedAt time.Time `json:"load_ended_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
RequestedDuration string `json:"requested_duration"`
|
||||
RecoveryDuration string `json:"recovery_duration"`
|
||||
ExpectedSessions int `json:"expected_sessions"`
|
||||
PeakReadySessions int `json:"peak_ready_sessions"`
|
||||
FinalReadySessions int `json:"final_ready_sessions"`
|
||||
SteadySamples int `json:"steady_samples"`
|
||||
SteadyReadyRatio float64 `json:"steady_ready_ratio"`
|
||||
MinSteadyReadySessions int `json:"min_steady_ready_sessions"`
|
||||
ConnectionAttempts uint64 `json:"connection_attempts"`
|
||||
Reconnects uint64 `json:"reconnects"`
|
||||
Disconnects uint64 `json:"disconnects"`
|
||||
UpdatesReceived uint64 `json:"updates_received"`
|
||||
DownloadedBytes uint64 `json:"downloaded_bytes"`
|
||||
WorkerFatalErrors uint64 `json:"worker_fatal_errors"`
|
||||
MessageRatePerSecond float64 `json:"message_rate_per_second"`
|
||||
MessageScheduled uint64 `json:"message_scheduled"`
|
||||
MessageEnqueued uint64 `json:"message_enqueued"`
|
||||
MessageCompleted uint64 `json:"message_completed"`
|
||||
MessageQueueFull uint64 `json:"message_queue_full"`
|
||||
MessageNotReady uint64 `json:"message_not_ready"`
|
||||
Delivery DeliveryReport `json:"delivery"`
|
||||
Operations map[string]OperationReport `json:"operations"`
|
||||
ResponseBytes map[string]StartupResponseBytes `json:"response_bytes,omitempty"`
|
||||
RPCDeliveryOutcomes map[string]map[string]uint64 `json:"rpc_delivery_outcomes,omitempty"`
|
||||
DatabaseWork map[string]StartupDatabaseWork `json:"database_work,omitempty"`
|
||||
BaselineServerMetrics map[string]float64 `json:"baseline_server_metrics,omitempty"`
|
||||
WorkloadEndServerMetrics map[string]float64 `json:"workload_end_server_metrics,omitempty"`
|
||||
FinalServerMetrics map[string]float64 `json:"final_server_metrics,omitempty"`
|
||||
ServerMetricsScrapes uint64 `json:"server_metrics_scrapes"`
|
||||
ServerMetricsErrors uint64 `json:"server_metrics_errors"`
|
||||
EventsWritten uint64 `json:"events_written"`
|
||||
EventsDropped uint64 `json:"events_dropped"`
|
||||
Pass bool `json:"pass"`
|
||||
Failures []string `json:"failures,omitempty"`
|
||||
}
|
||||
|
||||
func WriteReport(path string, report *RunReport) error {
|
||||
|
|
@ -191,12 +234,18 @@ func WriteReport(path string, report *RunReport) error {
|
|||
}
|
||||
|
||||
type eventWriter struct {
|
||||
mu sync.Mutex
|
||||
f *os.File
|
||||
written uint64
|
||||
dropped uint64
|
||||
mu sync.Mutex
|
||||
f *os.File
|
||||
written uint64
|
||||
dropped uint64
|
||||
connectionDeadWritten uint64
|
||||
}
|
||||
|
||||
const (
|
||||
maxEventLines = 100000
|
||||
maxConnectionDeadEventLines = 5000
|
||||
)
|
||||
|
||||
func newEventWriter(path string) (*eventWriter, error) {
|
||||
if path == "" {
|
||||
return &eventWriter{}, nil
|
||||
|
|
@ -220,7 +269,15 @@ func (w *eventWriter) write(value any) {
|
|||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
if w.written >= 10000 {
|
||||
if event, ok := value.(map[string]any); ok && event["type"] == "connection_dead" {
|
||||
if w.connectionDeadWritten >= maxConnectionDeadEventLines {
|
||||
w.dropped++
|
||||
w.mu.Unlock()
|
||||
return
|
||||
}
|
||||
w.connectionDeadWritten++
|
||||
}
|
||||
if w.written >= maxEventLines {
|
||||
w.dropped++
|
||||
w.mu.Unlock()
|
||||
return
|
||||
|
|
@ -230,6 +287,16 @@ func (w *eventWriter) write(value any) {
|
|||
w.mu.Unlock()
|
||||
}
|
||||
|
||||
func (w *eventWriter) counts() (written, dropped uint64) {
|
||||
if w == nil {
|
||||
return 0, 0
|
||||
}
|
||||
w.mu.Lock()
|
||||
written, dropped = w.written, w.dropped
|
||||
w.mu.Unlock()
|
||||
return written, dropped
|
||||
}
|
||||
|
||||
func (w *eventWriter) close() error {
|
||||
if w == nil || w.f == nil {
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
|
@ -12,6 +14,43 @@ import (
|
|||
tdrpc "github.com/iamxvbaba/td/rpc"
|
||||
)
|
||||
|
||||
func TestMarkClientReadyAvoidsDuplicateInitialCatchUp(t *testing.T) {
|
||||
var everReady atomic.Bool
|
||||
var firstClient atomic.Bool
|
||||
if markClientReady(&everReady, &firstClient) {
|
||||
t.Fatal("first Ready transition requested a catch-up")
|
||||
}
|
||||
if !markClientReady(&everReady, &firstClient) {
|
||||
t.Fatal("transport reconnect did not request a catch-up")
|
||||
}
|
||||
|
||||
var replacementClient atomic.Bool
|
||||
if markClientReady(&everReady, &replacementClient) {
|
||||
t.Fatal("replacement Client duplicated its callback-owned initial catch-up")
|
||||
}
|
||||
if !markClientReady(&everReady, &replacementClient) {
|
||||
t.Fatal("replacement Client transport reconnect did not request a catch-up")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventWriterCapsConnectionDetailsWithoutDroppingSamples(t *testing.T) {
|
||||
w, err := newEventWriter(filepath.Join(t.TempDir(), "events.ndjson"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < maxConnectionDeadEventLines+1; i++ {
|
||||
w.write(map[string]any{"type": "connection_dead", "class": "connection"})
|
||||
}
|
||||
w.write(map[string]any{"type": "sample", "ready": 0})
|
||||
written, dropped := w.counts()
|
||||
if written != maxConnectionDeadEventLines+1 || dropped != 1 {
|
||||
t.Fatalf("event counts = %d/%d", written, dropped)
|
||||
}
|
||||
if err := w.close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOperationMetricsUsesBoundedHistogramAndFixedErrorClasses(t *testing.T) {
|
||||
metrics := &operationMetrics{}
|
||||
metrics.observe(time.Now().Add(-20*time.Millisecond), nil)
|
||||
|
|
@ -33,6 +72,7 @@ func TestClassifyErrorReasonUsesFiniteRedactedVocabulary(t *testing.T) {
|
|||
{errors.New("dial tcp 10.0.0.1:2398: socket: too many open files"), "file_descriptor_limit"},
|
||||
{errors.New("read: temporary auth key not found: pfs reconnect required"), "pfs_reconnect"},
|
||||
{errors.New("read tcp: EOF auth_key_id=secret"), "eof"},
|
||||
{errors.New("startup session ended before business readiness"), "business_readiness_incomplete"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
if got := classifyErrorReason(test.err); got != test.want {
|
||||
|
|
@ -64,6 +104,34 @@ func TestOperationMetricsSeparatesHarnessCancellation(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestMetricSetFreezeExcludesCoordinatedTeardown(t *testing.T) {
|
||||
metrics := newMetricSet("ping")
|
||||
metrics.observe("ping", time.Now(), nil)
|
||||
cut := metrics.freeze()
|
||||
metrics.observe("ping", time.Now(), fmt.Errorf("engine forcibly closed: %w", context.Canceled))
|
||||
|
||||
if got := cut["ping"]; got.Count != 1 || got.Errors != 0 || got.Canceled != 0 {
|
||||
t.Fatalf("workload cut = %#v", got)
|
||||
}
|
||||
if got := metrics.report()["ping"]; got != cut["ping"] {
|
||||
t.Fatalf("post-freeze metrics changed: got %#v want %#v", got, cut["ping"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateReportRejectsFixedRateDeliveryLoss(t *testing.T) {
|
||||
report := &RunReport{
|
||||
ExpectedSessions: 1, PeakReadySessions: 1,
|
||||
SteadySamples: 1, SteadyReadyRatio: 1, MinSteadyReadySessions: 1,
|
||||
MessageScheduled: 2, MessageEnqueued: 2, MessageCompleted: 2,
|
||||
Delivery: DeliveryReport{Expected: 2, Delivered: 1, Missing: 1},
|
||||
Operations: map[string]OperationReport{},
|
||||
}
|
||||
evaluateReport(report, RunConfig{MinimumReadyRatio: 1, MessageRate: 1})
|
||||
if report.Pass {
|
||||
t.Fatal("fixed-rate report with a missing recipient delivery passed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateReportAllowsOnlyConnectionErrorsForExpectedRestart(t *testing.T) {
|
||||
report := &RunReport{
|
||||
ExpectedSessions: 2, PeakReadySessions: 2, Reconnects: 2,
|
||||
|
|
@ -84,6 +152,24 @@ func TestEvaluateReportAllowsOnlyConnectionErrorsForExpectedRestart(t *testing.T
|
|||
}
|
||||
}
|
||||
|
||||
func TestEvaluateReportRejectsServerDeliveryAndDatabaseErrors(t *testing.T) {
|
||||
report := &RunReport{
|
||||
ExpectedSessions: 1, PeakReadySessions: 1,
|
||||
SteadySamples: 1, SteadyReadyRatio: 1, MinSteadyReadySessions: 1,
|
||||
Operations: map[string]OperationReport{},
|
||||
RPCDeliveryOutcomes: map[string]map[string]uint64{
|
||||
"updates.getState": {"ok": 1, "edge_overload": 2},
|
||||
},
|
||||
DatabaseWork: map[string]StartupDatabaseWork{
|
||||
"messages.getDialogs": {Errors: 3},
|
||||
},
|
||||
}
|
||||
evaluateReport(report, RunConfig{MinimumReadyRatio: 1})
|
||||
if report.Pass || len(report.Failures) != 2 {
|
||||
t.Fatalf("report = %#v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateReportRequiresReclamationAndNoFloodWait(t *testing.T) {
|
||||
report := &RunReport{
|
||||
ExpectedSessions: 10, PeakReadySessions: 10, ServerMetricsScrapes: 1,
|
||||
|
|
@ -97,6 +183,7 @@ func TestEvaluateReportRequiresReclamationAndNoFloodWait(t *testing.T) {
|
|||
"telesrv_mtproto_raw_connections": 2,
|
||||
"telesrv_mtproto_logical_outbox_bytes": 4,
|
||||
},
|
||||
WorkloadEndServerMetrics: map[string]float64{},
|
||||
}
|
||||
evaluateReport(report, RunConfig{MinimumReadyRatio: 1, RecoveryDuration: time.Minute, ServerMetricsURL: "http://metrics"})
|
||||
if report.Pass || len(report.Failures) != 1 {
|
||||
|
|
@ -119,6 +206,7 @@ func TestEvaluateReportAcceptsReturnToNonZeroSharedServerBaseline(t *testing.T)
|
|||
"telesrv_mtproto_logical_sessions": 2,
|
||||
"telesrv_mtproto_logical_outbox_bytes": 1024,
|
||||
},
|
||||
WorkloadEndServerMetrics: map[string]float64{},
|
||||
}
|
||||
evaluateReport(report, RunConfig{MinimumReadyRatio: 1, RecoveryDuration: time.Minute, ServerMetricsURL: "http://metrics"})
|
||||
if !report.Pass || len(report.Failures) != 0 {
|
||||
|
|
|
|||
232
internal/loadharness/rich_state.go
Normal file
232
internal/loadharness/rich_state.go
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rsa"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/iamxvbaba/td/tg"
|
||||
)
|
||||
|
||||
type richAccountStatePlan struct {
|
||||
PinnedPeerAccount int
|
||||
ReadPeerAccount int
|
||||
ReadGroupPosition int
|
||||
DraftMarker string
|
||||
}
|
||||
|
||||
func planRichAccountState(dataset *Dataset, account int) (richAccountStatePlan, error) {
|
||||
if dataset == nil || account < 0 || account >= dataset.Config.Accounts {
|
||||
return richAccountStatePlan{}, errors.New("invalid rich-state account")
|
||||
}
|
||||
if dataset.Config.PrivateFanout < 1 {
|
||||
return richAccountStatePlan{}, errors.New("rich startup dataset requires at least one private peer per account")
|
||||
}
|
||||
groupPosition := -1
|
||||
for position, group := range dataset.Groups {
|
||||
if group.HistoryMessages > 0 && datasetGroupHasAccount(group, account) {
|
||||
groupPosition = position
|
||||
break
|
||||
}
|
||||
}
|
||||
if groupPosition < 0 {
|
||||
return richAccountStatePlan{}, errors.New("rich startup dataset requires a non-empty supergroup per account")
|
||||
}
|
||||
return richAccountStatePlan{
|
||||
PinnedPeerAccount: (account + 1) % dataset.Config.Accounts,
|
||||
ReadPeerAccount: (account - 1 + dataset.Config.Accounts) % dataset.Config.Accounts,
|
||||
ReadGroupPosition: groupPosition,
|
||||
DraftMarker: fmt.Sprintf("[%s draft account %04d]", dataset.RunID, account),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// seedRichAccountState creates synchronized dialog state exclusively through
|
||||
// public MTProto RPCs. Every mutation is absolute and then read back through
|
||||
// messages.getPeerDialogs before the resumable journal is committed.
|
||||
func seedRichAccountState(
|
||||
ctx context.Context,
|
||||
cfg SeedConfig,
|
||||
manifest *Manifest,
|
||||
dataset *Dataset,
|
||||
journal *seedJournal,
|
||||
targets []SessionRecord,
|
||||
key [32]byte,
|
||||
publicKey *rsa.PublicKey,
|
||||
account int,
|
||||
) error {
|
||||
if journal.richStateComplete(account) {
|
||||
return nil
|
||||
}
|
||||
plan, err := planRichAccountState(dataset, account)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return withAuthorizedSeedSession(ctx, cfg, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
|
||||
pinnedTarget := targets[plan.PinnedPeerAccount]
|
||||
readTarget := targets[plan.ReadPeerAccount]
|
||||
pinnedPeer := &tg.InputPeerUser{UserID: pinnedTarget.UserID, AccessHash: pinnedTarget.AccessHash}
|
||||
readPeer := &tg.InputPeerUser{UserID: readTarget.UserID, AccessHash: readTarget.AccessHash}
|
||||
groupIdentity := journal.group(plan.ReadGroupPosition)
|
||||
if groupIdentity.ChannelID <= 0 || groupIdentity.AccessHash == 0 {
|
||||
return errors.New("rich-state group identity is incomplete")
|
||||
}
|
||||
channelPeer := &tg.InputPeerChannel{ChannelID: groupIdentity.ChannelID, AccessHash: groupIdentity.AccessHash}
|
||||
|
||||
before, err := getRichStateDialogs(ctx, cfg.OperationTimeout, raw, pinnedPeer, readPeer, channelPeer)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read rich-state cursors: %w", err)
|
||||
}
|
||||
readPrivate, ok := before[clientPeerKey{typ: "user", id: readTarget.UserID}]
|
||||
if !ok || readPrivate.TopMessage <= 0 {
|
||||
return errors.New("read private peer omitted its top message")
|
||||
}
|
||||
readChannel, ok := before[clientPeerKey{typ: "channel", id: groupIdentity.ChannelID}]
|
||||
if !ok || readChannel.TopMessage <= 0 {
|
||||
return errors.New("read channel omitted its top message")
|
||||
}
|
||||
|
||||
pinned, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (bool, error) {
|
||||
return raw.MessagesToggleDialogPin(rpcCtx, &tg.MessagesToggleDialogPinRequest{
|
||||
Pinned: true, Peer: &tg.InputDialogPeer{Peer: pinnedPeer},
|
||||
})
|
||||
})
|
||||
if err != nil || !pinned {
|
||||
return rpcBooleanError("messages.toggleDialogPin", pinned, err)
|
||||
}
|
||||
saved, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (bool, error) {
|
||||
return raw.MessagesSaveDraft(rpcCtx, &tg.MessagesSaveDraftRequest{Peer: pinnedPeer, Message: plan.DraftMarker})
|
||||
})
|
||||
if err != nil || !saved {
|
||||
return rpcBooleanError("messages.saveDraft", saved, err)
|
||||
}
|
||||
if _, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (*tg.MessagesAffectedMessages, error) {
|
||||
return raw.MessagesReadHistory(rpcCtx, &tg.MessagesReadHistoryRequest{Peer: readPeer, MaxID: readPrivate.TopMessage})
|
||||
}); err != nil {
|
||||
return fmt.Errorf("messages.readHistory: %w", err)
|
||||
}
|
||||
channelRead, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (bool, error) {
|
||||
return raw.ChannelsReadHistory(rpcCtx, &tg.ChannelsReadHistoryRequest{
|
||||
Channel: &tg.InputChannel{ChannelID: groupIdentity.ChannelID, AccessHash: groupIdentity.AccessHash},
|
||||
MaxID: readChannel.TopMessage,
|
||||
})
|
||||
})
|
||||
if err != nil || !channelRead {
|
||||
return rpcBooleanError("channels.readHistory", channelRead, err)
|
||||
}
|
||||
|
||||
after, err := getRichStateDialogs(ctx, cfg.OperationTimeout, raw, pinnedPeer, readPeer, channelPeer)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verify rich-state dialogs: %w", err)
|
||||
}
|
||||
pinnedDialog, ok := after[clientPeerKey{typ: "user", id: pinnedTarget.UserID}]
|
||||
if !ok || !pinnedDialog.Pinned || !pinnedDialog.HasDraft || pinnedDialog.DraftText != plan.DraftMarker {
|
||||
return errors.New("pinned private dialog or exact draft was not persisted")
|
||||
}
|
||||
readPrivate = after[clientPeerKey{typ: "user", id: readTarget.UserID}]
|
||||
if readPrivate.ReadInboxMaxID < before[clientPeerKey{typ: "user", id: readTarget.UserID}].TopMessage || readPrivate.UnreadCount != 0 {
|
||||
return errors.New("private read boundary did not converge")
|
||||
}
|
||||
readChannel = after[clientPeerKey{typ: "channel", id: groupIdentity.ChannelID}]
|
||||
if readChannel.ReadInboxMaxID < before[clientPeerKey{typ: "channel", id: groupIdentity.ChannelID}].TopMessage || readChannel.UnreadCount != 0 {
|
||||
return errors.New("channel read boundary did not converge")
|
||||
}
|
||||
return journal.setRichStateComplete(account)
|
||||
})
|
||||
}
|
||||
|
||||
func rpcBooleanError(operation string, result bool, err error) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s result=%v: %w", operation, result, err)
|
||||
}
|
||||
return fmt.Errorf("%s returned false", operation)
|
||||
}
|
||||
|
||||
func getRichStateDialogs(
|
||||
ctx context.Context,
|
||||
timeout time.Duration,
|
||||
raw *tg.Client,
|
||||
peers ...tg.InputPeerClass,
|
||||
) (map[clientPeerKey]ClientDialogState, error) {
|
||||
requests := make([]tg.InputDialogPeerClass, 0, len(peers))
|
||||
seen := make(map[clientPeerKey]struct{}, len(peers))
|
||||
for _, peer := range peers {
|
||||
key, ok := clientPeerFromInput(peer)
|
||||
if !ok {
|
||||
return nil, errors.New("invalid rich-state input peer")
|
||||
}
|
||||
if _, exists := seen[key]; exists {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
requests = append(requests, &tg.InputDialogPeer{Peer: peer})
|
||||
}
|
||||
response, err := rpcWithFloodWaitRetry(ctx, timeout, func(rpcCtx context.Context) (*tg.MessagesPeerDialogs, error) {
|
||||
return raw.MessagesGetPeerDialogs(rpcCtx, requests)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dialogs := make(map[clientPeerKey]ClientDialogState, len(response.Dialogs))
|
||||
if _, err := mergeDialogPage(dialogs, response.Dialogs, response.Messages, response.Chats, response.Users, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(dialogs) != len(requests) {
|
||||
return nil, fmt.Errorf("messages.getPeerDialogs returned %d/%d dialogs", len(dialogs), len(requests))
|
||||
}
|
||||
return dialogs, nil
|
||||
}
|
||||
|
||||
func clientPeerFromInput(peer tg.InputPeerClass) (clientPeerKey, bool) {
|
||||
switch value := peer.(type) {
|
||||
case *tg.InputPeerUser:
|
||||
return clientPeerKey{typ: "user", id: value.UserID}, value.UserID > 0 && value.AccessHash != 0
|
||||
case *tg.InputPeerChannel:
|
||||
return clientPeerKey{typ: "channel", id: value.ChannelID}, value.ChannelID > 0 && value.AccessHash != 0
|
||||
default:
|
||||
return clientPeerKey{}, false
|
||||
}
|
||||
}
|
||||
|
||||
func validateSeededRichDialogs(
|
||||
dataset *Dataset,
|
||||
seedState *DatasetSeedState,
|
||||
targets []SessionRecord,
|
||||
account int,
|
||||
dialogs []ClientDialogState,
|
||||
requireReadBoundaries bool,
|
||||
) error {
|
||||
if len(seedState.RichStateByAccount) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(seedState.RichStateByAccount) != dataset.Config.Accounts || !seedState.RichStateByAccount[account] {
|
||||
return errors.New("rich-state seed is incomplete")
|
||||
}
|
||||
plan, err := planRichAccountState(dataset, account)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
byPeer := make(map[clientPeerKey]ClientDialogState, len(dialogs))
|
||||
for _, dialog := range dialogs {
|
||||
byPeer[clientPeerKey{typ: dialog.PeerType, id: dialog.PeerID}] = dialog
|
||||
}
|
||||
pinned := byPeer[clientPeerKey{typ: "user", id: targets[plan.PinnedPeerAccount].UserID}]
|
||||
if !pinned.Pinned || !pinned.HasDraft || pinned.DraftText != plan.DraftMarker {
|
||||
return fmt.Errorf("seeded pinned dialog state mismatch: present=%v pinned=%v has_draft=%v draft_matches=%v got_draft=%q want_draft=%q",
|
||||
pinned.PeerID != 0, pinned.Pinned, pinned.HasDraft, pinned.DraftText == plan.DraftMarker, pinned.DraftText, plan.DraftMarker)
|
||||
}
|
||||
if !requireReadBoundaries {
|
||||
return nil
|
||||
}
|
||||
readPrivate := byPeer[clientPeerKey{typ: "user", id: targets[plan.ReadPeerAccount].UserID}]
|
||||
if readPrivate.TopMessage <= 0 || readPrivate.ReadInboxMaxID < readPrivate.TopMessage || readPrivate.UnreadCount != 0 {
|
||||
return errors.New("seeded private read boundary is stale")
|
||||
}
|
||||
group := seedState.Groups[plan.ReadGroupPosition]
|
||||
readChannel := byPeer[clientPeerKey{typ: "channel", id: group.ChannelID}]
|
||||
if readChannel.TopMessage <= 0 || readChannel.ReadInboxMaxID < readChannel.TopMessage || readChannel.UnreadCount != 0 {
|
||||
return errors.New("seeded channel read boundary is stale")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
89
internal/loadharness/rpc_retry.go
Normal file
89
internal/loadharness/rpc_retry.go
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/iamxvbaba/td/tgerr"
|
||||
)
|
||||
|
||||
const (
|
||||
maxDatasetFloodWaitRetries = 16
|
||||
maxDatasetFloodWait = 2 * time.Minute
|
||||
datasetFloodWaitPadding = time.Second
|
||||
)
|
||||
|
||||
type floodWaitPolicy struct {
|
||||
maxRetries int
|
||||
maxWait time.Duration
|
||||
padding time.Duration
|
||||
wait func(context.Context, time.Duration) error
|
||||
}
|
||||
|
||||
func defaultFloodWaitPolicy() floodWaitPolicy {
|
||||
return floodWaitPolicy{
|
||||
maxRetries: maxDatasetFloodWaitRetries,
|
||||
maxWait: maxDatasetFloodWait,
|
||||
padding: datasetFloodWaitPadding,
|
||||
wait: waitForContext,
|
||||
}
|
||||
}
|
||||
|
||||
// rpcWithFloodWaitRetry is reserved for dataset preparation. Startup-run must
|
||||
// observe FLOOD_WAIT as a measured failure instead of hiding it behind a retry.
|
||||
func rpcWithFloodWaitRetry[T any](
|
||||
ctx context.Context,
|
||||
timeout time.Duration,
|
||||
call func(context.Context) (T, error),
|
||||
) (T, error) {
|
||||
return rpcWithFloodWaitPolicy(ctx, timeout, defaultFloodWaitPolicy(), call)
|
||||
}
|
||||
|
||||
func rpcWithFloodWaitPolicy[T any](
|
||||
ctx context.Context,
|
||||
timeout time.Duration,
|
||||
policy floodWaitPolicy,
|
||||
call func(context.Context) (T, error),
|
||||
) (T, error) {
|
||||
var zero T
|
||||
if timeout <= 0 {
|
||||
return zero, fmt.Errorf("RPC timeout must be positive")
|
||||
}
|
||||
if policy.maxRetries < 0 || policy.maxWait < 0 || policy.padding < 0 || policy.wait == nil {
|
||||
return zero, fmt.Errorf("invalid FLOOD_WAIT retry policy")
|
||||
}
|
||||
for retry := 0; ; retry++ {
|
||||
rpcCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
result, err := call(rpcCtx)
|
||||
cancel()
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
wait, ok := tgerr.AsFloodWait(err)
|
||||
if !ok {
|
||||
return zero, err
|
||||
}
|
||||
if retry >= policy.maxRetries {
|
||||
return zero, fmt.Errorf("FLOOD_WAIT retry limit %d exhausted: %w", policy.maxRetries, err)
|
||||
}
|
||||
wait += policy.padding
|
||||
if wait > policy.maxWait {
|
||||
return zero, fmt.Errorf("FLOOD_WAIT %s exceeds dataset preparation limit %s: %w", wait, policy.maxWait, err)
|
||||
}
|
||||
if err := policy.wait(ctx, wait); err != nil {
|
||||
return zero, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func waitForContext(ctx context.Context, delay time.Duration) error {
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
63
internal/loadharness/rpc_retry_test.go
Normal file
63
internal/loadharness/rpc_retry_test.go
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/iamxvbaba/td/tgerr"
|
||||
)
|
||||
|
||||
func TestRPCWithFloodWaitPolicyRetriesTheSameCall(t *testing.T) {
|
||||
attempts := 0
|
||||
waits := make([]time.Duration, 0, 1)
|
||||
policy := floodWaitPolicy{
|
||||
maxRetries: 2,
|
||||
maxWait: 5 * time.Second,
|
||||
padding: 100 * time.Millisecond,
|
||||
wait: func(_ context.Context, delay time.Duration) error {
|
||||
waits = append(waits, delay)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
result, err := rpcWithFloodWaitPolicy(context.Background(), time.Second, policy, func(context.Context) (int, error) {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
return 0, tgerr.New(420, "FLOOD_WAIT_2")
|
||||
}
|
||||
return 42, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result != 42 || attempts != 2 {
|
||||
t.Fatalf("result=%d attempts=%d", result, attempts)
|
||||
}
|
||||
if len(waits) != 1 || waits[0] != 2100*time.Millisecond {
|
||||
t.Fatalf("waits=%v", waits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCWithFloodWaitPolicyDoesNotRetryOrdinaryError(t *testing.T) {
|
||||
want := errors.New("boom")
|
||||
attempts := 0
|
||||
policy := floodWaitPolicy{maxRetries: 2, maxWait: time.Second, wait: waitForContext}
|
||||
_, err := rpcWithFloodWaitPolicy(context.Background(), time.Second, policy, func(context.Context) (int, error) {
|
||||
attempts++
|
||||
return 0, want
|
||||
})
|
||||
if !errors.Is(err, want) || attempts != 1 {
|
||||
t.Fatalf("err=%v attempts=%d", err, attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCWithFloodWaitPolicyRejectsExcessiveWait(t *testing.T) {
|
||||
policy := floodWaitPolicy{maxRetries: 2, maxWait: time.Second, wait: waitForContext}
|
||||
_, err := rpcWithFloodWaitPolicy(context.Background(), time.Second, policy, func(context.Context) (int, error) {
|
||||
return 0, tgerr.New(420, "FLOOD_WAIT_2")
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("excessive FLOOD_WAIT unexpectedly retried")
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@ import (
|
|||
"net"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
|
@ -43,12 +44,17 @@ type RunConfig struct {
|
|||
EventsPath string
|
||||
FileFixturePath string
|
||||
ServerMetricsURL string
|
||||
StartOrder string
|
||||
StartOrderSeed int64
|
||||
SessionLimit int
|
||||
Duration time.Duration
|
||||
RecoveryDuration time.Duration
|
||||
RampDuration time.Duration
|
||||
RPCInterval time.Duration
|
||||
MessageInterval time.Duration
|
||||
MessageRate float64
|
||||
MessageQueueDepth int
|
||||
DeliverySettle time.Duration
|
||||
FileInterval time.Duration
|
||||
FileSizeBytes int
|
||||
FileChunkBytes int
|
||||
|
|
@ -62,6 +68,8 @@ type RunConfig struct {
|
|||
ExpectServerRestart bool
|
||||
}
|
||||
|
||||
const RunReportVersion = 7
|
||||
|
||||
func (c RunConfig) validate() error {
|
||||
if c.ManifestPath == "" || c.SessionKeyPath == "" || c.ReportPath == "" {
|
||||
return errors.New("manifest, session-key and report paths are required")
|
||||
|
|
@ -69,6 +77,15 @@ func (c RunConfig) validate() error {
|
|||
if c.Duration <= 0 || c.RecoveryDuration < 0 || c.RampDuration < 0 || c.RPCInterval <= 0 || c.OperationTimeout <= 0 || c.SampleInterval <= 0 {
|
||||
return errors.New("run durations and intervals are invalid")
|
||||
}
|
||||
if c.MessageRate < 0 || c.MessageRate > 100000 || c.MessageQueueDepth < 0 || c.MessageQueueDepth > 1024 || c.DeliverySettle < 0 {
|
||||
return errors.New("message rate, queue depth or delivery settle is invalid")
|
||||
}
|
||||
if c.MessageRate > 0 && c.MessageInterval > 0 {
|
||||
return errors.New("message-rate and message-interval workloads are mutually exclusive")
|
||||
}
|
||||
if c.MessageRate > 0 && (c.MessageQueueDepth == 0 || c.RampDuration >= c.Duration) {
|
||||
return errors.New("fixed-rate workload requires a queue depth and load duration beyond the connection ramp")
|
||||
}
|
||||
if c.FileSizeBytes < 0 || c.FileChunkBytes < 0 || c.FileChunkBytes > 1<<20 || c.FileSizeBytes > 64<<20 {
|
||||
return errors.New("file size must be <=64MiB and chunk size must be <=1MiB")
|
||||
}
|
||||
|
|
@ -84,6 +101,9 @@ func (c RunConfig) validate() error {
|
|||
if c.OfflineFraction > 0 && (c.OfflineAt <= 0 || c.OfflineFor <= 0 || c.OfflineAt+c.OfflineFor >= c.Duration) {
|
||||
return errors.New("offline window must be positive and fit inside load duration")
|
||||
}
|
||||
if c.StartOrder != "" && c.StartOrder != StartupOrderShuffled && c.StartOrder != StartupOrderAccountIndex {
|
||||
return fmt.Errorf("unknown run start order %q", c.StartOrder)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -113,6 +133,11 @@ type harnessCounters struct {
|
|||
updates atomic.Uint64
|
||||
fatalErrors atomic.Uint64
|
||||
downloadBytes atomic.Uint64
|
||||
messageScheduled atomic.Uint64
|
||||
messageEnqueued atomic.Uint64
|
||||
messageCompleted atomic.Uint64
|
||||
messageQueueFull atomic.Uint64
|
||||
messageNotReady atomic.Uint64
|
||||
}
|
||||
|
||||
var debugConnectionErrors atomic.Uint64
|
||||
|
|
@ -159,21 +184,26 @@ type loadWorker struct {
|
|||
fileInterval time.Duration
|
||||
operationTimeout time.Duration
|
||||
fileFixture *downloadFixture
|
||||
delivery *deliveryTracker
|
||||
|
||||
desired atomic.Bool
|
||||
state atomic.Int32
|
||||
everReady atomic.Bool
|
||||
signal chan struct{}
|
||||
lastUpdate updateState
|
||||
messageSeq atomic.Uint64
|
||||
desired atomic.Bool
|
||||
state atomic.Int32
|
||||
everReady atomic.Bool
|
||||
signal chan struct{}
|
||||
lastUpdate updateState
|
||||
deliveryState updateState
|
||||
messageSeq atomic.Uint64
|
||||
sendQueue chan struct{}
|
||||
reconcile chan chan struct{}
|
||||
}
|
||||
|
||||
func newLoadWorker(record, target SessionRecord, endpoint Endpoint, publicKey *rsa.PublicKey, storage *EncryptedFileStorage, metrics *metricSet, counters *harnessCounters, events *eventWriter, rpcInterval, messageInterval, fileInterval, operationTimeout time.Duration, fixture *downloadFixture) *loadWorker {
|
||||
func newLoadWorker(record, target SessionRecord, endpoint Endpoint, publicKey *rsa.PublicKey, storage *EncryptedFileStorage, metrics *metricSet, counters *harnessCounters, events *eventWriter, rpcInterval, messageInterval, fileInterval, operationTimeout time.Duration, fixture *downloadFixture, delivery *deliveryTracker, messageQueueDepth int) *loadWorker {
|
||||
w := &loadWorker{
|
||||
record: record, target: target, endpoint: endpoint, publicKey: publicKey, storage: storage,
|
||||
metrics: metrics, counters: counters, events: events, rpcInterval: rpcInterval,
|
||||
msgInterval: messageInterval, fileInterval: fileInterval, operationTimeout: operationTimeout, fileFixture: fixture,
|
||||
signal: make(chan struct{}, 1),
|
||||
delivery: delivery, signal: make(chan struct{}, 1), sendQueue: make(chan struct{}, messageQueueDepth),
|
||||
reconcile: make(chan chan struct{}),
|
||||
}
|
||||
w.state.Store(workerStopped)
|
||||
return w
|
||||
|
|
@ -259,9 +289,11 @@ func (w *loadWorker) supervise(ctx context.Context, wg *sync.WaitGroup) {
|
|||
|
||||
func (w *loadWorker) runClient(ctx context.Context) error {
|
||||
reconnectSignal := make(chan struct{}, 1)
|
||||
var readySeen atomic.Bool
|
||||
client, err := newClient(w.endpoint, w.publicKey, w.storage, clientHooks{
|
||||
Update: telegram.UpdateHandlerFunc(func(context.Context, tg.UpdatesClass) error {
|
||||
Update: telegram.UpdateHandlerFunc(func(_ context.Context, updates tg.UpdatesClass) error {
|
||||
w.counters.updates.Add(1)
|
||||
observeUpdatesClass(w.delivery, w.record.UserID, updates, deliveryLive)
|
||||
return nil
|
||||
}),
|
||||
ConnectionState: func(state telegram.ConnectionState) {
|
||||
|
|
@ -273,9 +305,9 @@ func (w *loadWorker) runClient(ctx context.Context) error {
|
|||
w.counters.reconnects.Add(1)
|
||||
}
|
||||
case telegram.ConnectionStateReady:
|
||||
wasReady := w.everReady.Swap(true)
|
||||
needsCatchUp := markClientReady(&w.everReady, &readySeen)
|
||||
w.state.Store(workerReady)
|
||||
if wasReady {
|
||||
if needsCatchUp {
|
||||
select {
|
||||
case reconnectSignal <- struct{}{}:
|
||||
default:
|
||||
|
|
@ -316,6 +348,9 @@ func (w *loadWorker) runClient(ctx context.Context) error {
|
|||
} else {
|
||||
w.refreshUpdateState(ctx, raw)
|
||||
}
|
||||
if _, valid := w.deliveryState.load(); !valid {
|
||||
w.refreshDeliveryState(ctx, raw)
|
||||
}
|
||||
|
||||
rpcTicker := time.NewTicker(w.rpcInterval)
|
||||
defer rpcTicker.Stop()
|
||||
|
|
@ -345,6 +380,11 @@ func (w *loadWorker) runClient(ctx context.Context) error {
|
|||
cycle++
|
||||
case <-messageC:
|
||||
w.sendMessage(ctx, raw)
|
||||
case <-w.sendQueue:
|
||||
w.sendMessage(ctx, raw)
|
||||
case done := <-w.reconcile:
|
||||
w.catchUpDelivery(ctx, raw)
|
||||
close(done)
|
||||
case <-fileC:
|
||||
w.downloadFileChunk(ctx, raw)
|
||||
}
|
||||
|
|
@ -352,6 +392,19 @@ func (w *loadWorker) runClient(ctx context.Context) error {
|
|||
})
|
||||
}
|
||||
|
||||
// markClientReady distinguishes a transport reconnect inside one live gotd
|
||||
// Client from the first Ready transition of a newly constructed Client. The
|
||||
// client.Run callback already performs one cursor catch-up when it starts, so
|
||||
// enqueueing a second catch-up for that first transition would duplicate every
|
||||
// explicit offline->online getDifference request. Later Ready transitions do
|
||||
// need the signal because the callback remains running across transport-level
|
||||
// reconnects.
|
||||
func markClientReady(everReady, readySeen *atomic.Bool) bool {
|
||||
firstForClient := !readySeen.Swap(true)
|
||||
wasReady := everReady.Swap(true)
|
||||
return wasReady && !firstForClient
|
||||
}
|
||||
|
||||
func (w *loadWorker) runRPC(ctx context.Context, client *telegram.Client, raw *tg.Client, cycle int) {
|
||||
start := time.Now()
|
||||
operationCtx, cancel := w.operationContext(ctx)
|
||||
|
|
@ -385,6 +438,20 @@ func (w *loadWorker) refreshUpdateState(ctx context.Context, raw *tg.Client) {
|
|||
w.metrics.observe("updates.getState", start, err)
|
||||
if err == nil {
|
||||
w.lastUpdate.store(*state)
|
||||
if _, valid := w.deliveryState.load(); !valid {
|
||||
w.deliveryState.store(*state)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *loadWorker) refreshDeliveryState(ctx context.Context, raw *tg.Client) {
|
||||
start := time.Now()
|
||||
operationCtx, cancel := w.operationContext(ctx)
|
||||
state, err := raw.UpdatesGetState(operationCtx)
|
||||
cancel()
|
||||
w.metrics.observe("updates.getState.delivery", start, err)
|
||||
if err == nil {
|
||||
w.deliveryState.store(*state)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -427,7 +494,49 @@ func (w *loadWorker) catchUp(ctx context.Context, raw *tg.Client) {
|
|||
}
|
||||
}
|
||||
|
||||
func (w *loadWorker) catchUpDelivery(ctx context.Context, raw *tg.Client) {
|
||||
state, valid := w.deliveryState.load()
|
||||
if !valid {
|
||||
w.refreshDeliveryState(ctx, raw)
|
||||
return
|
||||
}
|
||||
for page := 0; page < 256; page++ {
|
||||
start := time.Now()
|
||||
operationCtx, cancel := w.operationContext(ctx)
|
||||
difference, err := raw.UpdatesGetDifference(operationCtx, &tg.UpdatesGetDifferenceRequest{Pts: state.Pts, Date: state.Date, Qts: state.Qts})
|
||||
cancel()
|
||||
w.metrics.observe("updates.getDifference.delivery", start, err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
switch value := difference.(type) {
|
||||
case *tg.UpdatesDifferenceEmpty:
|
||||
state.Date, state.Seq = value.Date, value.Seq
|
||||
w.deliveryState.store(state)
|
||||
return
|
||||
case *tg.UpdatesDifference:
|
||||
observeMessageClasses(w.delivery, w.record.UserID, value.NewMessages, deliveryDifference)
|
||||
observeUpdateClasses(w.delivery, w.record.UserID, value.OtherUpdates, deliveryDifference)
|
||||
state = value.State
|
||||
w.deliveryState.store(state)
|
||||
return
|
||||
case *tg.UpdatesDifferenceSlice:
|
||||
observeMessageClasses(w.delivery, w.record.UserID, value.NewMessages, deliveryDifference)
|
||||
observeUpdateClasses(w.delivery, w.record.UserID, value.OtherUpdates, deliveryDifference)
|
||||
state = value.IntermediateState
|
||||
w.deliveryState.store(state)
|
||||
case *tg.UpdatesDifferenceTooLong:
|
||||
state.Pts = value.Pts
|
||||
w.deliveryState.store(state)
|
||||
return
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *loadWorker) sendMessage(ctx context.Context, raw *tg.Client) {
|
||||
defer w.counters.messageCompleted.Add(1)
|
||||
sequence := w.messageSeq.Add(1)
|
||||
var randomBytes [8]byte
|
||||
if _, err := cryptorand.Read(randomBytes[:]); err != nil {
|
||||
|
|
@ -438,13 +547,16 @@ func (w *loadWorker) sendMessage(ctx context.Context, raw *tg.Client) {
|
|||
randomID = int64(sequence)
|
||||
}
|
||||
start := time.Now()
|
||||
marker := w.delivery.marker(w.record.Index, sequence)
|
||||
w.delivery.begin(marker, w.record.UserID, w.target.UserID, start)
|
||||
operationCtx, cancel := w.operationContext(ctx)
|
||||
_, err := raw.MessagesSendMessage(operationCtx, &tg.MessagesSendMessageRequest{
|
||||
Peer: &tg.InputPeerUser{UserID: w.target.UserID, AccessHash: w.target.AccessHash},
|
||||
Message: fmt.Sprintf("load/%d/%d", w.record.Index, sequence), RandomID: randomID,
|
||||
Message: marker, RandomID: randomID,
|
||||
})
|
||||
cancel()
|
||||
w.metrics.observe("messages.sendMessage", start, err)
|
||||
w.delivery.finish(marker, err == nil)
|
||||
}
|
||||
|
||||
func (w *loadWorker) downloadFileChunk(ctx context.Context, raw *tg.Client) {
|
||||
|
|
@ -654,8 +766,13 @@ func Run(ctx context.Context, cfg RunConfig) (*RunReport, error) {
|
|||
return nil, err
|
||||
}
|
||||
defer events.close()
|
||||
metrics := newMetricSet("auth.status", "connection.dead", "ping", "updates.getState", "updates.getDifference", "messages.getDialogs", "help.getConfig", "messages.sendMessage", "upload.saveFilePart", "messages.uploadMedia", "upload.getFile")
|
||||
metrics := newMetricSet("auth.status", "connection.dead", "ping", "updates.getState", "updates.getDifference", "updates.getState.delivery", "updates.getDifference.delivery", "messages.getDialogs", "help.getConfig", "messages.sendMessage", "upload.saveFilePart", "messages.uploadMedia", "upload.getFile")
|
||||
counters := &harnessCounters{}
|
||||
runID, err := newLoadRunID()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create load run id: %w", err)
|
||||
}
|
||||
delivery := newDeliveryTracker(runID)
|
||||
serverMetrics := newServerMetricsClient(cfg.ServerMetricsURL)
|
||||
var baselineServerMetrics map[string]float64
|
||||
if serverMetrics != nil {
|
||||
|
|
@ -681,8 +798,10 @@ func Run(ctx context.Context, cfg RunConfig) (*RunReport, error) {
|
|||
record, target, manifest.Endpoint, publicKey,
|
||||
&EncryptedFileStorage{Path: resolveSessionPath(cfg.ManifestPath, record), Key: key},
|
||||
metrics, counters, events, cfg.RPCInterval, cfg.MessageInterval, cfg.FileInterval, cfg.OperationTimeout, fixture,
|
||||
delivery, cfg.MessageQueueDepth,
|
||||
))
|
||||
}
|
||||
messageWorkers := primaryWorkers(workers)
|
||||
|
||||
startedAt := time.Now().UTC()
|
||||
loadCtx, stopLoad := context.WithCancel(ctx)
|
||||
|
|
@ -691,10 +810,20 @@ func Run(ctx context.Context, cfg RunConfig) (*RunReport, error) {
|
|||
workerWG.Add(1)
|
||||
go worker.supervise(loadCtx, &workerWG)
|
||||
}
|
||||
for i, worker := range workers {
|
||||
startOrder := cfg.StartOrder
|
||||
if startOrder == "" {
|
||||
startOrder = StartupOrderAccountIndex
|
||||
}
|
||||
startOrderSeed := cfg.StartOrderSeed
|
||||
if startOrderSeed == 0 {
|
||||
startOrderSeed = 20260827
|
||||
}
|
||||
launchOrder := startupAccountOrder(len(workers), startOrder, startOrderSeed)
|
||||
for position, workerIndex := range launchOrder {
|
||||
worker := workers[workerIndex]
|
||||
delay := time.Duration(0)
|
||||
if len(workers) > 1 {
|
||||
delay = time.Duration(i) * cfg.RampDuration / time.Duration(len(workers)-1)
|
||||
delay = time.Duration(position) * cfg.RampDuration / time.Duration(len(workers)-1)
|
||||
}
|
||||
go func(w *loadWorker, d time.Duration) {
|
||||
timer := time.NewTimer(d)
|
||||
|
|
@ -710,6 +839,13 @@ func Run(ctx context.Context, cfg RunConfig) (*RunReport, error) {
|
|||
if cfg.OfflineFraction > 0 {
|
||||
go runOfflineWindow(loadCtx, workers, cfg.OfflineFraction, cfg.OfflineAt, cfg.OfflineFor, events)
|
||||
}
|
||||
messageCtx, stopMessages := context.WithCancel(loadCtx)
|
||||
defer stopMessages()
|
||||
var messageWG sync.WaitGroup
|
||||
if cfg.MessageRate > 0 {
|
||||
messageWG.Add(1)
|
||||
go runFixedMessageSchedule(messageCtx, &messageWG, cfg.RampDuration, cfg.MessageRate, messageWorkers, counters, events)
|
||||
}
|
||||
loadTimer := time.NewTimer(cfg.Duration)
|
||||
sampleTicker := time.NewTicker(cfg.SampleInterval)
|
||||
peakReady := 0
|
||||
|
|
@ -741,11 +877,64 @@ func Run(ctx context.Context, cfg RunConfig) (*RunReport, error) {
|
|||
|
||||
loadFinished:
|
||||
sampleTicker.Stop()
|
||||
stopMessages()
|
||||
messageWG.Wait()
|
||||
loadEndedAt := time.Now().UTC()
|
||||
if cfg.MessageRate > 0 {
|
||||
drainCtx, cancelDrain := context.WithTimeout(ctx, cfg.OperationTimeout+time.Duration(cfg.MessageQueueDepth)*cfg.OperationTimeout)
|
||||
waitMessageDrain(drainCtx, counters)
|
||||
cancelDrain()
|
||||
}
|
||||
if cfg.DeliverySettle > 0 && delivery.report().Expected > delivery.report().Delivered {
|
||||
settleTimer := time.NewTimer(cfg.DeliverySettle)
|
||||
settleTicker := time.NewTicker(min(cfg.SampleInterval, time.Second))
|
||||
settling:
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
settleTimer.Stop()
|
||||
settleTicker.Stop()
|
||||
stopLoad()
|
||||
workerWG.Wait()
|
||||
return nil, ctx.Err()
|
||||
case <-settleTicker.C:
|
||||
if current := delivery.report(); current.Missing == 0 {
|
||||
settleTimer.Stop()
|
||||
settleTicker.Stop()
|
||||
break settling
|
||||
}
|
||||
case <-settleTimer.C:
|
||||
settleTicker.Stop()
|
||||
break settling
|
||||
}
|
||||
}
|
||||
}
|
||||
if delivery.report().Missing > 0 {
|
||||
reconcileCtx, cancelReconcile := context.WithTimeout(ctx, cfg.OperationTimeout*2)
|
||||
reconcileDeliveries(reconcileCtx, workers)
|
||||
cancelReconcile()
|
||||
}
|
||||
// Take the authoritative business-work cutoff before canceling clients.
|
||||
// Coordinated teardown can cancel an RPC already in flight; both server
|
||||
// outcome counters and client operation counters must therefore stop before
|
||||
// that cancellation begins. FinalServerMetrics remains the post-recovery
|
||||
// resource-reclamation snapshot.
|
||||
var workloadEndServerMetrics map[string]float64
|
||||
if serverMetrics != nil {
|
||||
if sample, scrapeErr := serverMetrics.scrape(ctx); scrapeErr == nil {
|
||||
workloadEndServerMetrics = sample
|
||||
finalServerMetrics = sample
|
||||
events.write(map[string]any{"type": "server_workload_end", "at": time.Now().UTC(), "server_metrics": sample})
|
||||
} else {
|
||||
events.write(map[string]any{"type": "server_workload_end_error", "at": time.Now().UTC(), "class": classifyError(scrapeErr)})
|
||||
}
|
||||
}
|
||||
workloadEndOperations := metrics.freeze()
|
||||
finalReady := countWorkerState(workers, workerReady)
|
||||
stopLoad()
|
||||
workerWG.Wait()
|
||||
loadEndedAt := time.Now().UTC()
|
||||
if ready := countWorkerState(workers, workerReady); ready > peakReady {
|
||||
peakReady = ready
|
||||
if finalReady > peakReady {
|
||||
peakReady = finalReady
|
||||
}
|
||||
|
||||
if cfg.RecoveryDuration > 0 {
|
||||
|
|
@ -779,16 +968,24 @@ recoveryFinished:
|
|||
steadyRatio = float64(steadyReadySum) / float64(steadySamples*len(workers))
|
||||
}
|
||||
report := &RunReport{
|
||||
Version: 2, StartedAt: startedAt, LoadEndedAt: loadEndedAt, FinishedAt: time.Now().UTC(),
|
||||
Version: RunReportVersion, StartedAt: startedAt, LoadEndedAt: loadEndedAt, FinishedAt: time.Now().UTC(),
|
||||
StartOrder: startOrder, StartOrderSeed: startOrderSeed,
|
||||
RequestedDuration: cfg.Duration.String(), RecoveryDuration: cfg.RecoveryDuration.String(),
|
||||
ExpectedSessions: len(workers), PeakReadySessions: peakReady, FinalReadySessions: countWorkerState(workers, workerReady),
|
||||
ExpectedSessions: len(workers), PeakReadySessions: peakReady, FinalReadySessions: finalReady,
|
||||
ConnectionAttempts: counters.connectionAttempts.Load(), Reconnects: counters.reconnects.Load(),
|
||||
Disconnects: counters.disconnects.Load(), UpdatesReceived: counters.updates.Load(), DownloadedBytes: counters.downloadBytes.Load(),
|
||||
WorkerFatalErrors: counters.fatalErrors.Load(), Operations: metrics.report(),
|
||||
BaselineServerMetrics: baselineServerMetrics, FinalServerMetrics: finalServerMetrics,
|
||||
WorkerFatalErrors: counters.fatalErrors.Load(), Operations: workloadEndOperations,
|
||||
BaselineServerMetrics: baselineServerMetrics, WorkloadEndServerMetrics: workloadEndServerMetrics, FinalServerMetrics: finalServerMetrics,
|
||||
ServerMetricsScrapes: serverMetrics.successes(), ServerMetricsErrors: serverMetrics.failures(),
|
||||
SteadySamples: steadySamples, SteadyReadyRatio: steadyRatio, MinSteadyReadySessions: steadyReadyMinimum,
|
||||
MessageRatePerSecond: cfg.MessageRate, MessageScheduled: counters.messageScheduled.Load(),
|
||||
MessageEnqueued: counters.messageEnqueued.Load(), MessageCompleted: counters.messageCompleted.Load(), MessageQueueFull: counters.messageQueueFull.Load(),
|
||||
MessageNotReady: counters.messageNotReady.Load(), Delivery: delivery.report(),
|
||||
}
|
||||
report.ResponseBytes = startupResponseBytes(baselineServerMetrics, workloadEndServerMetrics)
|
||||
report.RPCDeliveryOutcomes = startupRPCDeliveryOutcomes(baselineServerMetrics, workloadEndServerMetrics)
|
||||
report.DatabaseWork = startupDatabaseWork(baselineServerMetrics, workloadEndServerMetrics)
|
||||
report.EventsWritten, report.EventsDropped = events.counts()
|
||||
evaluateReport(report, cfg)
|
||||
if err := WriteReport(cfg.ReportPath, report); err != nil {
|
||||
return nil, err
|
||||
|
|
@ -814,6 +1011,119 @@ func primaryTargets(records []SessionRecord) []SessionRecord {
|
|||
return targets
|
||||
}
|
||||
|
||||
func newLoadRunID() (string, error) {
|
||||
var value [8]byte
|
||||
if _, err := cryptorand.Read(value[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(value[:]), nil
|
||||
}
|
||||
|
||||
func primaryWorkers(workers []*loadWorker) []*loadWorker {
|
||||
primary := make([]*loadWorker, 0, len(workers))
|
||||
for _, worker := range workers {
|
||||
if worker.record.DeviceIndex == 0 && worker.target.UserID > 0 {
|
||||
primary = append(primary, worker)
|
||||
}
|
||||
}
|
||||
return primary
|
||||
}
|
||||
|
||||
func runFixedMessageSchedule(ctx context.Context, wg *sync.WaitGroup, startDelay time.Duration, rate float64, workers []*loadWorker, counters *harnessCounters, events *eventWriter) {
|
||||
defer wg.Done()
|
||||
if rate <= 0 || len(workers) == 0 {
|
||||
return
|
||||
}
|
||||
startTimer := time.NewTimer(startDelay)
|
||||
defer startTimer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-startTimer.C:
|
||||
}
|
||||
readyTicker := time.NewTicker(10 * time.Millisecond)
|
||||
for countWorkerState(workers, workerReady) != len(workers) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
readyTicker.Stop()
|
||||
return
|
||||
case <-readyTicker.C:
|
||||
}
|
||||
}
|
||||
readyTicker.Stop()
|
||||
interval := time.Duration(float64(time.Second) / rate)
|
||||
if interval < time.Microsecond {
|
||||
interval = time.Microsecond
|
||||
}
|
||||
events.write(map[string]any{"type": "fixed_message_rate_start", "at": time.Now().UTC(), "rate_per_second": rate, "senders": len(workers)})
|
||||
next := time.Now()
|
||||
workerIndex := 0
|
||||
timer := time.NewTimer(0)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
events.write(map[string]any{
|
||||
"type": "fixed_message_rate_stop", "at": time.Now().UTC(),
|
||||
"scheduled": counters.messageScheduled.Load(), "enqueued": counters.messageEnqueued.Load(),
|
||||
"queue_full": counters.messageQueueFull.Load(), "not_ready": counters.messageNotReady.Load(),
|
||||
})
|
||||
return
|
||||
case <-timer.C:
|
||||
worker := workers[workerIndex]
|
||||
workerIndex = (workerIndex + 1) % len(workers)
|
||||
counters.messageScheduled.Add(1)
|
||||
if worker.state.Load() != workerReady {
|
||||
counters.messageNotReady.Add(1)
|
||||
} else {
|
||||
select {
|
||||
case worker.sendQueue <- struct{}{}:
|
||||
counters.messageEnqueued.Add(1)
|
||||
default:
|
||||
counters.messageQueueFull.Add(1)
|
||||
}
|
||||
}
|
||||
next = next.Add(interval)
|
||||
timer.Reset(max(time.Until(next), time.Duration(0)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func reconcileDeliveries(ctx context.Context, workers []*loadWorker) {
|
||||
waits := make([]chan struct{}, 0, len(workers))
|
||||
for _, worker := range workers {
|
||||
if worker.state.Load() != workerReady {
|
||||
continue
|
||||
}
|
||||
done := make(chan struct{})
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case worker.reconcile <- done:
|
||||
waits = append(waits, done)
|
||||
}
|
||||
}
|
||||
for _, done := range waits {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-done:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func waitMessageDrain(ctx context.Context, counters *harnessCounters) {
|
||||
ticker := time.NewTicker(10 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for counters.messageCompleted.Load() < counters.messageEnqueued.Load() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func minimumOpenFiles(sessions int) int {
|
||||
if sessions < 0 {
|
||||
sessions = 0
|
||||
|
|
@ -901,6 +1211,46 @@ func evaluateReport(report *RunReport, cfg RunConfig) {
|
|||
if report.WorkerFatalErrors > 0 {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("worker fatal errors: %d", report.WorkerFatalErrors))
|
||||
}
|
||||
if cfg.MessageRate > 0 {
|
||||
if report.MessageScheduled == 0 {
|
||||
report.Failures = append(report.Failures, "fixed-rate message scheduler produced no arrivals")
|
||||
}
|
||||
if report.MessageNotReady > 0 {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("fixed-rate arrivals rejected because sender was not ready: %d", report.MessageNotReady))
|
||||
}
|
||||
if report.MessageQueueFull > 0 {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("fixed-rate arrivals rejected by bounded sender queues: %d", report.MessageQueueFull))
|
||||
}
|
||||
if report.MessageEnqueued != report.MessageScheduled {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("fixed-rate scheduler enqueued %d of %d arrivals", report.MessageEnqueued, report.MessageScheduled))
|
||||
}
|
||||
if report.MessageCompleted != report.MessageEnqueued {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("message workers completed %d of %d enqueued sends", report.MessageCompleted, report.MessageEnqueued))
|
||||
}
|
||||
sendOperation := report.Operations["messages.sendMessage"]
|
||||
if sendOperation.Count != report.MessageCompleted {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("messages.sendMessage recorded %d of %d completed jobs", sendOperation.Count, report.MessageCompleted))
|
||||
}
|
||||
successfulSends := sendOperation.Count - min(sendOperation.Count, sendOperation.Errors+sendOperation.Canceled)
|
||||
if report.Delivery.Expected != successfulSends {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("delivery tracker committed %d of %d successful send RPCs", report.Delivery.Expected, successfulSends))
|
||||
}
|
||||
if report.Delivery.Missing > 0 || report.Delivery.Delivered != report.Delivery.Expected {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("recipient delivery incomplete: delivered %d of %d, missing %d", report.Delivery.Delivered, report.Delivery.Expected, report.Delivery.Missing))
|
||||
}
|
||||
if cfg.OfflineFraction == 0 && report.Delivery.DifferenceRecovered > 0 {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("online recipients recovered %d messages only through updates.getDifference", report.Delivery.DifferenceRecovered))
|
||||
}
|
||||
if report.Delivery.DuplicateObservations > 0 {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("recipient observed %d duplicate message updates", report.Delivery.DuplicateObservations))
|
||||
}
|
||||
if report.Delivery.WrongAccountObserved > 0 {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("load markers appeared on %d wrong recipient accounts", report.Delivery.WrongAccountObserved))
|
||||
}
|
||||
if report.Delivery.UnmatchedMarkers > 0 {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("observed %d load markers without a successful send RPC", report.Delivery.UnmatchedMarkers))
|
||||
}
|
||||
}
|
||||
for name, operation := range report.Operations {
|
||||
if operation.FloodWaits > 0 {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("%s returned FLOOD_WAIT %d times", name, operation.FloodWaits))
|
||||
|
|
@ -913,6 +1263,34 @@ func evaluateReport(report *RunReport, cfg RunConfig) {
|
|||
report.Failures = append(report.Failures, fmt.Sprintf("%s returned %d unexpected non-cancel errors", name, unexpectedErrors))
|
||||
}
|
||||
}
|
||||
methods := make([]string, 0, len(report.RPCDeliveryOutcomes))
|
||||
for method := range report.RPCDeliveryOutcomes {
|
||||
methods = append(methods, method)
|
||||
}
|
||||
sort.Strings(methods)
|
||||
for _, method := range methods {
|
||||
outcomes := report.RPCDeliveryOutcomes[method]
|
||||
outcomeNames := make([]string, 0, len(outcomes))
|
||||
for outcome := range outcomes {
|
||||
outcomeNames = append(outcomeNames, outcome)
|
||||
}
|
||||
sort.Strings(outcomeNames)
|
||||
for _, outcome := range outcomeNames {
|
||||
if count := outcomes[outcome]; outcome != "ok" && count > 0 {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("%s rpc_result delivery outcome %s: %d", method, outcome, count))
|
||||
}
|
||||
}
|
||||
}
|
||||
methods = methods[:0]
|
||||
for method := range report.DatabaseWork {
|
||||
methods = append(methods, method)
|
||||
}
|
||||
sort.Strings(methods)
|
||||
for _, method := range methods {
|
||||
if errors := report.DatabaseWork[method].Errors; errors > 0 {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("%s database errors: %d", method, errors))
|
||||
}
|
||||
}
|
||||
if cfg.ExpectServerRestart && report.Reconnects < uint64(requiredReady) {
|
||||
report.Failures = append(report.Failures, fmt.Sprintf("server restart expected at least %d reconnect attempts, observed %d", requiredReady, report.Reconnects))
|
||||
}
|
||||
|
|
@ -938,6 +1316,9 @@ func evaluateReport(report *RunReport, cfg RunConfig) {
|
|||
if strings.TrimSpace(cfg.ServerMetricsURL) != "" && report.FinalServerMetrics == nil {
|
||||
report.Failures = append(report.Failures, "final post-recovery server metrics scrape failed")
|
||||
}
|
||||
if strings.TrimSpace(cfg.ServerMetricsURL) != "" && report.WorkloadEndServerMetrics == nil {
|
||||
report.Failures = append(report.Failures, "pre-teardown workload-end server metrics scrape failed")
|
||||
}
|
||||
if strings.TrimSpace(cfg.ServerMetricsURL) != "" && report.BaselineServerMetrics == nil {
|
||||
report.Failures = append(report.Failures, "pre-load server metrics baseline scrape failed")
|
||||
}
|
||||
|
|
@ -945,9 +1326,16 @@ func evaluateReport(report *RunReport, cfg RunConfig) {
|
|||
}
|
||||
|
||||
func metricValue(values map[string]float64, name string) float64 {
|
||||
// The scraper always stores an aggregate family value in the bare key and
|
||||
// may additionally retain bounded state/method label breakdowns. Prefer that
|
||||
// aggregate; summing both would double-count every labeled family in resource
|
||||
// recovery checks (for example retained/offline logical sessions).
|
||||
if value, ok := values[name]; ok {
|
||||
return value
|
||||
}
|
||||
var total float64
|
||||
for key, value := range values {
|
||||
if key == name || strings.HasPrefix(key, name+"{") {
|
||||
if strings.HasPrefix(key, name+"{") {
|
||||
total += value
|
||||
}
|
||||
}
|
||||
|
|
@ -1032,6 +1420,8 @@ func classifyErrorReason(err error) string {
|
|||
return "dns"
|
||||
case strings.Contains(message, "BROKEN PIPE"):
|
||||
return "broken_pipe"
|
||||
case strings.Contains(message, "ENDED BEFORE BUSINESS READINESS"):
|
||||
return "business_readiness_incomplete"
|
||||
case strings.Contains(message, "EOF"):
|
||||
return "eof"
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
|
|
|
|||
714
internal/loadharness/seed.go
Normal file
714
internal/loadharness/seed.go
Normal file
|
|
@ -0,0 +1,714 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rsa"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/iamxvbaba/td/tg"
|
||||
"github.com/iamxvbaba/td/tgerr"
|
||||
)
|
||||
|
||||
const maxSeedInviteBatch = 200
|
||||
|
||||
type SeedConfig struct {
|
||||
ManifestPath string
|
||||
SessionKeyPath string
|
||||
RSAKeyOverride string
|
||||
DatasetPath string
|
||||
SeedStatePath string
|
||||
Concurrency int
|
||||
OperationTimeout time.Duration
|
||||
}
|
||||
|
||||
type SeedEvent struct {
|
||||
Phase string
|
||||
Completed int
|
||||
Total int
|
||||
Account int
|
||||
Err error
|
||||
}
|
||||
|
||||
type SeedResult struct {
|
||||
PrivateMessages int
|
||||
Groups int
|
||||
InvitedMembers int
|
||||
GroupMessages int
|
||||
RichStateAccounts int
|
||||
}
|
||||
|
||||
func (c SeedConfig) validate() error {
|
||||
if c.ManifestPath == "" || c.SessionKeyPath == "" || c.DatasetPath == "" || c.SeedStatePath == "" {
|
||||
return errors.New("manifest, session-key, dataset and seed-state paths are required")
|
||||
}
|
||||
if c.Concurrency <= 0 || c.Concurrency > 64 {
|
||||
return errors.New("seed concurrency must be between 1 and 64")
|
||||
}
|
||||
if c.OperationTimeout <= 0 {
|
||||
return errors.New("seed operation timeout must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Seed materializes a Dataset only through authenticated MTProto RPCs. It never
|
||||
// calls a server-internal handler or store. The journal is persisted before any
|
||||
// non-idempotent channel operation, so an interrupted run can reconcile using
|
||||
// the same account's public RPC view before it proceeds.
|
||||
func Seed(ctx context.Context, cfg SeedConfig, progress func(SeedEvent)) (*SeedResult, error) {
|
||||
if err := cfg.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
manifest, err := LoadManifest(cfg.ManifestPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dataset, err := LoadDataset(cfg.DatasetPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targets, err := seedPrimaryTargets(manifest, dataset.Config.Accounts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := LoadSessionKey(cfg.SessionKeyPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
publicKey, err := loadManifestPublicKey(cfg.ManifestPath, manifest.Endpoint, cfg.RSAKeyOverride)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
state, err := LoadDatasetSeedState(cfg.SeedStatePath, dataset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
journal := &seedJournal{path: cfg.SeedStatePath, dataset: dataset, state: state}
|
||||
if err := journal.enableRichState(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := journal.persist(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
accounts := make([]int, dataset.Config.Accounts)
|
||||
for account := range accounts {
|
||||
accounts[account] = account
|
||||
}
|
||||
if err := runSeedAccountPhase(ctx, "private", accounts, cfg.Concurrency, progress, func(ctx context.Context, account int) error {
|
||||
return seedPrivateMessages(ctx, cfg, manifest, dataset, journal, targets, key, publicKey, account)
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
groupsByCreator := make(map[int][]int)
|
||||
for position, group := range dataset.Groups {
|
||||
groupsByCreator[group.CreatorAccount] = append(groupsByCreator[group.CreatorAccount], position)
|
||||
}
|
||||
creators := make([]int, 0, len(groupsByCreator))
|
||||
for account := range groupsByCreator {
|
||||
creators = append(creators, account)
|
||||
}
|
||||
sort.Ints(creators)
|
||||
if err := runSeedAccountPhase(ctx, "groups", creators, cfg.Concurrency, progress, func(ctx context.Context, account int) error {
|
||||
return withAuthorizedSeedSession(ctx, cfg, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
|
||||
for _, position := range groupsByCreator[account] {
|
||||
if err := seedGroup(ctx, cfg, dataset, journal, targets, raw, position); err != nil {
|
||||
return fmt.Errorf("group %d: %w", dataset.Groups[position].Index, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
historyTasks := datasetHistoryTasks(dataset)
|
||||
if err := runSeedAccountPhase(ctx, "group-history", accounts, cfg.Concurrency, progress, func(ctx context.Context, account int) error {
|
||||
return seedGroupHistory(ctx, cfg, manifest, dataset, journal, targets, key, publicKey, account, historyTasks[account])
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := runSeedAccountPhase(ctx, "rich-state", accounts, cfg.Concurrency, progress, func(ctx context.Context, account int) error {
|
||||
return seedRichAccountState(ctx, cfg, manifest, dataset, journal, targets, key, publicKey, account)
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := journal.assertComplete(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := &SeedResult{PrivateMessages: len(dataset.PrivateEdges), Groups: len(dataset.Groups), RichStateAccounts: dataset.Config.Accounts}
|
||||
for _, group := range dataset.Groups {
|
||||
result.InvitedMembers += len(group.MemberAccounts) - 1
|
||||
result.GroupMessages += group.HistoryMessages
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func seedPrimaryTargets(manifest *Manifest, accounts int) ([]SessionRecord, error) {
|
||||
targets := primaryTargets(manifest.Sessions)
|
||||
if len(targets) < accounts {
|
||||
return nil, fmt.Errorf("dataset requires %d primary accounts, manifest has account range 0..%d", accounts, len(targets)-1)
|
||||
}
|
||||
targets = targets[:accounts]
|
||||
for account, target := range targets {
|
||||
if target.AccountIndex != account || target.UserID <= 0 || target.AccessHash == 0 || target.SessionFile == "" {
|
||||
return nil, fmt.Errorf("manifest has no complete primary session for account %d", account)
|
||||
}
|
||||
}
|
||||
return targets, nil
|
||||
}
|
||||
|
||||
func seedPrivateMessages(
|
||||
ctx context.Context,
|
||||
cfg SeedConfig,
|
||||
manifest *Manifest,
|
||||
dataset *Dataset,
|
||||
journal *seedJournal,
|
||||
targets []SessionRecord,
|
||||
key [32]byte,
|
||||
publicKey *rsa.PublicKey,
|
||||
account int,
|
||||
) error {
|
||||
cursor := journal.privateCursor(account)
|
||||
if cursor == dataset.Config.PrivateFanout {
|
||||
return nil
|
||||
}
|
||||
start := account * dataset.Config.PrivateFanout
|
||||
edges := dataset.PrivateEdges[start : start+dataset.Config.PrivateFanout]
|
||||
return withAuthorizedSeedSession(ctx, cfg, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
|
||||
for i := cursor; i < len(edges); i++ {
|
||||
edge := edges[i]
|
||||
target := targets[edge.RecipientAccount]
|
||||
_, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
|
||||
return raw.MessagesSendMessage(rpcCtx, &tg.MessagesSendMessageRequest{
|
||||
Peer: &tg.InputPeerUser{UserID: target.UserID, AccessHash: target.AccessHash},
|
||||
Message: edge.Marker, RandomID: edge.RandomID,
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("messages.sendMessage edge %d: %w", i, err)
|
||||
}
|
||||
}
|
||||
return journal.setPrivateCursor(account, len(edges))
|
||||
})
|
||||
}
|
||||
|
||||
func seedGroup(
|
||||
ctx context.Context,
|
||||
cfg SeedConfig,
|
||||
dataset *Dataset,
|
||||
journal *seedJournal,
|
||||
targets []SessionRecord,
|
||||
raw *tg.Client,
|
||||
position int,
|
||||
) error {
|
||||
group := dataset.Groups[position]
|
||||
groupState := journal.group(position)
|
||||
if groupState.ChannelID == 0 && groupState.CreatePending {
|
||||
channel, found, err := reconcilePendingChannelCreate(ctx, cfg.OperationTimeout, raw, group)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found {
|
||||
if err := journal.commitChannel(position, channel.ID, channel.AccessHash); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err := journal.clearCreatePending(position); err != nil {
|
||||
return err
|
||||
}
|
||||
groupState = journal.group(position)
|
||||
}
|
||||
if groupState.ChannelID == 0 {
|
||||
if err := journal.beginCreate(position); err != nil {
|
||||
return err
|
||||
}
|
||||
updates, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
|
||||
return raw.ChannelsCreateChannel(rpcCtx, &tg.ChannelsCreateChannelRequest{
|
||||
Megagroup: true, Title: group.Title, About: group.About,
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("channels.createChannel pending reconciliation: %w", err)
|
||||
}
|
||||
channel, err := createdChannelFromUpdates(updates, group.Title)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := journal.commitChannel(position, channel.ID, channel.AccessHash); err != nil {
|
||||
return err
|
||||
}
|
||||
groupState = journal.group(position)
|
||||
}
|
||||
|
||||
invitees := groupInvitees(group)
|
||||
for groupState.InviteCursor < len(invitees) {
|
||||
if groupState.InvitePendingEnd > groupState.InviteCursor {
|
||||
if err := reconcilePendingInvites(ctx, cfg.OperationTimeout, raw, groupState, invitees, targets); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := journal.commitInvite(position, groupState.InvitePendingEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
groupState = journal.group(position)
|
||||
continue
|
||||
}
|
||||
end := min(groupState.InviteCursor+maxSeedInviteBatch, len(invitees))
|
||||
if err := journal.beginInvite(position, end); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inviteAccounts(ctx, cfg.OperationTimeout, raw, groupState, invitees[groupState.InviteCursor:end], targets); err != nil {
|
||||
return fmt.Errorf("channels.inviteToChannel pending reconciliation: %w", err)
|
||||
}
|
||||
if err := journal.commitInvite(position, end); err != nil {
|
||||
return err
|
||||
}
|
||||
groupState = journal.group(position)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reconcilePendingChannelCreate(ctx context.Context, timeout time.Duration, raw *tg.Client, group DatasetGroup) (*tg.Channel, bool, error) {
|
||||
rpcCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
dialogs, err := raw.MessagesGetDialogs(rpcCtx, &tg.MessagesGetDialogsRequest{
|
||||
OffsetPeer: &tg.InputPeerEmpty{}, Limit: 500,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("reconcile create with messages.getDialogs: %w", err)
|
||||
}
|
||||
var chats []tg.ChatClass
|
||||
switch value := dialogs.(type) {
|
||||
case *tg.MessagesDialogs:
|
||||
chats = value.Chats
|
||||
case *tg.MessagesDialogsSlice:
|
||||
chats = value.Chats
|
||||
case *tg.MessagesDialogsNotModified:
|
||||
return nil, false, errors.New("reconcile create unexpectedly returned dialogsNotModified")
|
||||
default:
|
||||
return nil, false, fmt.Errorf("reconcile create messages.getDialogs returned %T", dialogs)
|
||||
}
|
||||
matches := make([]*tg.Channel, 0, 1)
|
||||
for _, chat := range chats {
|
||||
channel, ok := chat.(*tg.Channel)
|
||||
if !ok || channel.Title != group.Title || !channel.Megagroup {
|
||||
continue
|
||||
}
|
||||
if channel.ID <= 0 || channel.AccessHash == 0 {
|
||||
return nil, false, fmt.Errorf("reconciled channel %q has incomplete identity", group.Title)
|
||||
}
|
||||
matches = append(matches, channel)
|
||||
}
|
||||
if len(matches) > 1 {
|
||||
return nil, false, fmt.Errorf("ambiguous create produced %d channels named %q", len(matches), group.Title)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
return matches[0], true, nil
|
||||
}
|
||||
|
||||
func createdChannelFromUpdates(updates tg.UpdatesClass, title string) (*tg.Channel, error) {
|
||||
var chats []tg.ChatClass
|
||||
switch value := updates.(type) {
|
||||
case *tg.Updates:
|
||||
chats = value.Chats
|
||||
case *tg.UpdatesCombined:
|
||||
chats = value.Chats
|
||||
default:
|
||||
return nil, fmt.Errorf("channels.createChannel returned %T", updates)
|
||||
}
|
||||
for _, chat := range chats {
|
||||
channel, ok := chat.(*tg.Channel)
|
||||
if ok && channel.Title == title && channel.Megagroup && channel.ID > 0 && channel.AccessHash != 0 {
|
||||
return channel, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("channels.createChannel response omitted supergroup %q", title)
|
||||
}
|
||||
|
||||
func groupInvitees(group DatasetGroup) []int {
|
||||
invitees := make([]int, 0, len(group.MemberAccounts)-1)
|
||||
for _, account := range group.MemberAccounts {
|
||||
if account != group.CreatorAccount {
|
||||
invitees = append(invitees, account)
|
||||
}
|
||||
}
|
||||
return invitees
|
||||
}
|
||||
|
||||
func inviteAccounts(
|
||||
ctx context.Context,
|
||||
timeout time.Duration,
|
||||
raw *tg.Client,
|
||||
groupState DatasetSeedGroupState,
|
||||
accounts []int,
|
||||
targets []SessionRecord,
|
||||
) error {
|
||||
if len(accounts) == 0 || len(accounts) > maxSeedInviteBatch {
|
||||
return fmt.Errorf("invalid invite batch size %d", len(accounts))
|
||||
}
|
||||
users := make([]tg.InputUserClass, 0, len(accounts))
|
||||
for _, account := range accounts {
|
||||
target := targets[account]
|
||||
users = append(users, &tg.InputUser{UserID: target.UserID, AccessHash: target.AccessHash})
|
||||
}
|
||||
result, err := rpcWithFloodWaitRetry(ctx, timeout, func(rpcCtx context.Context) (*tg.MessagesInvitedUsers, error) {
|
||||
return raw.ChannelsInviteToChannel(rpcCtx, &tg.ChannelsInviteToChannelRequest{
|
||||
Channel: &tg.InputChannel{ChannelID: groupState.ChannelID, AccessHash: groupState.AccessHash},
|
||||
Users: users,
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(result.MissingInvitees) != 0 {
|
||||
return fmt.Errorf("server reported %d missing_invitees", len(result.MissingInvitees))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reconcilePendingInvites(
|
||||
ctx context.Context,
|
||||
timeout time.Duration,
|
||||
raw *tg.Client,
|
||||
groupState DatasetSeedGroupState,
|
||||
invitees []int,
|
||||
targets []SessionRecord,
|
||||
) error {
|
||||
pending := invitees[groupState.InviteCursor:groupState.InvitePendingEnd]
|
||||
missing := make([]int, 0, len(pending))
|
||||
for _, account := range pending {
|
||||
target := targets[account]
|
||||
rpcCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
_, err := raw.ChannelsGetParticipant(rpcCtx, &tg.ChannelsGetParticipantRequest{
|
||||
Channel: &tg.InputChannel{ChannelID: groupState.ChannelID, AccessHash: groupState.AccessHash},
|
||||
Participant: &tg.InputPeerUser{UserID: target.UserID, AccessHash: target.AccessHash},
|
||||
})
|
||||
cancel()
|
||||
switch {
|
||||
case err == nil:
|
||||
case tgerr.Is(err, "USER_NOT_PARTICIPANT"):
|
||||
missing = append(missing, account)
|
||||
default:
|
||||
return fmt.Errorf("channels.getParticipant account %d: %w", account, err)
|
||||
}
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
return nil
|
||||
}
|
||||
return inviteAccounts(ctx, timeout, raw, groupState, missing, targets)
|
||||
}
|
||||
|
||||
type datasetHistoryTask struct {
|
||||
GroupPosition int
|
||||
MessageIndex int
|
||||
}
|
||||
|
||||
func datasetHistoryTasks(dataset *Dataset) [][]datasetHistoryTask {
|
||||
tasks := make([][]datasetHistoryTask, dataset.Config.Accounts)
|
||||
for position, group := range dataset.Groups {
|
||||
for message := 0; message < group.HistoryMessages; message++ {
|
||||
account := group.MemberAccounts[message%len(group.MemberAccounts)]
|
||||
tasks[account] = append(tasks[account], datasetHistoryTask{GroupPosition: position, MessageIndex: message})
|
||||
}
|
||||
}
|
||||
return tasks
|
||||
}
|
||||
|
||||
func seedGroupHistory(
|
||||
ctx context.Context,
|
||||
cfg SeedConfig,
|
||||
manifest *Manifest,
|
||||
dataset *Dataset,
|
||||
journal *seedJournal,
|
||||
targets []SessionRecord,
|
||||
key [32]byte,
|
||||
publicKey *rsa.PublicKey,
|
||||
account int,
|
||||
tasks []datasetHistoryTask,
|
||||
) error {
|
||||
cursor := journal.historyCursor(account)
|
||||
if cursor == len(tasks) {
|
||||
return nil
|
||||
}
|
||||
return withAuthorizedSeedSession(ctx, cfg, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
|
||||
for i := cursor; i < len(tasks); i++ {
|
||||
task := tasks[i]
|
||||
group := dataset.Groups[task.GroupPosition]
|
||||
groupState := journal.group(task.GroupPosition)
|
||||
if groupState.ChannelID == 0 || groupState.InviteCursor != len(group.MemberAccounts)-1 {
|
||||
return fmt.Errorf("group %d is not fully seeded", group.Index)
|
||||
}
|
||||
_, err := rpcWithFloodWaitRetry(ctx, cfg.OperationTimeout, func(rpcCtx context.Context) (tg.UpdatesClass, error) {
|
||||
return raw.MessagesSendMessage(rpcCtx, &tg.MessagesSendMessageRequest{
|
||||
Peer: &tg.InputPeerChannel{ChannelID: groupState.ChannelID, AccessHash: groupState.AccessHash},
|
||||
Message: fmt.Sprintf("[%s group %04d message %04d sender %04d]", dataset.RunID, group.Index, task.MessageIndex+1, account),
|
||||
RandomID: stableDatasetID(dataset.Config.Seed, "group-history", dataset.Config.Accounts, group.Index, task.MessageIndex),
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("messages.sendMessage group %d message %d: %w", group.Index, task.MessageIndex, err)
|
||||
}
|
||||
}
|
||||
return journal.setHistoryCursor(account, len(tasks))
|
||||
})
|
||||
}
|
||||
|
||||
func withAuthorizedSeedSession(
|
||||
ctx context.Context,
|
||||
cfg SeedConfig,
|
||||
manifest *Manifest,
|
||||
record SessionRecord,
|
||||
key [32]byte,
|
||||
publicKey *rsa.PublicKey,
|
||||
work func(context.Context, *tg.Client) error,
|
||||
) error {
|
||||
storage := &EncryptedFileStorage{Path: resolveSessionPath(cfg.ManifestPath, record), Key: key}
|
||||
client, err := newClient(manifest.Endpoint, publicKey, storage, clientHooks{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return client.Run(ctx, func(ctx context.Context) error {
|
||||
statusCtx, cancel := context.WithTimeout(ctx, cfg.OperationTimeout)
|
||||
status, err := client.Auth().Status(statusCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("authorization status: %w", err)
|
||||
}
|
||||
if !status.Authorized || status.User == nil || status.User.ID != record.UserID {
|
||||
return fmt.Errorf("session account %d is not authorized as expected user", record.AccountIndex)
|
||||
}
|
||||
return work(ctx, tg.NewClient(client))
|
||||
})
|
||||
}
|
||||
|
||||
type seedAccountResult struct {
|
||||
account int
|
||||
err error
|
||||
}
|
||||
|
||||
func runSeedAccountPhase(
|
||||
ctx context.Context,
|
||||
phase string,
|
||||
accounts []int,
|
||||
concurrency int,
|
||||
progress func(SeedEvent),
|
||||
work func(context.Context, int) error,
|
||||
) error {
|
||||
phaseCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
jobs := make(chan int)
|
||||
results := make(chan seedAccountResult, len(accounts))
|
||||
workers := min(concurrency, len(accounts))
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for account := range jobs {
|
||||
err := work(phaseCtx, account)
|
||||
results <- seedAccountResult{account: account, err: err}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
defer close(jobs)
|
||||
for _, account := range accounts {
|
||||
select {
|
||||
case jobs <- account:
|
||||
case <-phaseCtx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(results)
|
||||
}()
|
||||
completed := 0
|
||||
var firstErr error
|
||||
for result := range results {
|
||||
if result.err == nil {
|
||||
completed++
|
||||
} else if firstErr == nil {
|
||||
firstErr = fmt.Errorf("%s account %d: %w", phase, result.account, result.err)
|
||||
cancel()
|
||||
}
|
||||
if progress != nil {
|
||||
progress(SeedEvent{Phase: phase, Completed: completed, Total: len(accounts), Account: result.account, Err: result.err})
|
||||
}
|
||||
}
|
||||
if firstErr != nil {
|
||||
return firstErr
|
||||
}
|
||||
if completed != len(accounts) {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
return fmt.Errorf("%s completed %d/%d accounts", phase, completed, len(accounts))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type seedJournal struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
dataset *Dataset
|
||||
state *DatasetSeedState
|
||||
}
|
||||
|
||||
func (j *seedJournal) persist() error {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return WriteDatasetSeedState(j.path, j.dataset, j.state)
|
||||
}
|
||||
|
||||
func (j *seedJournal) privateCursor(account int) int {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.state.PrivateSentByAccount[account]
|
||||
}
|
||||
|
||||
func (j *seedJournal) historyCursor(account int) int {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.state.HistorySentByAccount[account]
|
||||
}
|
||||
|
||||
func (j *seedJournal) richStateComplete(account int) bool {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return len(j.state.RichStateByAccount) == j.dataset.Config.Accounts && j.state.RichStateByAccount[account]
|
||||
}
|
||||
|
||||
func (j *seedJournal) enableRichState() error {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
if len(j.state.RichStateByAccount) == j.dataset.Config.Accounts {
|
||||
return nil
|
||||
}
|
||||
if len(j.state.RichStateByAccount) != 0 {
|
||||
return errors.New("seed rich-state journal has invalid dimensions")
|
||||
}
|
||||
j.state.RichStateByAccount = make([]bool, j.dataset.Config.Accounts)
|
||||
return WriteDatasetSeedState(j.path, j.dataset, j.state)
|
||||
}
|
||||
|
||||
func (j *seedJournal) group(position int) DatasetSeedGroupState {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.state.Groups[position]
|
||||
}
|
||||
|
||||
func (j *seedJournal) setPrivateCursor(account, cursor int) error {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
old := j.state.PrivateSentByAccount[account]
|
||||
j.state.PrivateSentByAccount[account] = cursor
|
||||
if err := WriteDatasetSeedState(j.path, j.dataset, j.state); err != nil {
|
||||
j.state.PrivateSentByAccount[account] = old
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j *seedJournal) setHistoryCursor(account, cursor int) error {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
old := j.state.HistorySentByAccount[account]
|
||||
j.state.HistorySentByAccount[account] = cursor
|
||||
if err := WriteDatasetSeedState(j.path, j.dataset, j.state); err != nil {
|
||||
j.state.HistorySentByAccount[account] = old
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j *seedJournal) setRichStateComplete(account int) error {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
if len(j.state.RichStateByAccount) != j.dataset.Config.Accounts {
|
||||
return errors.New("seed rich-state journal is not enabled")
|
||||
}
|
||||
old := j.state.RichStateByAccount[account]
|
||||
j.state.RichStateByAccount[account] = true
|
||||
if err := WriteDatasetSeedState(j.path, j.dataset, j.state); err != nil {
|
||||
j.state.RichStateByAccount[account] = old
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j *seedJournal) updateGroup(position int, update func(*DatasetSeedGroupState)) error {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
old := j.state.Groups[position]
|
||||
update(&j.state.Groups[position])
|
||||
if err := WriteDatasetSeedState(j.path, j.dataset, j.state); err != nil {
|
||||
j.state.Groups[position] = old
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j *seedJournal) beginCreate(position int) error {
|
||||
return j.updateGroup(position, func(state *DatasetSeedGroupState) { state.CreatePending = true })
|
||||
}
|
||||
|
||||
func (j *seedJournal) clearCreatePending(position int) error {
|
||||
return j.updateGroup(position, func(state *DatasetSeedGroupState) { state.CreatePending = false })
|
||||
}
|
||||
|
||||
func (j *seedJournal) commitChannel(position int, channelID, accessHash int64) error {
|
||||
if channelID <= 0 || accessHash == 0 {
|
||||
return errors.New("cannot commit incomplete channel identity")
|
||||
}
|
||||
return j.updateGroup(position, func(state *DatasetSeedGroupState) {
|
||||
state.ChannelID = channelID
|
||||
state.AccessHash = accessHash
|
||||
state.CreatePending = false
|
||||
})
|
||||
}
|
||||
|
||||
func (j *seedJournal) beginInvite(position, end int) error {
|
||||
return j.updateGroup(position, func(state *DatasetSeedGroupState) { state.InvitePendingEnd = end })
|
||||
}
|
||||
|
||||
func (j *seedJournal) commitInvite(position, end int) error {
|
||||
return j.updateGroup(position, func(state *DatasetSeedGroupState) {
|
||||
state.InviteCursor = end
|
||||
state.InvitePendingEnd = end
|
||||
})
|
||||
}
|
||||
|
||||
func (j *seedJournal) assertComplete() error {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
if err := j.state.Validate(j.dataset); err != nil {
|
||||
return err
|
||||
}
|
||||
historyCounts := datasetHistoryTaskCounts(j.dataset)
|
||||
for account := 0; account < j.dataset.Config.Accounts; account++ {
|
||||
if j.state.PrivateSentByAccount[account] != j.dataset.Config.PrivateFanout || j.state.HistorySentByAccount[account] != historyCounts[account] {
|
||||
return fmt.Errorf("account %d seed is incomplete", account)
|
||||
}
|
||||
if len(j.state.RichStateByAccount) != 0 && !j.state.RichStateByAccount[account] {
|
||||
return fmt.Errorf("account %d rich state is incomplete", account)
|
||||
}
|
||||
}
|
||||
for position, group := range j.dataset.Groups {
|
||||
state := j.state.Groups[position]
|
||||
if state.ChannelID == 0 || state.CreatePending || state.InviteCursor != len(group.MemberAccounts)-1 || state.InvitePendingEnd != state.InviteCursor {
|
||||
return fmt.Errorf("group %d seed is incomplete", group.Index)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
177
internal/loadharness/seed_test.go
Normal file
177
internal/loadharness/seed_test.go
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/iamxvbaba/td/tg"
|
||||
)
|
||||
|
||||
func TestSeedPrimaryTargetsSelectsPrimaryAndRejectsGap(t *testing.T) {
|
||||
manifest := &Manifest{Sessions: []SessionRecord{
|
||||
{Index: 2, AccountIndex: 0, DeviceIndex: 1, SessionFile: "extra", UserID: 10, AccessHash: 100},
|
||||
{Index: 0, AccountIndex: 0, DeviceIndex: 0, SessionFile: "primary-0", UserID: 10, AccessHash: 100},
|
||||
{Index: 1, AccountIndex: 1, DeviceIndex: 0, SessionFile: "primary-1", UserID: 11, AccessHash: 101},
|
||||
}}
|
||||
targets, err := seedPrimaryTargets(manifest, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if targets[0].SessionFile != "primary-0" || targets[1].SessionFile != "primary-1" {
|
||||
t.Fatalf("primary targets = %+v", targets)
|
||||
}
|
||||
manifest.Sessions = append(manifest.Sessions, SessionRecord{
|
||||
Index: 3, AccountIndex: 2, DeviceIndex: 0, SessionFile: "primary-2", UserID: 12, AccessHash: 102,
|
||||
})
|
||||
if targets, err := seedPrimaryTargets(manifest, 2); err != nil || len(targets) != 2 {
|
||||
t.Fatalf("manifest superset targets=%d err=%v", len(targets), err)
|
||||
}
|
||||
manifest.Sessions = manifest.Sessions[:2]
|
||||
if _, err := seedPrimaryTargets(manifest, 2); err == nil {
|
||||
t.Fatal("manifest account gap passed validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanRichAccountStateUsesDeterministicPrivateAndGroupPeers(t *testing.T) {
|
||||
dataset, _, _ := snapshotFixture(t)
|
||||
plan, err := planRichAccountState(dataset, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.PinnedPeerAccount != 1 || plan.ReadPeerAccount != 3 || plan.ReadGroupPosition != 0 || plan.DraftMarker == "" {
|
||||
t.Fatalf("rich plan = %+v", plan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSeededRichDialogs(t *testing.T) {
|
||||
dataset, seedState, targets := snapshotFixture(t)
|
||||
seedState.RichStateByAccount = make([]bool, dataset.Config.Accounts)
|
||||
seedState.RichStateByAccount[0] = true
|
||||
plan, err := planRichAccountState(dataset, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dialogs := []ClientDialogState{
|
||||
{PeerType: "user", PeerID: targets[plan.PinnedPeerAccount].UserID, Pinned: true, HasDraft: true, DraftText: plan.DraftMarker},
|
||||
{PeerType: "user", PeerID: targets[plan.ReadPeerAccount].UserID, TopMessage: 8, ReadInboxMaxID: 8},
|
||||
{PeerType: "channel", PeerID: seedState.Groups[plan.ReadGroupPosition].ChannelID, TopMessage: 9, ReadInboxMaxID: 9},
|
||||
}
|
||||
if err := validateSeededRichDialogs(dataset, seedState, targets, 0, dialogs, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dialogs[0].DraftText = "wrong"
|
||||
if err := validateSeededRichDialogs(dataset, seedState, targets, 0, dialogs, true); err == nil {
|
||||
t.Fatal("wrong draft marker passed rich-state validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreatedChannelFromUpdates(t *testing.T) {
|
||||
channel := &tg.Channel{ID: 41, AccessHash: 42, Title: "target", Megagroup: true}
|
||||
got, err := createdChannelFromUpdates(&tg.Updates{Chats: []tg.ChatClass{
|
||||
&tg.Chat{ID: 1, Title: "other"}, channel,
|
||||
}}, "target")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != channel {
|
||||
t.Fatalf("created channel = %#v, want target", got)
|
||||
}
|
||||
if _, err := createdChannelFromUpdates(&tg.UpdateShort{}, "target"); err == nil {
|
||||
t.Fatal("unexpected create response passed validation")
|
||||
}
|
||||
if _, err := createdChannelFromUpdates(&tg.UpdatesCombined{Chats: []tg.ChatClass{
|
||||
&tg.Channel{ID: 41, AccessHash: 42, Title: "target"},
|
||||
}}, "target"); err == nil {
|
||||
t.Fatal("broadcast/non-megagroup response passed validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatasetHistoryTasksRotateSenders(t *testing.T) {
|
||||
cfg := DefaultDatasetConfig(20)
|
||||
cfg.HotGroups, cfg.MediumGroups, cfg.HeavyGroups = 0, 0, 0
|
||||
cfg.SmallGroups, cfg.SmallMembers, cfg.SmallHistory = 1, 4, 9
|
||||
dataset, err := PlanDataset(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tasks := datasetHistoryTasks(dataset)
|
||||
wantCounts := datasetHistoryTaskCounts(dataset)
|
||||
gotCounts := make([]int, len(tasks))
|
||||
for account := range tasks {
|
||||
gotCounts[account] = len(tasks[account])
|
||||
for _, task := range tasks[account] {
|
||||
group := dataset.Groups[task.GroupPosition]
|
||||
if got := group.MemberAccounts[task.MessageIndex%len(group.MemberAccounts)]; got != account {
|
||||
t.Fatalf("message %d sender = %d, task account %d", task.MessageIndex, got, account)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(gotCounts, wantCounts) {
|
||||
t.Fatalf("history task counts = %v, want %v", gotCounts, wantCounts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedJournalPersistsPendingBoundaries(t *testing.T) {
|
||||
dataset, err := PlanDataset(DefaultDatasetConfig(20))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err := NewDatasetSeedState(dataset)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "seed-state.json")
|
||||
journal := &seedJournal{path: path, dataset: dataset, state: state}
|
||||
if err := journal.persist(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := journal.beginCreate(0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := LoadDatasetSeedState(path, dataset)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !loaded.Groups[0].CreatePending || loaded.Groups[0].ChannelID != 0 {
|
||||
t.Fatalf("pending create state = %+v", loaded.Groups[0])
|
||||
}
|
||||
if err := journal.commitChannel(0, 101, 202); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := journal.beginInvite(0, 7); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err = LoadDatasetSeedState(path, dataset)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loaded.Groups[0].InviteCursor != 0 || loaded.Groups[0].InvitePendingEnd != 7 {
|
||||
t.Fatalf("pending invite state = %+v", loaded.Groups[0])
|
||||
}
|
||||
if err := journal.commitInvite(0, 7); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err = LoadDatasetSeedState(path, dataset)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loaded.Groups[0].InviteCursor != 7 || loaded.Groups[0].InvitePendingEnd != 7 {
|
||||
t.Fatalf("committed invite state = %+v", loaded.Groups[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSeedAccountPhaseStopsAfterFailure(t *testing.T) {
|
||||
want := errors.New("stop")
|
||||
err := runSeedAccountPhase(context.Background(), "test", []int{0, 1, 2}, 1, nil, func(_ context.Context, account int) error {
|
||||
if account == 1 {
|
||||
return want
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("phase error = %v, want wrapped stop", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -16,43 +16,88 @@ import (
|
|||
const maxServerMetricsBytes = 4 << 20
|
||||
|
||||
var selectedServerMetrics = map[string]struct{}{
|
||||
"telesrv_mtproto_raw_connections": {},
|
||||
"telesrv_mtproto_sessions": {},
|
||||
"telesrv_mtproto_logical_sessions": {},
|
||||
"telesrv_mtproto_logical_outbox_frames": {},
|
||||
"telesrv_mtproto_logical_outbox_bytes": {},
|
||||
"telesrv_mtproto_logical_outbox_acked_frames_total": {},
|
||||
"telesrv_mtproto_logical_outbox_acked_bytes_total": {},
|
||||
"telesrv_mtproto_logical_outbox_retained_seconds_count": {},
|
||||
"telesrv_mtproto_logical_outbox_retained_seconds_sum": {},
|
||||
"telesrv_mtproto_pending_push_bytes": {},
|
||||
"telesrv_mtproto_inbound_rpc_tasks": {},
|
||||
"telesrv_mtproto_inbound_rpc_bytes": {},
|
||||
"telesrv_mtproto_inbound_frame_bytes": {},
|
||||
"telesrv_mtproto_outbound_tracked_bytes": {},
|
||||
"telesrv_mtproto_outbound_write_bytes": {},
|
||||
"telesrv_mtproto_rpc_execution_owners": {},
|
||||
"telesrv_mtproto_rpc_execution_reserved_entries": {},
|
||||
"telesrv_mtproto_rpc_execution_receipts": {},
|
||||
"telesrv_mtproto_rpc_execution_receipt_budget_bytes": {},
|
||||
"telesrv_mtproto_rpc_execution_subscribers": {},
|
||||
"telesrv_mtproto_rpc_result_inner_bytes_total": {},
|
||||
"telesrv_mtproto_rpc_result_wire_bytes_total": {},
|
||||
"telesrv_mtproto_rpc_result_delivered_bytes_total": {},
|
||||
"telesrv_go_goroutines": {},
|
||||
"telesrv_go_heap_alloc_bytes": {},
|
||||
"telesrv_go_heap_inuse_bytes": {},
|
||||
"telesrv_go_heap_objects": {},
|
||||
"telesrv_go_sys_bytes": {},
|
||||
"telesrv_postgres_pool_connections": {},
|
||||
"telesrv_postgres_pool_acquire_wait_seconds": {},
|
||||
"telesrv_postgres_pool_empty_acquire_count": {},
|
||||
"telesrv_postgres_pool_canceled_acquire_count": {},
|
||||
"telesrv_redis_pool_connections": {},
|
||||
"telesrv_redis_pool_pending_requests": {},
|
||||
"telesrv_redis_pool_timeouts": {},
|
||||
"telesrv_redis_pool_wait_seconds": {},
|
||||
"telesrv_metrics_dropped_observations_total": {},
|
||||
"telesrv_mtproto_raw_connections": {},
|
||||
"telesrv_mtproto_connections_active": {},
|
||||
"telesrv_mtproto_sessions": {},
|
||||
"telesrv_mtproto_logical_sessions": {},
|
||||
"telesrv_mtproto_logical_outbox_frames": {},
|
||||
"telesrv_mtproto_logical_outbox_bytes": {},
|
||||
"telesrv_mtproto_logical_outbox_acked_frames_total": {},
|
||||
"telesrv_mtproto_logical_outbox_acked_bytes_total": {},
|
||||
"telesrv_mtproto_logical_outbox_retained_seconds_count": {},
|
||||
"telesrv_mtproto_logical_outbox_retained_seconds_sum": {},
|
||||
"telesrv_mtproto_pending_push_bytes": {},
|
||||
"telesrv_mtproto_inbound_rpc_tasks": {},
|
||||
"telesrv_mtproto_inbound_rpc_bytes": {},
|
||||
"telesrv_mtproto_rpc_delivery_hook_workers": {},
|
||||
"telesrv_mtproto_rpc_delivery_hook_capacity": {},
|
||||
"telesrv_mtproto_rpc_delivery_hook_reserved": {},
|
||||
"telesrv_mtproto_rpc_delivery_hook_queued": {},
|
||||
"telesrv_mtproto_rpc_delivery_hook_running": {},
|
||||
"telesrv_mtproto_rpc_delivery_hook_completed_total": {},
|
||||
"telesrv_mtproto_rpc_delivery_hook_rejected_total": {},
|
||||
"telesrv_mtproto_rpc_delivery_hook_panics_total": {},
|
||||
"telesrv_mtproto_rpc_delivery_hook_duration_seconds_total": {},
|
||||
"telesrv_mtproto_inbound_frame_bytes": {},
|
||||
"telesrv_mtproto_outbound_tracked_bytes": {},
|
||||
"telesrv_mtproto_outbound_write_bytes": {},
|
||||
"telesrv_mtproto_rpc_execution_owners": {},
|
||||
"telesrv_mtproto_rpc_execution_reserved_entries": {},
|
||||
"telesrv_mtproto_rpc_execution_receipts": {},
|
||||
"telesrv_mtproto_rpc_execution_receipt_budget_bytes": {},
|
||||
"telesrv_mtproto_rpc_execution_subscribers": {},
|
||||
"telesrv_mtproto_rpc_result_inner_bytes_total": {},
|
||||
"telesrv_mtproto_rpc_result_wire_bytes_total": {},
|
||||
"telesrv_mtproto_rpc_result_delivered_total": {},
|
||||
"telesrv_mtproto_rpc_result_delivered_bytes_total": {},
|
||||
"telesrv_go_goroutines": {},
|
||||
"telesrv_process_cpu_seconds": {},
|
||||
"telesrv_go_scheduler_busy_seconds": {},
|
||||
"telesrv_go_gc_cycles": {},
|
||||
"telesrv_go_gc_pause_seconds": {},
|
||||
"telesrv_go_heap_alloc_bytes": {},
|
||||
"telesrv_go_heap_inuse_bytes": {},
|
||||
"telesrv_go_heap_objects": {},
|
||||
"telesrv_go_stack_inuse_bytes": {},
|
||||
"telesrv_go_sys_bytes": {},
|
||||
"telesrv_postgres_pool_connections": {},
|
||||
"telesrv_postgres_pool_acquire_count": {},
|
||||
"telesrv_postgres_pool_acquire_wait_seconds": {},
|
||||
"telesrv_postgres_pool_empty_acquire_count": {},
|
||||
"telesrv_postgres_pool_canceled_acquire_count": {},
|
||||
"telesrv_postgres_pool_max_connections": {},
|
||||
"telesrv_redis_pool_connections": {},
|
||||
"telesrv_redis_pool_hits": {},
|
||||
"telesrv_redis_pool_misses": {},
|
||||
"telesrv_redis_pool_pending_requests": {},
|
||||
"telesrv_redis_pool_timeouts": {},
|
||||
"telesrv_redis_pool_wait_count": {},
|
||||
"telesrv_redis_pool_wait_seconds": {},
|
||||
"telesrv_rpc_db_queries_total": {},
|
||||
"telesrv_rpc_db_errors_total": {},
|
||||
"telesrv_rpc_db_time_seconds_sum": {},
|
||||
"telesrv_rpc_db_time_seconds_count": {},
|
||||
"telesrv_channel_difference_cache_entries": {},
|
||||
"telesrv_channel_difference_cache_weight_bytes": {},
|
||||
"telesrv_channel_difference_cache_hits": {},
|
||||
"telesrv_channel_difference_cache_misses": {},
|
||||
"telesrv_channel_difference_cache_loads": {},
|
||||
"telesrv_channel_difference_cache_load_errors": {},
|
||||
"telesrv_bootstrap_ready_batches_total": {},
|
||||
"telesrv_bootstrap_ready_selectors_total": {},
|
||||
"telesrv_bootstrap_ready_pending": {},
|
||||
"telesrv_active_channel_ids_cache_total": {},
|
||||
"telesrv_active_channel_ids_batches_total": {},
|
||||
"telesrv_active_channel_ids_selectors_total": {},
|
||||
"telesrv_active_channel_ids_rows_total": {},
|
||||
"telesrv_active_channel_ids_pending": {},
|
||||
"telesrv_presence_last_seen_batches_total": {},
|
||||
"telesrv_presence_last_seen_updates_total": {},
|
||||
"telesrv_presence_last_seen_submitted_total": {},
|
||||
"telesrv_presence_last_seen_pending": {},
|
||||
"telesrv_presence_last_seen_overflow_total": {},
|
||||
"telesrv_presence_last_seen_drain_dropped_total": {},
|
||||
"telesrv_metrics_dropped_observations_total": {},
|
||||
}
|
||||
|
||||
type serverMetricsClient struct {
|
||||
|
|
@ -113,10 +158,31 @@ func (c *serverMetricsClient) scrape(ctx context.Context) (map[string]float64, e
|
|||
}
|
||||
// Reports need bounded, comparable capacity signals, not an unbounded copy
|
||||
// of Prometheus label series. Aggregate every selected family into one
|
||||
// key so method/encoding cardinality can never starve later gauges (the
|
||||
// endpoint orders counters before gauges). The source /metrics endpoint
|
||||
// retains full labels for detailed diagnosis.
|
||||
// key. Response-byte and DB-work families additionally retain only their
|
||||
// code-owned method label; bounded pool/session families retain their state
|
||||
// label. This supports attribution without copying auth/session/user
|
||||
// cardinality.
|
||||
values[name] += value
|
||||
if isPerMethodOutcomeServerMetric(name) {
|
||||
method, methodOK := prometheusLabelValue(fields[0], "method")
|
||||
outcome, outcomeOK := prometheusLabelValue(fields[0], "outcome")
|
||||
if methodOK && outcomeOK {
|
||||
values[name+`{method="`+method+`",outcome="`+outcome+`"}`] += value
|
||||
}
|
||||
} else if isPerMethodServerMetric(name) {
|
||||
if method, ok := prometheusLabelValue(fields[0], "method"); ok {
|
||||
values[name+`{method="`+method+`"}`] += value
|
||||
}
|
||||
} else if isOutcomeServerMetric(name) {
|
||||
if outcome, ok := prometheusLabelValue(fields[0], "outcome"); ok {
|
||||
values[name+`{outcome="`+outcome+`"}`] += value
|
||||
}
|
||||
}
|
||||
if isStateServerMetric(name) {
|
||||
if state, ok := prometheusLabelValue(fields[0], "state"); ok {
|
||||
values[name+`{state="`+state+`"}`] += value
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := reader.Err(); err != nil {
|
||||
c.errors.Add(1)
|
||||
|
|
@ -126,6 +192,112 @@ func (c *serverMetricsClient) scrape(ctx context.Context) (map[string]float64, e
|
|||
return values, nil
|
||||
}
|
||||
|
||||
// waitForPresenceLastSeenSettlement waits until every expected lifecycle event
|
||||
// has reached the server-owned batch queue and all accepted work has drained.
|
||||
// It is report-only synchronization: it does not participate in RPC success or
|
||||
// alter the server's presence semantics.
|
||||
func (c *serverMetricsClient) waitForPresenceLastSeenSettlement(
|
||||
ctx context.Context,
|
||||
baselineSubmitted float64,
|
||||
expectedSubmitted uint64,
|
||||
timeout time.Duration,
|
||||
) (map[string]float64, error) {
|
||||
if c == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if timeout <= 0 {
|
||||
timeout = 15 * time.Second
|
||||
}
|
||||
deadline := time.NewTimer(timeout)
|
||||
defer deadline.Stop()
|
||||
poll := time.NewTicker(100 * time.Millisecond)
|
||||
defer poll.Stop()
|
||||
var last map[string]float64
|
||||
for {
|
||||
sample, err := c.scrape(ctx)
|
||||
if err != nil {
|
||||
return last, err
|
||||
}
|
||||
last = sample
|
||||
submitted := metricValue(sample, "telesrv_presence_last_seen_submitted_total") - baselineSubmitted
|
||||
pending := metricValue(sample, "telesrv_presence_last_seen_pending")
|
||||
bootstrapPending := metricValue(sample, "telesrv_bootstrap_ready_pending")
|
||||
activeChannelIDsPending := metricValue(sample, "telesrv_active_channel_ids_pending")
|
||||
if submitted >= float64(expectedSubmitted) && pending == 0 && bootstrapPending == 0 && activeChannelIDsPending == 0 {
|
||||
return sample, nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return last, ctx.Err()
|
||||
case <-deadline.C:
|
||||
return last, fmt.Errorf("startup settlement timeout: presence submitted=%.0f expected=%d pending=%.0f bootstrap_pending=%.0f active_channel_ids_pending=%.0f", submitted, expectedSubmitted, pending, bootstrapPending, activeChannelIDsPending)
|
||||
case <-poll.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isOutcomeServerMetric(name string) bool {
|
||||
switch name {
|
||||
case "telesrv_presence_last_seen_batches_total", "telesrv_presence_last_seen_updates_total",
|
||||
"telesrv_bootstrap_ready_batches_total", "telesrv_bootstrap_ready_selectors_total",
|
||||
"telesrv_active_channel_ids_cache_total", "telesrv_active_channel_ids_batches_total",
|
||||
"telesrv_active_channel_ids_selectors_total":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isPerMethodServerMetric(name string) bool {
|
||||
switch name {
|
||||
case "telesrv_mtproto_rpc_result_inner_bytes_total",
|
||||
"telesrv_mtproto_rpc_result_wire_bytes_total",
|
||||
"telesrv_rpc_db_queries_total",
|
||||
"telesrv_rpc_db_errors_total",
|
||||
"telesrv_rpc_db_time_seconds_sum",
|
||||
"telesrv_rpc_db_time_seconds_count":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isPerMethodOutcomeServerMetric(name string) bool {
|
||||
switch name {
|
||||
case "telesrv_mtproto_rpc_result_delivered_total", "telesrv_mtproto_rpc_result_delivered_bytes_total":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isStateServerMetric(name string) bool {
|
||||
switch name {
|
||||
case "telesrv_mtproto_sessions", "telesrv_mtproto_logical_sessions",
|
||||
"telesrv_postgres_pool_connections", "telesrv_redis_pool_connections":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func prometheusLabelValue(series, label string) (string, bool) {
|
||||
needle := label + `="`
|
||||
start := strings.Index(series, needle)
|
||||
if start < 0 {
|
||||
return "", false
|
||||
}
|
||||
start += len(needle)
|
||||
end := start
|
||||
for end < len(series) {
|
||||
if series[end] == '"' && (end == start || series[end-1] != '\\') {
|
||||
return series[start:end], true
|
||||
}
|
||||
end++
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func (c *serverMetricsClient) successes() uint64 {
|
||||
if c == nil {
|
||||
return 0
|
||||
|
|
|
|||
|
|
@ -5,7 +5,9 @@ import (
|
|||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestServerMetricsScrapeSelectsBoundedCapacitySignals(t *testing.T) {
|
||||
|
|
@ -16,6 +18,24 @@ func TestServerMetricsScrapeSelectsBoundedCapacitySignals(t *testing.T) {
|
|||
for i := 0; i < 256; i++ {
|
||||
fmt.Fprintf(w, "telesrv_mtproto_rpc_result_wire_bytes_total{method=%q} 1\n", fmt.Sprintf("method-%d", i))
|
||||
}
|
||||
fmt.Fprintln(w, `telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="ok"} 7`)
|
||||
fmt.Fprintln(w, `telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="edge_overload"} 3`)
|
||||
fmt.Fprintln(w, `telesrv_mtproto_rpc_result_delivered_bytes_total{method="users.getUsers",outcome="ok"} 700`)
|
||||
fmt.Fprintln(w, `telesrv_mtproto_rpc_result_delivered_bytes_total{method="users.getUsers",outcome="edge_overload"} 300`)
|
||||
fmt.Fprintln(w, `telesrv_presence_last_seen_batches_total{outcome="ok"} 17`)
|
||||
fmt.Fprintln(w, `telesrv_presence_last_seen_batches_total{outcome="error"} 2`)
|
||||
fmt.Fprintln(w, `telesrv_presence_last_seen_updates_total{outcome="ok"} 900`)
|
||||
fmt.Fprintln(w, `telesrv_presence_last_seen_updates_total{outcome="error"} 23`)
|
||||
fmt.Fprintln(w, `telesrv_presence_last_seen_submitted_total 923`)
|
||||
fmt.Fprintln(w, `telesrv_presence_last_seen_pending 11`)
|
||||
fmt.Fprintln(w, `telesrv_presence_last_seen_overflow_total 1`)
|
||||
fmt.Fprintln(w, `telesrv_presence_last_seen_drain_dropped_total 4`)
|
||||
fmt.Fprintln(w, `telesrv_bootstrap_ready_batches_total{outcome="ok"} 10`)
|
||||
fmt.Fprintln(w, `telesrv_bootstrap_ready_batches_total{outcome="error"} 1`)
|
||||
fmt.Fprintln(w, `telesrv_bootstrap_ready_selectors_total{outcome="matched"} 2`)
|
||||
fmt.Fprintln(w, `telesrv_bootstrap_ready_selectors_total{outcome="miss"} 800`)
|
||||
fmt.Fprintln(w, `telesrv_bootstrap_ready_selectors_total{outcome="error"} 3`)
|
||||
fmt.Fprintln(w, `telesrv_bootstrap_ready_pending 1`)
|
||||
fmt.Fprintln(w, `unrelated_high_cardinality{user_id="secret"} 1`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
|
@ -27,7 +47,89 @@ func TestServerMetricsScrapeSelectsBoundedCapacitySignals(t *testing.T) {
|
|||
if values["telesrv_mtproto_raw_connections"] != 500 || values["telesrv_mtproto_sessions"] != 500 || values["telesrv_mtproto_rpc_result_wire_bytes_total"] != 256 {
|
||||
t.Fatalf("values = %#v", values)
|
||||
}
|
||||
if len(values) != 3 || client.successes() != 1 || client.failures() != 0 {
|
||||
if values[`telesrv_mtproto_rpc_result_wire_bytes_total{method="method-17"}`] != 1 {
|
||||
t.Fatalf("method response bytes = %#v", values)
|
||||
}
|
||||
if values[`telesrv_mtproto_sessions{state="active"}`] != 499 || values[`telesrv_mtproto_sessions{state="provisional"}`] != 1 {
|
||||
t.Fatalf("session state values = %#v", values)
|
||||
}
|
||||
if values[`telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="ok"}`] != 7 ||
|
||||
values[`telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="edge_overload"}`] != 3 {
|
||||
t.Fatalf("delivery outcomes = %#v", values)
|
||||
}
|
||||
if values[`telesrv_presence_last_seen_batches_total{outcome="ok"}`] != 17 ||
|
||||
values[`telesrv_presence_last_seen_batches_total{outcome="error"}`] != 2 ||
|
||||
values[`telesrv_presence_last_seen_updates_total{outcome="ok"}`] != 900 ||
|
||||
values[`telesrv_presence_last_seen_updates_total{outcome="error"}`] != 23 ||
|
||||
values["telesrv_presence_last_seen_submitted_total"] != 923 ||
|
||||
values["telesrv_presence_last_seen_pending"] != 11 ||
|
||||
values["telesrv_presence_last_seen_overflow_total"] != 1 ||
|
||||
values["telesrv_presence_last_seen_drain_dropped_total"] != 4 {
|
||||
t.Fatalf("presence batch values = %#v", values)
|
||||
}
|
||||
if values[`telesrv_bootstrap_ready_batches_total{outcome="ok"}`] != 10 ||
|
||||
values[`telesrv_bootstrap_ready_batches_total{outcome="error"}`] != 1 ||
|
||||
values[`telesrv_bootstrap_ready_selectors_total{outcome="matched"}`] != 2 ||
|
||||
values[`telesrv_bootstrap_ready_selectors_total{outcome="miss"}`] != 800 ||
|
||||
values[`telesrv_bootstrap_ready_selectors_total{outcome="error"}`] != 3 ||
|
||||
values["telesrv_bootstrap_ready_pending"] != 1 {
|
||||
t.Fatalf("bootstrap readiness values = %#v", values)
|
||||
}
|
||||
if len(values) != 285 || client.successes() != 1 || client.failures() != 0 {
|
||||
t.Fatalf("bounded values/scrapes = %#v, %d/%d", values, client.successes(), client.failures())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrometheusLabelValue(t *testing.T) {
|
||||
value, ok := prometheusLabelValue(`metric{encoding="gzip",method="messages.getDialogs",outcome="ok"}`, "method")
|
||||
if !ok || value != "messages.getDialogs" {
|
||||
t.Fatalf("method = %q, %v", value, ok)
|
||||
}
|
||||
if _, ok := prometheusLabelValue(`metric{encoding="gzip"}`, "method"); ok {
|
||||
t.Fatal("missing method label was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerMetricsWaitsForPresenceLastSeenSettlement(t *testing.T) {
|
||||
var calls atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
call := calls.Add(1)
|
||||
if call < 3 {
|
||||
fmt.Fprintln(w, `telesrv_presence_last_seen_submitted_total 1`)
|
||||
fmt.Fprintln(w, `telesrv_presence_last_seen_pending 1`)
|
||||
fmt.Fprintln(w, `telesrv_bootstrap_ready_pending 1`)
|
||||
return
|
||||
}
|
||||
fmt.Fprintln(w, `telesrv_presence_last_seen_submitted_total 2`)
|
||||
fmt.Fprintln(w, `telesrv_presence_last_seen_pending 0`)
|
||||
fmt.Fprintln(w, `telesrv_bootstrap_ready_pending 0`)
|
||||
}))
|
||||
defer server.Close()
|
||||
client := newServerMetricsClient(server.URL)
|
||||
values, err := client.waitForPresenceLastSeenSettlement(context.Background(), 0, 2, time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
if values["telesrv_presence_last_seen_submitted_total"] != 2 || values["telesrv_presence_last_seen_pending"] != 0 ||
|
||||
values["telesrv_bootstrap_ready_pending"] != 0 {
|
||||
t.Fatalf("settled values = %#v", values)
|
||||
}
|
||||
if calls.Load() != 3 {
|
||||
t.Fatalf("scrape calls = %d, want 3", calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricValueDoesNotDoubleCountAggregateAndStateBreakdown(t *testing.T) {
|
||||
values := map[string]float64{
|
||||
"telesrv_mtproto_logical_sessions": 254,
|
||||
`telesrv_mtproto_logical_sessions{state="retained"}`: 254,
|
||||
`telesrv_mtproto_logical_sessions{state="offline"}`: 0,
|
||||
}
|
||||
if got := metricValue(values, "telesrv_mtproto_logical_sessions"); got != 254 {
|
||||
t.Fatalf("logical sessions = %v, want aggregate 254", got)
|
||||
}
|
||||
delete(values, "telesrv_mtproto_logical_sessions")
|
||||
if got := metricValue(values, "telesrv_mtproto_logical_sessions"); got != 254 {
|
||||
t.Fatalf("legacy labeled-only logical sessions = %v, want 254", got)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
723
internal/loadharness/snapshot.go
Normal file
723
internal/loadharness/snapshot.go
Normal file
|
|
@ -0,0 +1,723 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/iamxvbaba/td/tg"
|
||||
)
|
||||
|
||||
const ClientStateVersion = 1
|
||||
|
||||
type ClientUpdateState struct {
|
||||
Pts int `json:"pts"`
|
||||
Qts int `json:"qts"`
|
||||
Date int `json:"date"`
|
||||
Seq int `json:"seq"`
|
||||
UnreadCount int `json:"unread_count"`
|
||||
}
|
||||
|
||||
type ClientDialogState struct {
|
||||
PeerType string `json:"peer_type"`
|
||||
PeerID int64 `json:"peer_id"`
|
||||
AccessHash int64 `json:"access_hash"`
|
||||
TopMessage int `json:"top_message"`
|
||||
TopMessageDate int `json:"top_message_date"`
|
||||
Pts int `json:"pts,omitempty"`
|
||||
HasPts bool `json:"has_pts,omitempty"`
|
||||
ReadInboxMaxID int `json:"read_inbox_max_id"`
|
||||
ReadOutboxMaxID int `json:"read_outbox_max_id"`
|
||||
UnreadCount int `json:"unread_count"`
|
||||
UnreadMentions int `json:"unread_mentions"`
|
||||
UnreadReactions int `json:"unread_reactions"`
|
||||
Pinned bool `json:"pinned,omitempty"`
|
||||
HasDraft bool `json:"has_draft,omitempty"`
|
||||
DraftText string `json:"draft_text,omitempty"`
|
||||
DatasetExpected bool `json:"dataset_expected,omitempty"`
|
||||
}
|
||||
|
||||
type ClientAccountState struct {
|
||||
AccountIndex int `json:"account_index"`
|
||||
UserID int64 `json:"user_id"`
|
||||
State ClientUpdateState `json:"state"`
|
||||
Dialogs []ClientDialogState `json:"dialogs"`
|
||||
}
|
||||
|
||||
type ClientState struct {
|
||||
Version int `json:"version"`
|
||||
DatasetSHA256 string `json:"dataset_sha256"`
|
||||
SeedIdentitySHA string `json:"seed_identity_sha256"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Accounts []ClientAccountState `json:"accounts"`
|
||||
}
|
||||
|
||||
type SnapshotConfig struct {
|
||||
ManifestPath string
|
||||
SessionKeyPath string
|
||||
RSAKeyOverride string
|
||||
DatasetPath string
|
||||
SeedStatePath string
|
||||
ClientStatePath string
|
||||
Concurrency int
|
||||
OperationTimeout time.Duration
|
||||
}
|
||||
|
||||
type SnapshotEvent struct {
|
||||
Completed int
|
||||
Total int
|
||||
Account int
|
||||
Resumed bool
|
||||
Err error
|
||||
}
|
||||
|
||||
type SnapshotResult struct {
|
||||
Accounts int
|
||||
Dialogs int
|
||||
Channels int
|
||||
}
|
||||
|
||||
func (c SnapshotConfig) validate() error {
|
||||
if c.ManifestPath == "" || c.SessionKeyPath == "" || c.DatasetPath == "" || c.SeedStatePath == "" || c.ClientStatePath == "" {
|
||||
return errors.New("manifest, session-key, dataset, seed-state and client-state paths are required")
|
||||
}
|
||||
if c.Concurrency <= 0 || c.Concurrency > 64 {
|
||||
return errors.New("snapshot concurrency must be between 1 and 64")
|
||||
}
|
||||
if c.OperationTimeout <= 0 {
|
||||
return errors.New("snapshot operation timeout must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SnapshotClientState establishes the old client cursors that later offline
|
||||
// mutations must advance from. Dialogs are read through public paginated RPCs;
|
||||
// per-account part files make a 1,000-account snapshot safely resumable without
|
||||
// rewriting the growing aggregate after every account.
|
||||
func SnapshotClientState(ctx context.Context, cfg SnapshotConfig, progress func(SnapshotEvent)) (*SnapshotResult, error) {
|
||||
if err := cfg.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
manifest, err := LoadManifest(cfg.ManifestPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dataset, err := LoadDataset(cfg.DatasetPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targets, err := seedPrimaryTargets(manifest, dataset.Config.Accounts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seedState, err := LoadDatasetSeedState(cfg.SeedStatePath, dataset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seedJournal := &seedJournal{dataset: dataset, state: seedState}
|
||||
if err := seedJournal.assertComplete(); err != nil {
|
||||
return nil, fmt.Errorf("snapshot requires a complete seed: %w", err)
|
||||
}
|
||||
seedIdentity, err := seedIdentitySHA256(seedState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if existing, loadErr := LoadClientState(cfg.ClientStatePath); loadErr == nil {
|
||||
if err := existing.Validate(dataset, seedState, targets); err != nil {
|
||||
return nil, fmt.Errorf("existing client state does not match requested dataset: %w", err)
|
||||
}
|
||||
return clientStateResult(existing), nil
|
||||
} else if !os.IsNotExist(loadErr) {
|
||||
return nil, loadErr
|
||||
}
|
||||
key, err := LoadSessionKey(cfg.SessionKeyPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
publicKey, err := loadManifestPublicKey(cfg.ManifestPath, manifest.Endpoint, cfg.RSAKeyOverride)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
accounts := make([]int, dataset.Config.Accounts)
|
||||
for account := range accounts {
|
||||
accounts[account] = account
|
||||
}
|
||||
resumed := make([]bool, len(accounts))
|
||||
for _, account := range accounts {
|
||||
if _, err := loadClientStatePart(clientStatePartPath(cfg.ClientStatePath, account), dataset, seedState, targets, account); err == nil {
|
||||
resumed[account] = true
|
||||
} else if !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
completed := 0
|
||||
if err := runSeedAccountPhase(ctx, "snapshot", accounts, cfg.Concurrency, func(event SeedEvent) {
|
||||
if event.Err == nil {
|
||||
completed++
|
||||
}
|
||||
if progress != nil {
|
||||
progress(SnapshotEvent{Completed: completed, Total: len(accounts), Account: event.Account, Resumed: resumed[event.Account], Err: event.Err})
|
||||
}
|
||||
}, func(ctx context.Context, account int) error {
|
||||
if resumed[account] {
|
||||
return nil
|
||||
}
|
||||
state, err := snapshotAccount(ctx, cfg, manifest, dataset, seedState, targets, key, publicKey, account)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeClientStatePart(clientStatePartPath(cfg.ClientStatePath, account), dataset.PlanSHA256, seedIdentity, state)
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clientState := &ClientState{
|
||||
Version: ClientStateVersion, DatasetSHA256: dataset.PlanSHA256, SeedIdentitySHA: seedIdentity, CreatedAt: time.Now().UTC(),
|
||||
Accounts: make([]ClientAccountState, 0, len(accounts)),
|
||||
}
|
||||
for _, account := range accounts {
|
||||
state, err := loadClientStatePart(clientStatePartPath(cfg.ClientStatePath, account), dataset, seedState, targets, account)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clientState.Accounts = append(clientState.Accounts, *state)
|
||||
}
|
||||
if err := clientState.Validate(dataset, seedState, targets); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := WriteClientState(cfg.ClientStatePath, clientState); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return clientStateResult(clientState), nil
|
||||
}
|
||||
|
||||
func snapshotAccount(
|
||||
ctx context.Context,
|
||||
cfg SnapshotConfig,
|
||||
manifest *Manifest,
|
||||
dataset *Dataset,
|
||||
seedState *DatasetSeedState,
|
||||
targets []SessionRecord,
|
||||
key [32]byte,
|
||||
publicKey *rsa.PublicKey,
|
||||
account int,
|
||||
) (*ClientAccountState, error) {
|
||||
var result *ClientAccountState
|
||||
err := withAuthorizedSeedSession(ctx, SeedConfig{
|
||||
ManifestPath: cfg.ManifestPath, OperationTimeout: cfg.OperationTimeout,
|
||||
}, manifest, targets[account], key, publicKey, func(ctx context.Context, raw *tg.Client) error {
|
||||
dialogs, err := snapshotDialogs(ctx, cfg.OperationTimeout, raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
expected := expectedDatasetPeers(dataset, seedState, targets, account)
|
||||
for i := range dialogs {
|
||||
_, dialogs[i].DatasetExpected = expected[clientPeerKey{typ: dialogs[i].PeerType, id: dialogs[i].PeerID}]
|
||||
if dialogs[i].DatasetExpected {
|
||||
delete(expected, clientPeerKey{typ: dialogs[i].PeerType, id: dialogs[i].PeerID})
|
||||
}
|
||||
}
|
||||
if len(expected) != 0 {
|
||||
missing := make([]string, 0, min(len(expected), 5))
|
||||
for peer := range expected {
|
||||
missing = append(missing, fmt.Sprintf("%s:%d", peer.typ, peer.id))
|
||||
if len(missing) == 5 {
|
||||
break
|
||||
}
|
||||
}
|
||||
sort.Strings(missing)
|
||||
return fmt.Errorf("messages.getDialogs omitted %d expected dataset peers (sample %s)", len(expected), strings.Join(missing, ","))
|
||||
}
|
||||
if err := validateSeededRichDialogs(dataset, seedState, targets, account, dialogs, true); err != nil {
|
||||
return fmt.Errorf("rich dialog state: %w", err)
|
||||
}
|
||||
stateCtx, cancel := context.WithTimeout(ctx, cfg.OperationTimeout)
|
||||
state, err := raw.UpdatesGetState(stateCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("updates.getState: %w", err)
|
||||
}
|
||||
result = &ClientAccountState{
|
||||
AccountIndex: account, UserID: targets[account].UserID,
|
||||
State: ClientUpdateState{Pts: state.Pts, Qts: state.Qts, Date: state.Date, Seq: state.Seq, UnreadCount: state.UnreadCount},
|
||||
Dialogs: dialogs,
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err == nil && result == nil {
|
||||
return nil, errors.New("snapshot session ended without producing account state")
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
|
||||
type clientPeerKey struct {
|
||||
typ string
|
||||
id int64
|
||||
}
|
||||
|
||||
func snapshotDialogs(ctx context.Context, timeout time.Duration, raw *tg.Client) ([]ClientDialogState, error) {
|
||||
dialogs, _, err := snapshotDialogsObserved(ctx, timeout, raw, snapshotPaginationProfile, nil)
|
||||
return dialogs, err
|
||||
}
|
||||
|
||||
func snapshotDialogsObserved(
|
||||
ctx context.Context,
|
||||
timeout time.Duration,
|
||||
raw *tg.Client,
|
||||
profile dialogPaginationProfile,
|
||||
observe func(string, time.Time, error),
|
||||
) ([]ClientDialogState, StartupDialogsCounts, error) {
|
||||
if profile.FirstLimit <= 0 || profile.SubsequentLimit <= 0 {
|
||||
return nil, StartupDialogsCounts{}, errors.New("invalid dialogs pagination profile")
|
||||
}
|
||||
dialogsByPeer := make(map[clientPeerKey]ClientDialogState)
|
||||
counts := StartupDialogsCounts{}
|
||||
start := time.Now()
|
||||
pinnedCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
pinned, err := raw.MessagesGetPinnedDialogs(pinnedCtx, 0)
|
||||
cancel()
|
||||
if observe != nil {
|
||||
observe("messages.getPinnedDialogs", start, err)
|
||||
}
|
||||
counts.PinnedCalls++
|
||||
if err != nil {
|
||||
return nil, counts, fmt.Errorf("messages.getPinnedDialogs: %w", err)
|
||||
}
|
||||
if _, err := mergeDialogPage(dialogsByPeer, pinned.Dialogs, pinned.Messages, pinned.Chats, pinned.Users, true); err != nil {
|
||||
return nil, counts, err
|
||||
}
|
||||
pinnedPeers := make(map[clientPeerKey]struct{}, len(dialogsByPeer))
|
||||
for peer := range dialogsByPeer {
|
||||
pinnedPeers[peer] = struct{}{}
|
||||
}
|
||||
|
||||
request := &tg.MessagesGetDialogsRequest{ExcludePinned: true, OffsetPeer: &tg.InputPeerEmpty{}, Limit: profile.limit(0)}
|
||||
for page := 0; page < 100; page++ {
|
||||
request.Limit = profile.limit(page)
|
||||
start := time.Now()
|
||||
rpcCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
response, err := raw.MessagesGetDialogs(rpcCtx, request)
|
||||
cancel()
|
||||
if observe != nil {
|
||||
observe("messages.getDialogs", start, err)
|
||||
if page == 0 {
|
||||
observe("messages.getDialogs.first", start, err)
|
||||
} else {
|
||||
observe("messages.getDialogs.next", start, err)
|
||||
}
|
||||
}
|
||||
counts.Calls++
|
||||
if err != nil {
|
||||
return nil, counts, fmt.Errorf("messages.getDialogs page %d: %w", page+1, err)
|
||||
}
|
||||
var pageDialogs []tg.DialogClass
|
||||
var messages []tg.MessageClass
|
||||
var chats []tg.ChatClass
|
||||
var users []tg.UserClass
|
||||
complete := false
|
||||
switch value := response.(type) {
|
||||
case *tg.MessagesDialogs:
|
||||
if observe != nil {
|
||||
observe("messages.getDialogs.full", start, nil)
|
||||
}
|
||||
counts.Full++
|
||||
pageDialogs, messages, chats, users, complete = value.Dialogs, value.Messages, value.Chats, value.Users, true
|
||||
case *tg.MessagesDialogsSlice:
|
||||
if observe != nil {
|
||||
observe("messages.getDialogs.slice", start, nil)
|
||||
}
|
||||
counts.Slice++
|
||||
pageDialogs, messages, chats, users = value.Dialogs, value.Messages, value.Chats, value.Users
|
||||
case *tg.MessagesDialogsNotModified:
|
||||
return nil, counts, errors.New("messages.getDialogs with hash=0 returned dialogsNotModified")
|
||||
default:
|
||||
return nil, counts, fmt.Errorf("messages.getDialogs returned %T", response)
|
||||
}
|
||||
last, overlap, err := mergeDialogPageKnownOverlap(dialogsByPeer, pageDialogs, messages, chats, users, false, pinnedPeers)
|
||||
counts.PinnedOverlap += overlap
|
||||
if err != nil {
|
||||
return nil, counts, fmt.Errorf("messages.getDialogs page %d: %w", page+1, err)
|
||||
}
|
||||
// A dialogsSlice is explicitly non-final. The server may enforce a
|
||||
// smaller per-page cap than the client-requested limit (TDesktop asks
|
||||
// for 500 after its first page while telesrv currently returns at most
|
||||
// 100). Only the full constructor or an empty slice proves completion;
|
||||
// treating a short slice as EOF truncates the real TDesktop workload.
|
||||
if dialogsPaginationDone(complete, len(pageDialogs)) {
|
||||
break
|
||||
}
|
||||
if last.TopMessage <= 0 || last.TopMessageDate <= 0 {
|
||||
return nil, counts, fmt.Errorf("messages.getDialogs page %d has no usable offset", page+1)
|
||||
}
|
||||
request.OffsetDate = last.TopMessageDate
|
||||
request.OffsetID = last.TopMessage
|
||||
request.OffsetPeer = clientDialogInputPeer(last)
|
||||
if request.OffsetPeer == nil {
|
||||
return nil, counts, fmt.Errorf("messages.getDialogs page %d has invalid offset peer", page+1)
|
||||
}
|
||||
if page == 99 {
|
||||
return nil, counts, errors.New("messages.getDialogs exceeded 100 pages")
|
||||
}
|
||||
}
|
||||
result := make([]ClientDialogState, 0, len(dialogsByPeer))
|
||||
for _, dialog := range dialogsByPeer {
|
||||
result = append(result, dialog)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].PeerType != result[j].PeerType {
|
||||
return result[i].PeerType < result[j].PeerType
|
||||
}
|
||||
return result[i].PeerID < result[j].PeerID
|
||||
})
|
||||
counts.Dialogs = len(result)
|
||||
return result, counts, nil
|
||||
}
|
||||
|
||||
func dialogsPaginationDone(complete bool, pageSize int) bool {
|
||||
return complete || pageSize == 0
|
||||
}
|
||||
|
||||
func mergeDialogPage(
|
||||
destination map[clientPeerKey]ClientDialogState,
|
||||
dialogClasses []tg.DialogClass,
|
||||
messages []tg.MessageClass,
|
||||
chats []tg.ChatClass,
|
||||
users []tg.UserClass,
|
||||
pinnedPage bool,
|
||||
) (ClientDialogState, error) {
|
||||
last, _, err := mergeDialogPageKnownOverlap(destination, dialogClasses, messages, chats, users, pinnedPage, nil)
|
||||
return last, err
|
||||
}
|
||||
|
||||
func mergeDialogPageKnownOverlap(
|
||||
destination map[clientPeerKey]ClientDialogState,
|
||||
dialogClasses []tg.DialogClass,
|
||||
messages []tg.MessageClass,
|
||||
chats []tg.ChatClass,
|
||||
users []tg.UserClass,
|
||||
pinnedPage bool,
|
||||
allowedOverlap map[clientPeerKey]struct{},
|
||||
) (ClientDialogState, int, error) {
|
||||
accessHashes := make(map[clientPeerKey]int64, len(chats)+len(users))
|
||||
for _, chat := range chats {
|
||||
channel, ok := chat.(*tg.Channel)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
hash, ok := channel.GetAccessHash()
|
||||
if ok {
|
||||
accessHashes[clientPeerKey{typ: "channel", id: channel.ID}] = hash
|
||||
}
|
||||
}
|
||||
for _, userClass := range users {
|
||||
user, ok := userClass.(*tg.User)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
hash, ok := user.GetAccessHash()
|
||||
if ok {
|
||||
accessHashes[clientPeerKey{typ: "user", id: user.ID}] = hash
|
||||
}
|
||||
}
|
||||
messageDates := make(map[string]int, len(messages))
|
||||
for _, messageClass := range messages {
|
||||
message, ok := messageClass.AsNotEmpty()
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
peer, ok := clientPeerFromTG(message.GetPeerID())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
messageDates[clientMessageKey(peer, message.GetID())] = message.GetDate()
|
||||
}
|
||||
var last ClientDialogState
|
||||
overlaps := 0
|
||||
for _, dialogClass := range dialogClasses {
|
||||
dialog, ok := dialogClass.(*tg.Dialog)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
peer, ok := clientPeerFromTG(dialog.Peer)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
hash := accessHashes[peer]
|
||||
if hash == 0 {
|
||||
return ClientDialogState{}, overlaps, fmt.Errorf("dialog %s:%d omitted access hash", peer.typ, peer.id)
|
||||
}
|
||||
pts, hasPts := dialog.GetPts()
|
||||
if peer.typ == "channel" && (!hasPts || pts <= 0) {
|
||||
return ClientDialogState{}, overlaps, fmt.Errorf("channel dialog %d omitted pts", peer.id)
|
||||
}
|
||||
date := messageDates[clientMessageKey(peer, dialog.TopMessage)]
|
||||
if dialog.TopMessage <= 0 || date <= 0 {
|
||||
return ClientDialogState{}, overlaps, fmt.Errorf("dialog %s:%d omitted top message payload", peer.typ, peer.id)
|
||||
}
|
||||
state := ClientDialogState{
|
||||
PeerType: peer.typ, PeerID: peer.id, AccessHash: hash,
|
||||
TopMessage: dialog.TopMessage, TopMessageDate: date, Pts: pts, HasPts: hasPts,
|
||||
ReadInboxMaxID: dialog.ReadInboxMaxID, ReadOutboxMaxID: dialog.ReadOutboxMaxID,
|
||||
UnreadCount: dialog.UnreadCount, UnreadMentions: dialog.UnreadMentionsCount,
|
||||
UnreadReactions: dialog.UnreadReactionsCount, Pinned: pinnedPage || dialog.Pinned,
|
||||
}
|
||||
if draftClass, ok := dialog.GetDraft(); ok {
|
||||
if draft, ok := draftClass.(*tg.DraftMessage); ok && draft.Message != "" {
|
||||
state.HasDraft, state.DraftText = true, draft.Message
|
||||
}
|
||||
}
|
||||
if existing, exists := destination[peer]; exists {
|
||||
if _, allowed := allowedOverlap[peer]; !allowed || !existing.Pinned || !state.Pinned || existing.TopMessage != state.TopMessage || existing.AccessHash != state.AccessHash {
|
||||
return ClientDialogState{}, overlaps, fmt.Errorf("duplicate dialog %s:%d", peer.typ, peer.id)
|
||||
}
|
||||
delete(allowedOverlap, peer)
|
||||
if !state.HasDraft && existing.HasDraft {
|
||||
state.HasDraft, state.DraftText = existing.HasDraft, existing.DraftText
|
||||
}
|
||||
overlaps++
|
||||
}
|
||||
destination[peer] = state
|
||||
last = state
|
||||
}
|
||||
return last, overlaps, nil
|
||||
}
|
||||
|
||||
func clientPeerFromTG(peer tg.PeerClass) (clientPeerKey, bool) {
|
||||
switch value := peer.(type) {
|
||||
case *tg.PeerUser:
|
||||
return clientPeerKey{typ: "user", id: value.UserID}, value.UserID > 0
|
||||
case *tg.PeerChannel:
|
||||
return clientPeerKey{typ: "channel", id: value.ChannelID}, value.ChannelID > 0
|
||||
default:
|
||||
return clientPeerKey{}, false
|
||||
}
|
||||
}
|
||||
|
||||
func clientMessageKey(peer clientPeerKey, messageID int) string {
|
||||
return fmt.Sprintf("%s:%d:%d", peer.typ, peer.id, messageID)
|
||||
}
|
||||
|
||||
func clientDialogInputPeer(dialog ClientDialogState) tg.InputPeerClass {
|
||||
switch dialog.PeerType {
|
||||
case "user":
|
||||
return &tg.InputPeerUser{UserID: dialog.PeerID, AccessHash: dialog.AccessHash}
|
||||
case "channel":
|
||||
return &tg.InputPeerChannel{ChannelID: dialog.PeerID, AccessHash: dialog.AccessHash}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func expectedDatasetPeers(dataset *Dataset, seedState *DatasetSeedState, targets []SessionRecord, account int) map[clientPeerKey]struct{} {
|
||||
expected := make(map[clientPeerKey]struct{})
|
||||
for _, edge := range dataset.PrivateEdges {
|
||||
switch account {
|
||||
case edge.SenderAccount:
|
||||
expected[clientPeerKey{typ: "user", id: targets[edge.RecipientAccount].UserID}] = struct{}{}
|
||||
case edge.RecipientAccount:
|
||||
expected[clientPeerKey{typ: "user", id: targets[edge.SenderAccount].UserID}] = struct{}{}
|
||||
}
|
||||
}
|
||||
for position, group := range dataset.Groups {
|
||||
memberIndex := sort.SearchInts(group.MemberAccounts, account)
|
||||
if memberIndex < len(group.MemberAccounts) && group.MemberAccounts[memberIndex] == account {
|
||||
expected[clientPeerKey{typ: "channel", id: seedState.Groups[position].ChannelID}] = struct{}{}
|
||||
}
|
||||
}
|
||||
return expected
|
||||
}
|
||||
|
||||
func clientStatePartPath(clientStatePath string, account int) string {
|
||||
return filepath.Join(clientStatePath+".parts", fmt.Sprintf("account-%04d.json", account))
|
||||
}
|
||||
|
||||
type clientStatePart struct {
|
||||
Version int `json:"version"`
|
||||
DatasetSHA256 string `json:"dataset_sha256"`
|
||||
SeedIdentitySHA string `json:"seed_identity_sha256"`
|
||||
Account ClientAccountState `json:"account"`
|
||||
}
|
||||
|
||||
func writeClientStatePart(path, datasetSHA, seedIdentity string, account *ClientAccountState) error {
|
||||
if account == nil {
|
||||
return errors.New("cannot write nil client account state")
|
||||
}
|
||||
part := clientStatePart{Version: ClientStateVersion, DatasetSHA256: datasetSHA, SeedIdentitySHA: seedIdentity, Account: *account}
|
||||
data, err := json.MarshalIndent(part, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeFileAtomic(path, append(data, '\n'), 0o600)
|
||||
}
|
||||
|
||||
func loadClientStatePart(path string, dataset *Dataset, seedState *DatasetSeedState, targets []SessionRecord, account int) (*ClientAccountState, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var part clientStatePart
|
||||
decoder := json.NewDecoder(strings.NewReader(string(data)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&part); err != nil {
|
||||
return nil, fmt.Errorf("decode client state part: %w", err)
|
||||
}
|
||||
seedIdentity, err := seedIdentitySHA256(seedState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target := targets[account]
|
||||
if part.Version != ClientStateVersion || part.DatasetSHA256 != dataset.PlanSHA256 || part.SeedIdentitySHA != seedIdentity || part.Account.AccountIndex != target.AccountIndex || part.Account.UserID != target.UserID {
|
||||
return nil, errors.New("client state part does not match dataset or account")
|
||||
}
|
||||
if err := validateClientAccountState(&part.Account); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := validateExpectedDatasetPeers(dataset, seedState, targets, &part.Account); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &part.Account, nil
|
||||
}
|
||||
|
||||
func (s *ClientState) Validate(dataset *Dataset, seedState *DatasetSeedState, targets []SessionRecord) error {
|
||||
if s == nil || s.Version != ClientStateVersion || dataset == nil || s.DatasetSHA256 != dataset.PlanSHA256 {
|
||||
return errors.New("client state does not match dataset")
|
||||
}
|
||||
seedIdentity, err := seedIdentitySHA256(seedState)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s.SeedIdentitySHA != seedIdentity {
|
||||
return errors.New("client state does not match seeded channel identities")
|
||||
}
|
||||
if len(s.Accounts) != dataset.Config.Accounts || len(targets) != dataset.Config.Accounts {
|
||||
return errors.New("client state account count does not match dataset")
|
||||
}
|
||||
for account := range s.Accounts {
|
||||
if s.Accounts[account].AccountIndex != account || s.Accounts[account].UserID != targets[account].UserID {
|
||||
return fmt.Errorf("client state account %d has wrong identity", account)
|
||||
}
|
||||
if err := validateClientAccountState(&s.Accounts[account]); err != nil {
|
||||
return fmt.Errorf("client state account %d: %w", account, err)
|
||||
}
|
||||
if err := validateExpectedDatasetPeers(dataset, seedState, targets, &s.Accounts[account]); err != nil {
|
||||
return fmt.Errorf("client state account %d: %w", account, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateExpectedDatasetPeers(dataset *Dataset, seedState *DatasetSeedState, targets []SessionRecord, account *ClientAccountState) error {
|
||||
expected := expectedDatasetPeers(dataset, seedState, targets, account.AccountIndex)
|
||||
for _, dialog := range account.Dialogs {
|
||||
peer := clientPeerKey{typ: dialog.PeerType, id: dialog.PeerID}
|
||||
_, shouldBeExpected := expected[peer]
|
||||
if dialog.DatasetExpected != shouldBeExpected {
|
||||
return fmt.Errorf("dialog %s:%d has incorrect dataset_expected marker", peer.typ, peer.id)
|
||||
}
|
||||
if shouldBeExpected {
|
||||
delete(expected, peer)
|
||||
}
|
||||
}
|
||||
if len(expected) != 0 {
|
||||
return fmt.Errorf("client state omits %d expected dataset peers", len(expected))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func seedIdentitySHA256(state *DatasetSeedState) (string, error) {
|
||||
if state == nil {
|
||||
return "", errors.New("nil seed state")
|
||||
}
|
||||
type identity struct {
|
||||
GroupIndex int `json:"group_index"`
|
||||
ChannelID int64 `json:"channel_id"`
|
||||
AccessHash int64 `json:"access_hash"`
|
||||
}
|
||||
identities := make([]identity, len(state.Groups))
|
||||
for i, group := range state.Groups {
|
||||
if group.ChannelID <= 0 || group.AccessHash == 0 {
|
||||
return "", fmt.Errorf("group %d has incomplete identity", group.GroupIndex)
|
||||
}
|
||||
identities[i] = identity{GroupIndex: group.GroupIndex, ChannelID: group.ChannelID, AccessHash: group.AccessHash}
|
||||
}
|
||||
data, err := json.Marshal(identities)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return fmt.Sprintf("%x", sum[:]), nil
|
||||
}
|
||||
|
||||
func validateClientAccountState(account *ClientAccountState) error {
|
||||
if account == nil || account.AccountIndex < 0 || account.UserID <= 0 || account.State.Pts < 0 || account.State.Qts < 0 || account.State.Date <= 0 || account.State.Seq < 0 {
|
||||
return errors.New("invalid account state")
|
||||
}
|
||||
seen := make(map[clientPeerKey]struct{}, len(account.Dialogs))
|
||||
for _, dialog := range account.Dialogs {
|
||||
peer := clientPeerKey{typ: dialog.PeerType, id: dialog.PeerID}
|
||||
if (peer.typ != "user" && peer.typ != "channel") || peer.id <= 0 || dialog.AccessHash == 0 || dialog.TopMessage <= 0 || dialog.TopMessageDate <= 0 {
|
||||
return fmt.Errorf("invalid dialog %s:%d", peer.typ, peer.id)
|
||||
}
|
||||
if peer.typ == "channel" && (!dialog.HasPts || dialog.Pts <= 0) {
|
||||
return fmt.Errorf("invalid channel pts for %d", peer.id)
|
||||
}
|
||||
if dialog.HasDraft != (dialog.DraftText != "") {
|
||||
return fmt.Errorf("invalid draft state for %s:%d", peer.typ, peer.id)
|
||||
}
|
||||
if _, exists := seen[peer]; exists {
|
||||
return fmt.Errorf("duplicate dialog %s:%d", peer.typ, peer.id)
|
||||
}
|
||||
seen[peer] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func WriteClientState(path string, state *ClientState) error {
|
||||
if state == nil || state.Version != ClientStateVersion || state.DatasetSHA256 == "" || state.SeedIdentitySHA == "" {
|
||||
return errors.New("invalid client state")
|
||||
}
|
||||
data, err := json.MarshalIndent(state, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeFileAtomic(path, append(data, '\n'), 0o600)
|
||||
}
|
||||
|
||||
func LoadClientState(path string) (*ClientState, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var state ClientState
|
||||
decoder := json.NewDecoder(strings.NewReader(string(data)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&state); err != nil {
|
||||
return nil, fmt.Errorf("decode client state: %w", err)
|
||||
}
|
||||
if state.Version != ClientStateVersion || state.DatasetSHA256 == "" || state.SeedIdentitySHA == "" {
|
||||
return nil, errors.New("invalid client state")
|
||||
}
|
||||
return &state, nil
|
||||
}
|
||||
|
||||
func clientStateResult(state *ClientState) *SnapshotResult {
|
||||
result := &SnapshotResult{Accounts: len(state.Accounts)}
|
||||
for _, account := range state.Accounts {
|
||||
result.Dialogs += len(account.Dialogs)
|
||||
for _, dialog := range account.Dialogs {
|
||||
if dialog.PeerType == "channel" {
|
||||
result.Channels++
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
189
internal/loadharness/snapshot_test.go
Normal file
189
internal/loadharness/snapshot_test.go
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/iamxvbaba/td/tg"
|
||||
)
|
||||
|
||||
func TestMergeDialogPageCapturesChannelCursorAndOffset(t *testing.T) {
|
||||
user := &tg.User{ID: 11}
|
||||
user.SetAccessHash(111)
|
||||
channel := &tg.Channel{ID: 22, Title: "group", Megagroup: true}
|
||||
channel.SetAccessHash(222)
|
||||
userDialog := &tg.Dialog{Peer: &tg.PeerUser{UserID: 11}, TopMessage: 7}
|
||||
channelDialog := &tg.Dialog{Peer: &tg.PeerChannel{ChannelID: 22}, TopMessage: 8}
|
||||
channelDialog.SetPts(12)
|
||||
destination := make(map[clientPeerKey]ClientDialogState)
|
||||
last, err := mergeDialogPage(destination,
|
||||
[]tg.DialogClass{userDialog, channelDialog},
|
||||
[]tg.MessageClass{
|
||||
&tg.Message{ID: 7, PeerID: &tg.PeerUser{UserID: 11}, Date: 101},
|
||||
&tg.Message{ID: 8, PeerID: &tg.PeerChannel{ChannelID: 22}, Date: 102},
|
||||
},
|
||||
[]tg.ChatClass{channel}, []tg.UserClass{user}, false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(destination) != 2 || last.PeerType != "channel" || last.PeerID != 22 || last.Pts != 12 || !last.HasPts || last.TopMessageDate != 102 {
|
||||
t.Fatalf("merged dialogs = %+v, last = %+v", destination, last)
|
||||
}
|
||||
if _, err := mergeDialogPage(destination,
|
||||
[]tg.DialogClass{channelDialog},
|
||||
[]tg.MessageClass{&tg.Message{ID: 8, PeerID: &tg.PeerChannel{ChannelID: 22}, Date: 102}},
|
||||
[]tg.ChatClass{channel}, nil, false,
|
||||
); err == nil {
|
||||
t.Fatal("duplicate dialog page passed validation")
|
||||
}
|
||||
overlapDestination := make(map[clientPeerKey]ClientDialogState)
|
||||
pinnedDialog := *channelDialog
|
||||
pinnedDialog.Pinned = true
|
||||
if _, err := mergeDialogPage(overlapDestination,
|
||||
[]tg.DialogClass{&pinnedDialog},
|
||||
[]tg.MessageClass{&tg.Message{ID: 8, PeerID: &tg.PeerChannel{ChannelID: 22}, Date: 102}},
|
||||
[]tg.ChatClass{channel}, nil, true,
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allowed := map[clientPeerKey]struct{}{{typ: "channel", id: 22}: {}}
|
||||
if _, overlaps, err := mergeDialogPageKnownOverlap(overlapDestination,
|
||||
[]tg.DialogClass{&pinnedDialog},
|
||||
[]tg.MessageClass{&tg.Message{ID: 8, PeerID: &tg.PeerChannel{ChannelID: 22}, Date: 102}},
|
||||
[]tg.ChatClass{channel}, nil, false, allowed,
|
||||
); err != nil || overlaps != 1 {
|
||||
t.Fatalf("known pinned overlap count=%d err=%v", overlaps, err)
|
||||
}
|
||||
if _, _, err := mergeDialogPageKnownOverlap(overlapDestination,
|
||||
[]tg.DialogClass{&pinnedDialog},
|
||||
[]tg.MessageClass{&tg.Message{ID: 8, PeerID: &tg.PeerChannel{ChannelID: 22}, Date: 102}},
|
||||
[]tg.ChatClass{channel}, nil, false, allowed,
|
||||
); err == nil {
|
||||
t.Fatal("second copy of a consumed pinned overlap passed validation")
|
||||
}
|
||||
channelWithoutPts := &tg.Dialog{Peer: &tg.PeerChannel{ChannelID: 22}, TopMessage: 8}
|
||||
if _, err := mergeDialogPage(make(map[clientPeerKey]ClientDialogState),
|
||||
[]tg.DialogClass{channelWithoutPts},
|
||||
[]tg.MessageClass{&tg.Message{ID: 8, PeerID: &tg.PeerChannel{ChannelID: 22}, Date: 102}},
|
||||
[]tg.ChatClass{channel}, nil, false,
|
||||
); err == nil {
|
||||
t.Fatal("channel dialog without pts passed validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientStateRoundTripLocksSeededChannelIdentity(t *testing.T) {
|
||||
dataset, seedState, targets := snapshotFixture(t)
|
||||
seedIdentity, err := seedIdentitySHA256(seedState)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state := &ClientState{
|
||||
Version: ClientStateVersion, DatasetSHA256: dataset.PlanSHA256, SeedIdentitySHA: seedIdentity,
|
||||
Accounts: make([]ClientAccountState, dataset.Config.Accounts),
|
||||
}
|
||||
for account := range state.Accounts {
|
||||
state.Accounts[account] = ClientAccountState{
|
||||
AccountIndex: account, UserID: targets[account].UserID,
|
||||
State: ClientUpdateState{Pts: account + 1, Date: 100},
|
||||
}
|
||||
expected := expectedDatasetPeers(dataset, seedState, targets, account)
|
||||
for peer := range expected {
|
||||
dialog := ClientDialogState{
|
||||
PeerType: peer.typ, PeerID: peer.id, AccessHash: 99,
|
||||
TopMessage: 1, TopMessageDate: 100, DatasetExpected: true,
|
||||
}
|
||||
if peer.typ == "channel" {
|
||||
dialog.HasPts, dialog.Pts = true, 5
|
||||
}
|
||||
state.Accounts[account].Dialogs = append(state.Accounts[account].Dialogs, dialog)
|
||||
}
|
||||
}
|
||||
if err := state.Validate(dataset, seedState, targets); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "client-state.json")
|
||||
if err := WriteClientState(path, state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("client state mode = %o, want 600", info.Mode().Perm())
|
||||
}
|
||||
loaded, err := LoadClientState(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := loaded.Validate(dataset, seedState, targets); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seedState.Groups[0].ChannelID++
|
||||
if err := loaded.Validate(dataset, seedState, targets); err == nil {
|
||||
t.Fatal("client state accepted different seeded channel identity")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientStateRejectsMissingExpectedPeer(t *testing.T) {
|
||||
dataset, seedState, targets := snapshotFixture(t)
|
||||
expected := expectedDatasetPeers(dataset, seedState, targets, 0)
|
||||
account := ClientAccountState{AccountIndex: 0, UserID: targets[0].UserID, State: ClientUpdateState{Date: 100}}
|
||||
for peer := range expected {
|
||||
dialog := ClientDialogState{PeerType: peer.typ, PeerID: peer.id, AccessHash: 1, TopMessage: 1, TopMessageDate: 1, DatasetExpected: true}
|
||||
if peer.typ == "channel" {
|
||||
dialog.HasPts, dialog.Pts = true, 1
|
||||
}
|
||||
account.Dialogs = append(account.Dialogs, dialog)
|
||||
}
|
||||
account.Dialogs = account.Dialogs[1:]
|
||||
if err := validateExpectedDatasetPeers(dataset, seedState, targets, &account); err == nil {
|
||||
t.Fatal("account with missing expected peer passed validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialogsPaginationOnlyFinishesOnFullOrEmptyResponse(t *testing.T) {
|
||||
if dialogsPaginationDone(false, 100) {
|
||||
t.Fatal("non-empty dialogsSlice was treated as final when the client requested a larger page")
|
||||
}
|
||||
if !dialogsPaginationDone(false, 0) {
|
||||
t.Fatal("empty dialogsSlice did not finish pagination")
|
||||
}
|
||||
if !dialogsPaginationDone(true, 100) {
|
||||
t.Fatal("messages.dialogs full constructor did not finish pagination")
|
||||
}
|
||||
}
|
||||
|
||||
func snapshotFixture(t *testing.T) (*Dataset, *DatasetSeedState, []SessionRecord) {
|
||||
t.Helper()
|
||||
cfg := DatasetConfig{
|
||||
Accounts: 4, Seed: 7, PrivateFanout: 1,
|
||||
HotGroups: 1, HotMembers: 4, HotHistory: 1,
|
||||
}
|
||||
dataset, err := PlanDataset(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seedState, err := NewDatasetSeedState(dataset)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// This fixture models a pre-rich-state journal unless an individual test
|
||||
// explicitly opts into the newer phase.
|
||||
seedState.RichStateByAccount = nil
|
||||
for account := 0; account < cfg.Accounts; account++ {
|
||||
seedState.PrivateSentByAccount[account] = cfg.PrivateFanout
|
||||
}
|
||||
seedState.Groups[0].ChannelID = 500
|
||||
seedState.Groups[0].AccessHash = 600
|
||||
seedState.Groups[0].InviteCursor = 3
|
||||
seedState.Groups[0].InvitePendingEnd = 3
|
||||
seedState.HistorySentByAccount[dataset.Groups[0].MemberAccounts[0]] = 1
|
||||
targets := make([]SessionRecord, cfg.Accounts)
|
||||
for account := range targets {
|
||||
targets[account] = SessionRecord{AccountIndex: account, UserID: int64(100 + account), AccessHash: int64(200 + account), SessionFile: "session"}
|
||||
}
|
||||
return dataset, seedState, targets
|
||||
}
|
||||
1037
internal/loadharness/startup.go
Normal file
1037
internal/loadharness/startup.go
Normal file
File diff suppressed because it is too large
Load diff
67
internal/loadharness/startup_profile.go
Normal file
67
internal/loadharness/startup_profile.go
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/iamxvbaba/td/telegram"
|
||||
)
|
||||
|
||||
const (
|
||||
StartupProfileTDesktopReturningV1 = "tdesktop-cold-returning-v1"
|
||||
StartupProfileTDLibReturningV1 = "tdlib-returning-v1"
|
||||
)
|
||||
|
||||
type dialogPaginationProfile struct {
|
||||
FirstLimit int
|
||||
SubsequentLimit int
|
||||
}
|
||||
|
||||
type startupWorkloadProfile struct {
|
||||
Name string
|
||||
GetStateBeforeDifference bool
|
||||
AccountDifference bool
|
||||
Dialogs dialogPaginationProfile
|
||||
ForceChannelDifference bool
|
||||
}
|
||||
|
||||
func resolveStartupProfile(name string) (startupWorkloadProfile, error) {
|
||||
switch strings.TrimSpace(name) {
|
||||
case "", StartupProfileTDesktopReturningV1:
|
||||
return startupWorkloadProfile{
|
||||
Name: StartupProfileTDesktopReturningV1, GetStateBeforeDifference: true,
|
||||
Dialogs: dialogPaginationProfile{FirstLimit: 20, SubsequentLimit: 500},
|
||||
ForceChannelDifference: true,
|
||||
}, nil
|
||||
case StartupProfileTDLibReturningV1:
|
||||
return startupWorkloadProfile{
|
||||
Name: StartupProfileTDLibReturningV1,
|
||||
AccountDifference: true,
|
||||
Dialogs: dialogPaginationProfile{FirstLimit: 100, SubsequentLimit: 100},
|
||||
ForceChannelDifference: true,
|
||||
}, nil
|
||||
default:
|
||||
return startupWorkloadProfile{}, fmt.Errorf("unknown startup profile %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
func (p dialogPaginationProfile) limit(page int) int {
|
||||
if page == 0 {
|
||||
return p.FirstLimit
|
||||
}
|
||||
return p.SubsequentLimit
|
||||
}
|
||||
|
||||
func (p startupWorkloadProfile) device() telegram.DeviceConfig {
|
||||
if p.Name == StartupProfileTDLibReturningV1 {
|
||||
return telegram.DeviceConfig{
|
||||
DeviceModel: "telesrv-flutter TDLib", SystemVersion: "Android SDK 36", AppVersion: "load-profile-v1",
|
||||
SystemLangCode: "en-US", LangPack: "android", LangCode: "en",
|
||||
Params: telegram.TimezoneParams(time.Local),
|
||||
}
|
||||
}
|
||||
return telegram.DeviceTDesktopWindows()
|
||||
}
|
||||
|
||||
var snapshotPaginationProfile = dialogPaginationProfile{FirstLimit: 100, SubsequentLimit: 100}
|
||||
197
internal/loadharness/startup_test.go
Normal file
197
internal/loadharness/startup_test.go
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/iamxvbaba/td/tg"
|
||||
)
|
||||
|
||||
func TestStartupRampDelay(t *testing.T) {
|
||||
ramp := 30 * time.Second
|
||||
if got := startupRampDelay(ramp, 0, 4); got != 0 {
|
||||
t.Fatalf("first delay = %s", got)
|
||||
}
|
||||
if got := startupRampDelay(ramp, 1, 4); got != 10*time.Second {
|
||||
t.Fatalf("second delay = %s", got)
|
||||
}
|
||||
if got := startupRampDelay(ramp, 3, 4); got != ramp {
|
||||
t.Fatalf("last delay = %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartupAccountOrder(t *testing.T) {
|
||||
sequential := startupAccountOrder(6, StartupOrderAccountIndex, 7)
|
||||
if !slices.Equal(sequential, []int{0, 1, 2, 3, 4, 5}) {
|
||||
t.Fatalf("sequential order = %v", sequential)
|
||||
}
|
||||
first := startupAccountOrder(100, StartupOrderShuffled, 20260827)
|
||||
second := startupAccountOrder(100, StartupOrderShuffled, 20260827)
|
||||
if !slices.Equal(first, second) {
|
||||
t.Fatal("shuffled startup order is not deterministic")
|
||||
}
|
||||
if slices.Equal(first, startupAccountOrder(100, StartupOrderShuffled, 20260828)) {
|
||||
t.Fatal("different shuffled seeds produced identical order")
|
||||
}
|
||||
seen := make(map[int]bool, len(first))
|
||||
for _, account := range first {
|
||||
if account < 0 || account >= len(first) || seen[account] {
|
||||
t.Fatalf("invalid shuffled account %d in %v", account, first)
|
||||
}
|
||||
seen[account] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveStartupProfiles(t *testing.T) {
|
||||
tdesktop, err := resolveStartupProfile(StartupProfileTDesktopReturningV1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !tdesktop.GetStateBeforeDifference || tdesktop.AccountDifference || tdesktop.Dialogs.limit(0) != 20 || tdesktop.Dialogs.limit(1) != 500 || !tdesktop.ForceChannelDifference {
|
||||
t.Fatalf("tdesktop profile = %+v", tdesktop)
|
||||
}
|
||||
tdlib, err := resolveStartupProfile(StartupProfileTDLibReturningV1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tdlib.GetStateBeforeDifference || !tdlib.AccountDifference || tdlib.Dialogs.limit(0) != 100 || tdlib.Dialogs.limit(1) != 100 {
|
||||
t.Fatalf("tdlib profile = %+v", tdlib)
|
||||
}
|
||||
if device := tdlib.device(); device.LangPack != "android" || device.SystemVersion != "Android SDK 36" {
|
||||
t.Fatalf("tdlib device = %+v", device)
|
||||
}
|
||||
if _, err := resolveStartupProfile("unknown"); err == nil {
|
||||
t.Fatal("unknown startup profile passed validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequireExactMarkers(t *testing.T) {
|
||||
if err := requireExactMarkers(map[string]int{"a": 1, "b": 1}, []string{"a", "b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := requireExactMarkers(map[string]int{"a": 2, "b": 1}, []string{"a", "b"}); err == nil {
|
||||
t.Fatal("duplicate marker passed validation")
|
||||
}
|
||||
if err := requireExactMarkers(map[string]int{"a": 1, "b": 1, "wrong": 1}, []string{"a", "b"}); err == nil {
|
||||
t.Fatal("wrong-account marker passed validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectChannelMarkersIncludesEdits(t *testing.T) {
|
||||
runID := "run"
|
||||
markers := make(map[string]int)
|
||||
collectChannelMarkers(markers,
|
||||
[]tg.MessageClass{&tg.Message{Message: "[run offline channel 0000 message 0001]"}},
|
||||
[]tg.UpdateClass{&tg.UpdateEditChannelMessage{Message: &tg.Message{Message: "[run offline channel 0000 message 0001] edited"}}},
|
||||
runID,
|
||||
)
|
||||
if markers["[run offline channel 0000 message 0001]"] != 1 || markers["[run offline channel 0000 message 0001] edited"] != 1 {
|
||||
t.Fatalf("collected markers = %v", markers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateStartupDialogsRequiresCurrentDirtyChannelPts(t *testing.T) {
|
||||
dataset, seedState, targets := snapshotFixture(t)
|
||||
plan := planOfflineMutation(dataset)
|
||||
mutation := &OfflineMutationState{
|
||||
PrivateMessageIDs: []int{1, 1, 1, 1},
|
||||
Channels: []OfflineMutationChannelState{{LatestPts: 50}},
|
||||
}
|
||||
expected := expectedDatasetPeers(dataset, seedState, targets, 0)
|
||||
dialogs := make([]ClientDialogState, 0, len(expected))
|
||||
for peer := range expected {
|
||||
dialog := ClientDialogState{PeerType: peer.typ, PeerID: peer.id, AccessHash: 1, TopMessage: 1, TopMessageDate: 1}
|
||||
if peer.typ == "channel" {
|
||||
dialog.HasPts, dialog.Pts = true, 50
|
||||
}
|
||||
dialogs = append(dialogs, dialog)
|
||||
}
|
||||
if err := validateStartupDialogs(dataset, seedState, plan, mutation, targets, 0, dialogs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range dialogs {
|
||||
if dialogs[i].PeerType == "channel" {
|
||||
dialogs[i].Pts = 49
|
||||
}
|
||||
}
|
||||
if err := validateStartupDialogs(dataset, seedState, plan, mutation, targets, 0, dialogs); err == nil {
|
||||
t.Fatal("stale current channel pts passed validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteStartupReportOwnerOnly(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "startup-report.json")
|
||||
report := &StartupRunReport{Version: StartupReportVersion, DatasetSHA256: "plan", ExpectedAccounts: 1, BusinessReady: 1, Pass: true}
|
||||
if err := WriteStartupReport(path, report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("startup report mode = %o, want 600", info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartupResponseBytesUsesPerMethodCounterDeltas(t *testing.T) {
|
||||
baseline := map[string]float64{
|
||||
`telesrv_mtproto_rpc_result_inner_bytes_total{method="messages.getDialogs"}`: 100,
|
||||
`telesrv_mtproto_rpc_result_wire_bytes_total{method="messages.getDialogs"}`: 50,
|
||||
}
|
||||
final := map[string]float64{
|
||||
`telesrv_mtproto_rpc_result_inner_bytes_total{method="messages.getDialogs"}`: 900,
|
||||
`telesrv_mtproto_rpc_result_wire_bytes_total{method="messages.getDialogs"}`: 250,
|
||||
`telesrv_mtproto_rpc_result_delivered_bytes_total{method="messages.getDialogs",outcome="ok"}`: 200,
|
||||
`telesrv_mtproto_rpc_result_delivered_bytes_total{method="messages.getDialogs",outcome="edge_overload"}`: 40,
|
||||
}
|
||||
bytes := startupResponseBytes(baseline, final)["messages.getDialogs"]
|
||||
if bytes.Inner != 800 || bytes.Wire != 200 || bytes.Delivered != 200 {
|
||||
t.Fatalf("response bytes = %+v", bytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartupDatabaseWorkUsesPerMethodCounterDeltas(t *testing.T) {
|
||||
baseline := map[string]float64{
|
||||
`telesrv_rpc_db_queries_total{method="messages.getDialogs"}`: 10,
|
||||
`telesrv_rpc_db_time_seconds_sum{method="messages.getDialogs"}`: 0.5,
|
||||
`telesrv_rpc_db_time_seconds_count{method="messages.getDialogs"}`: 1,
|
||||
`telesrv_rpc_db_errors_total{method="messages.getDialogs"}`: 1,
|
||||
}
|
||||
final := map[string]float64{
|
||||
`telesrv_rpc_db_queries_total{method="messages.getDialogs"}`: 210,
|
||||
`telesrv_rpc_db_time_seconds_sum{method="messages.getDialogs"}`: 2.75,
|
||||
`telesrv_rpc_db_time_seconds_count{method="messages.getDialogs"}`: 11,
|
||||
`telesrv_rpc_db_errors_total{method="messages.getDialogs"}`: 1,
|
||||
}
|
||||
work := startupDatabaseWork(baseline, final)["messages.getDialogs"]
|
||||
if work.Queries != 200 || work.RPCs != 10 || work.Errors != 0 || work.DurationSeconds != 2.25 {
|
||||
t.Fatalf("database work = %+v", work)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartupRPCDeliveryOutcomesUsesBoundedMethodAndOutcomeDeltas(t *testing.T) {
|
||||
baseline := map[string]float64{
|
||||
`telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="ok"}`: 5,
|
||||
}
|
||||
final := map[string]float64{
|
||||
`telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="ok"}`: 12,
|
||||
`telesrv_mtproto_rpc_result_delivered_total{method="users.getUsers",outcome="edge_overload"}`: 3,
|
||||
}
|
||||
outcomes := startupRPCDeliveryOutcomes(baseline, final)
|
||||
if outcomes["users.getUsers"]["ok"] != 7 || outcomes["users.getUsers"]["edge_overload"] != 3 {
|
||||
t.Fatalf("delivery outcomes = %#v", outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMetricPeaks(t *testing.T) {
|
||||
peaks := map[string]float64{"heap": 10}
|
||||
updateMetricPeaks(peaks, map[string]float64{"heap": 9, "connections": 5})
|
||||
updateMetricPeaks(peaks, map[string]float64{"heap": 12, "connections": 2})
|
||||
if peaks["heap"] != 12 || peaks["connections"] != 5 {
|
||||
t.Fatalf("peaks = %v", peaks)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue