merged from gramsrv upstream
This commit is contained in:
onysd
parent
79c64ee916
commit
21a0856587
651 changed files with 54774 additions and 4590 deletions
446
internal/loadharness/dataset.go
Normal file
446
internal/loadharness/dataset.go
Normal file
|
|
@ -0,0 +1,446 @@
|
|||
package loadharness
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
DatasetVersion = 1
|
||||
maxDatasetGroups = 10000
|
||||
maxDatasetMembership = 2_000_000
|
||||
maxDatasetMessages = 2_000_000
|
||||
)
|
||||
|
||||
// DatasetConfig describes a deterministic social graph whose durable facts are
|
||||
// later materialized exclusively through real MTProto RPCs. Planning does not
|
||||
// contact the server and never embeds auth/session material.
|
||||
type DatasetConfig struct {
|
||||
Accounts int `json:"accounts"`
|
||||
Seed int64 `json:"seed"`
|
||||
PrivateFanout int `json:"private_fanout"`
|
||||
HotGroups int `json:"hot_groups"`
|
||||
HotMembers int `json:"hot_members"`
|
||||
HotHistory int `json:"hot_history"`
|
||||
MediumGroups int `json:"medium_groups"`
|
||||
MediumMembers int `json:"medium_members"`
|
||||
MediumHistory int `json:"medium_history"`
|
||||
SmallGroups int `json:"small_groups"`
|
||||
SmallMembers int `json:"small_members"`
|
||||
SmallHistory int `json:"small_history"`
|
||||
HeavyGroups int `json:"heavy_groups"`
|
||||
HeavyAccounts int `json:"heavy_accounts"`
|
||||
HeavyHistory int `json:"heavy_history"`
|
||||
}
|
||||
|
||||
func DefaultDatasetConfig(accounts int) DatasetConfig {
|
||||
return DatasetConfig{
|
||||
Accounts: accounts, Seed: 20260827, PrivateFanout: min(10, max(accounts-1, 0)),
|
||||
HotGroups: 10, HotMembers: accounts, HotHistory: 100,
|
||||
MediumGroups: 100, MediumMembers: min(100, accounts), MediumHistory: 30,
|
||||
SmallGroups: 200, SmallMembers: min(20, accounts), SmallHistory: 10,
|
||||
HeavyGroups: 200, HeavyAccounts: min(100, accounts), HeavyHistory: 30,
|
||||
}
|
||||
}
|
||||
|
||||
type DatasetPrivateEdge struct {
|
||||
SenderAccount int `json:"sender_account"`
|
||||
RecipientAccount int `json:"recipient_account"`
|
||||
RandomID int64 `json:"random_id"`
|
||||
Marker string `json:"marker"`
|
||||
}
|
||||
|
||||
type DatasetGroup struct {
|
||||
Index int `json:"index"`
|
||||
Tier string `json:"tier"`
|
||||
Title string `json:"title"`
|
||||
About string `json:"about"`
|
||||
CreatorAccount int `json:"creator_account"`
|
||||
MemberAccounts []int `json:"member_accounts"`
|
||||
HistoryMessages int `json:"history_messages"`
|
||||
}
|
||||
|
||||
// Dataset is the immutable plan. Real server identities and resumable progress
|
||||
// live in the separate compact DatasetSeedState so a large plan is not rewritten
|
||||
// after every reconciled RPC batch.
|
||||
type Dataset struct {
|
||||
Version int `json:"version"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
RunID string `json:"run_id"`
|
||||
Config DatasetConfig `json:"config"`
|
||||
PlanSHA256 string `json:"plan_sha256"`
|
||||
PrivateEdges []DatasetPrivateEdge `json:"private_edges"`
|
||||
Groups []DatasetGroup `json:"groups"`
|
||||
}
|
||||
|
||||
type DatasetSeedGroupState struct {
|
||||
GroupIndex int `json:"group_index"`
|
||||
ChannelID int64 `json:"channel_id,omitempty"`
|
||||
AccessHash int64 `json:"access_hash,omitempty"`
|
||||
CreatePending bool `json:"create_pending,omitempty"`
|
||||
InviteCursor int `json:"invite_cursor,omitempty"`
|
||||
InvitePendingEnd int `json:"invite_pending_end,omitempty"`
|
||||
}
|
||||
|
||||
type DatasetSeedState struct {
|
||||
Version int `json:"version"`
|
||||
PlanSHA256 string `json:"plan_sha256"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
PrivateSentByAccount []int `json:"private_sent_by_account"`
|
||||
HistorySentByAccount []int `json:"history_sent_by_account"`
|
||||
RichStateByAccount []bool `json:"rich_state_by_account,omitempty"`
|
||||
Groups []DatasetSeedGroupState `json:"groups"`
|
||||
}
|
||||
|
||||
func PlanDataset(cfg DatasetConfig) (*Dataset, error) {
|
||||
if err := cfg.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
runID := fmt.Sprintf("rpc-startup-%d-%016x", cfg.Accounts, uint64(cfg.Seed))
|
||||
dataset := &Dataset{
|
||||
Version: DatasetVersion, CreatedAt: time.Now().UTC(),
|
||||
RunID: runID, Config: cfg,
|
||||
PrivateEdges: make([]DatasetPrivateEdge, 0, cfg.Accounts*cfg.PrivateFanout),
|
||||
}
|
||||
for account := 0; account < cfg.Accounts; account++ {
|
||||
for offset := 1; offset <= cfg.PrivateFanout; offset++ {
|
||||
recipient := (account + offset) % cfg.Accounts
|
||||
dataset.PrivateEdges = append(dataset.PrivateEdges, DatasetPrivateEdge{
|
||||
SenderAccount: account, RecipientAccount: recipient,
|
||||
RandomID: stableDatasetID(cfg.Seed, "private", cfg.Accounts, account, offset),
|
||||
Marker: fmt.Sprintf("[%s private %04d/%02d]", runID, account, offset),
|
||||
})
|
||||
}
|
||||
}
|
||||
groupIndex := 0
|
||||
appendTier := func(tier string, count, members, history int, memberSet func(int) []int) {
|
||||
for i := 0; i < count; i++ {
|
||||
groupMembers := memberSet(i)
|
||||
creator := groupMembers[i%len(groupMembers)]
|
||||
dataset.Groups = append(dataset.Groups, DatasetGroup{
|
||||
Index: groupIndex, Tier: tier,
|
||||
Title: fmt.Sprintf("%s %s %04d", runID, tier, i+1),
|
||||
About: fmt.Sprintf("telesrv real-RPC load dataset %s group %d", tier, i+1),
|
||||
CreatorAccount: creator, MemberAccounts: groupMembers, HistoryMessages: history,
|
||||
})
|
||||
groupIndex++
|
||||
}
|
||||
}
|
||||
appendTier("hot", cfg.HotGroups, cfg.HotMembers, cfg.HotHistory, func(i int) []int {
|
||||
return cyclicMembers(cfg.Accounts, i*max(cfg.HotMembers, 1), cfg.HotMembers)
|
||||
})
|
||||
appendTier("medium", cfg.MediumGroups, cfg.MediumMembers, cfg.MediumHistory, func(i int) []int {
|
||||
return cyclicMembers(cfg.Accounts, i*max(cfg.MediumMembers, 1), cfg.MediumMembers)
|
||||
})
|
||||
appendTier("small", cfg.SmallGroups, cfg.SmallMembers, cfg.SmallHistory, func(i int) []int {
|
||||
return cyclicMembers(cfg.Accounts, i*max(cfg.SmallMembers, 1), cfg.SmallMembers)
|
||||
})
|
||||
appendTier("heavy", cfg.HeavyGroups, cfg.HeavyAccounts, cfg.HeavyHistory, func(int) []int {
|
||||
members := make([]int, cfg.HeavyAccounts)
|
||||
for i := range members {
|
||||
members[i] = i
|
||||
}
|
||||
return members
|
||||
})
|
||||
planHash, err := dataset.planHash()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dataset.PlanSHA256 = planHash
|
||||
if err := dataset.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dataset, nil
|
||||
}
|
||||
|
||||
func (c DatasetConfig) validate() error {
|
||||
if c.Accounts < 2 || c.Accounts > 100000 {
|
||||
return errors.New("dataset accounts must be between 2 and 100000")
|
||||
}
|
||||
if c.PrivateFanout < 0 || c.PrivateFanout >= c.Accounts {
|
||||
return errors.New("private fanout must be non-negative and smaller than accounts")
|
||||
}
|
||||
groups := c.HotGroups + c.MediumGroups + c.SmallGroups + c.HeavyGroups
|
||||
if groups <= 0 || groups > maxDatasetGroups {
|
||||
return fmt.Errorf("dataset group count must be between 1 and %d", maxDatasetGroups)
|
||||
}
|
||||
tiers := []struct {
|
||||
name string
|
||||
groups, members, history int
|
||||
}{
|
||||
{"hot", c.HotGroups, c.HotMembers, c.HotHistory},
|
||||
{"medium", c.MediumGroups, c.MediumMembers, c.MediumHistory},
|
||||
{"small", c.SmallGroups, c.SmallMembers, c.SmallHistory},
|
||||
{"heavy", c.HeavyGroups, c.HeavyAccounts, c.HeavyHistory},
|
||||
}
|
||||
memberships := 0
|
||||
messages := c.Accounts * c.PrivateFanout
|
||||
for _, tier := range tiers {
|
||||
if tier.groups < 0 || tier.members < 0 || tier.members > c.Accounts || tier.history < 0 {
|
||||
return fmt.Errorf("invalid %s dataset tier", tier.name)
|
||||
}
|
||||
if tier.groups > 0 && tier.members == 0 {
|
||||
return fmt.Errorf("%s dataset tier has groups without members", tier.name)
|
||||
}
|
||||
memberships += tier.groups * tier.members
|
||||
messages += tier.groups * tier.history
|
||||
}
|
||||
if memberships > maxDatasetMembership {
|
||||
return fmt.Errorf("dataset memberships %d exceed hard limit %d", memberships, maxDatasetMembership)
|
||||
}
|
||||
if messages > maxDatasetMessages {
|
||||
return fmt.Errorf("dataset messages %d exceed hard limit %d", messages, maxDatasetMessages)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dataset) Validate() error {
|
||||
if d == nil || d.Version != DatasetVersion {
|
||||
return errors.New("invalid dataset version")
|
||||
}
|
||||
if strings.TrimSpace(d.RunID) == "" || strings.TrimSpace(d.PlanSHA256) == "" {
|
||||
return errors.New("dataset is missing run id or plan hash")
|
||||
}
|
||||
if err := d.Config.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(d.PrivateEdges) != d.Config.Accounts*d.Config.PrivateFanout {
|
||||
return errors.New("dataset private graph size does not match config")
|
||||
}
|
||||
seenGroups := make(map[int]struct{}, len(d.Groups))
|
||||
for _, group := range d.Groups {
|
||||
if group.Index < 0 || group.CreatorAccount < 0 || group.CreatorAccount >= d.Config.Accounts || group.HistoryMessages < 0 {
|
||||
return fmt.Errorf("invalid group %d", group.Index)
|
||||
}
|
||||
if _, exists := seenGroups[group.Index]; exists {
|
||||
return fmt.Errorf("duplicate group index %d", group.Index)
|
||||
}
|
||||
seenGroups[group.Index] = struct{}{}
|
||||
members := make(map[int]struct{}, len(group.MemberAccounts))
|
||||
for _, account := range group.MemberAccounts {
|
||||
if account < 0 || account >= d.Config.Accounts {
|
||||
return fmt.Errorf("group %d has invalid account %d", group.Index, account)
|
||||
}
|
||||
if _, exists := members[account]; exists {
|
||||
return fmt.Errorf("group %d has duplicate account %d", group.Index, account)
|
||||
}
|
||||
members[account] = struct{}{}
|
||||
}
|
||||
if _, ok := members[group.CreatorAccount]; !ok {
|
||||
return fmt.Errorf("group %d creator is not a member", group.Index)
|
||||
}
|
||||
}
|
||||
wantHash, err := d.planHash()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if wantHash != d.PlanSHA256 {
|
||||
return errors.New("dataset immutable plan hash mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func WriteDataset(path string, dataset *Dataset) error {
|
||||
if err := dataset.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(dataset, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode dataset: %w", err)
|
||||
}
|
||||
return writeFileAtomic(path, append(data, '\n'), 0o600)
|
||||
}
|
||||
|
||||
func NewDatasetSeedState(dataset *Dataset) (*DatasetSeedState, error) {
|
||||
if err := dataset.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
state := &DatasetSeedState{
|
||||
Version: DatasetVersion, PlanSHA256: dataset.PlanSHA256, UpdatedAt: time.Now().UTC(),
|
||||
PrivateSentByAccount: make([]int, dataset.Config.Accounts),
|
||||
HistorySentByAccount: make([]int, dataset.Config.Accounts),
|
||||
RichStateByAccount: make([]bool, dataset.Config.Accounts),
|
||||
Groups: make([]DatasetSeedGroupState, len(dataset.Groups)),
|
||||
}
|
||||
for i, group := range dataset.Groups {
|
||||
state.Groups[i].GroupIndex = group.Index
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func (s *DatasetSeedState) Validate(dataset *Dataset) error {
|
||||
if s == nil || s.Version != DatasetVersion || dataset == nil || s.PlanSHA256 != dataset.PlanSHA256 {
|
||||
return errors.New("seed state does not match dataset plan")
|
||||
}
|
||||
if len(s.PrivateSentByAccount) != dataset.Config.Accounts || len(s.HistorySentByAccount) != dataset.Config.Accounts || len(s.Groups) != len(dataset.Groups) {
|
||||
return errors.New("seed state dimensions do not match dataset plan")
|
||||
}
|
||||
// A nil rich-state vector is accepted only for seed journals created before
|
||||
// the comprehensive startup workload added this phase. New journals always
|
||||
// allocate the full vector and therefore cannot silently skip it.
|
||||
if len(s.RichStateByAccount) != 0 && len(s.RichStateByAccount) != dataset.Config.Accounts {
|
||||
return errors.New("seed rich-state dimensions do not match dataset plan")
|
||||
}
|
||||
historyTasks := datasetHistoryTaskCounts(dataset)
|
||||
for account := 0; account < dataset.Config.Accounts; account++ {
|
||||
if s.PrivateSentByAccount[account] < 0 || s.PrivateSentByAccount[account] > dataset.Config.PrivateFanout {
|
||||
return fmt.Errorf("invalid private seed progress for account %d", account)
|
||||
}
|
||||
if s.HistorySentByAccount[account] < 0 || s.HistorySentByAccount[account] > historyTasks[account] {
|
||||
return fmt.Errorf("invalid history seed progress for account %d", account)
|
||||
}
|
||||
}
|
||||
for i, groupState := range s.Groups {
|
||||
group := dataset.Groups[i]
|
||||
invitees := len(group.MemberAccounts) - 1
|
||||
if groupState.GroupIndex != group.Index || groupState.InviteCursor < 0 || groupState.InviteCursor > invitees {
|
||||
return fmt.Errorf("invalid seed state for group %d", group.Index)
|
||||
}
|
||||
if (groupState.ChannelID == 0) != (groupState.AccessHash == 0) {
|
||||
return fmt.Errorf("group %d has partial channel identity", group.Index)
|
||||
}
|
||||
if groupState.ChannelID != 0 && groupState.CreatePending {
|
||||
return fmt.Errorf("group %d has both channel identity and pending create", group.Index)
|
||||
}
|
||||
if groupState.InvitePendingEnd < groupState.InviteCursor || groupState.InvitePendingEnd > invitees {
|
||||
return fmt.Errorf("group %d has invalid pending invite range", group.Index)
|
||||
}
|
||||
if groupState.ChannelID == 0 && (groupState.InviteCursor != 0 || groupState.InvitePendingEnd != 0) {
|
||||
return fmt.Errorf("group %d has invite progress without a channel identity", group.Index)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func WriteDatasetSeedState(path string, dataset *Dataset, state *DatasetSeedState) error {
|
||||
if err := state.Validate(dataset); err != nil {
|
||||
return err
|
||||
}
|
||||
state.UpdatedAt = time.Now().UTC()
|
||||
data, err := json.MarshalIndent(state, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode dataset seed state: %w", err)
|
||||
}
|
||||
return writeFileAtomic(path, append(data, '\n'), 0o600)
|
||||
}
|
||||
|
||||
func LoadDatasetSeedState(path string, dataset *Dataset) (*DatasetSeedState, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if os.IsNotExist(err) {
|
||||
return NewDatasetSeedState(dataset)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read dataset seed state: %w", err)
|
||||
}
|
||||
var state DatasetSeedState
|
||||
decoder := json.NewDecoder(strings.NewReader(string(data)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&state); err != nil {
|
||||
return nil, fmt.Errorf("decode dataset seed state: %w", err)
|
||||
}
|
||||
if err := state.Validate(dataset); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &state, nil
|
||||
}
|
||||
|
||||
func datasetHistoryTaskCounts(dataset *Dataset) []int {
|
||||
counts := make([]int, dataset.Config.Accounts)
|
||||
for _, group := range dataset.Groups {
|
||||
for message := 0; message < group.HistoryMessages; message++ {
|
||||
counts[group.MemberAccounts[message%len(group.MemberAccounts)]]++
|
||||
}
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
func LoadDataset(path string) (*Dataset, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read dataset: %w", err)
|
||||
}
|
||||
var dataset Dataset
|
||||
decoder := json.NewDecoder(strings.NewReader(string(data)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&dataset); err != nil {
|
||||
return nil, fmt.Errorf("decode dataset: %w", err)
|
||||
}
|
||||
if err := dataset.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dataset, nil
|
||||
}
|
||||
|
||||
func (d *Dataset) planHash() (string, error) {
|
||||
type immutableDataset struct {
|
||||
Version int `json:"version"`
|
||||
RunID string `json:"run_id"`
|
||||
Config DatasetConfig `json:"config"`
|
||||
PrivateEdges []DatasetPrivateEdge `json:"private_edges"`
|
||||
Groups []struct {
|
||||
Index int `json:"index"`
|
||||
Tier string `json:"tier"`
|
||||
Title string `json:"title"`
|
||||
About string `json:"about"`
|
||||
CreatorAccount int `json:"creator_account"`
|
||||
MemberAccounts []int `json:"member_accounts"`
|
||||
HistoryMessages int `json:"history_messages"`
|
||||
} `json:"groups"`
|
||||
}
|
||||
immutable := immutableDataset{Version: d.Version, RunID: d.RunID, Config: d.Config, PrivateEdges: d.PrivateEdges}
|
||||
for _, group := range d.Groups {
|
||||
immutable.Groups = append(immutable.Groups, struct {
|
||||
Index int `json:"index"`
|
||||
Tier string `json:"tier"`
|
||||
Title string `json:"title"`
|
||||
About string `json:"about"`
|
||||
CreatorAccount int `json:"creator_account"`
|
||||
MemberAccounts []int `json:"member_accounts"`
|
||||
HistoryMessages int `json:"history_messages"`
|
||||
}{group.Index, group.Tier, group.Title, group.About, group.CreatorAccount, group.MemberAccounts, group.HistoryMessages})
|
||||
}
|
||||
data, err := json.Marshal(immutable)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func cyclicMembers(accounts, start, count int) []int {
|
||||
if count <= 0 {
|
||||
return nil
|
||||
}
|
||||
members := make([]int, count)
|
||||
for i := range members {
|
||||
members[i] = (start + i) % accounts
|
||||
}
|
||||
sort.Ints(members)
|
||||
return members
|
||||
}
|
||||
|
||||
func stableDatasetID(seed int64, namespace string, values ...int) int64 {
|
||||
h := sha256.New()
|
||||
var buf [8]byte
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(seed))
|
||||
_, _ = h.Write(buf[:])
|
||||
_, _ = h.Write([]byte(namespace))
|
||||
for _, value := range values {
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(value))
|
||||
_, _ = h.Write(buf[:])
|
||||
}
|
||||
sum := h.Sum(nil)
|
||||
id := int64(binary.LittleEndian.Uint64(sum[:8]) & ^(uint64(1) << 63))
|
||||
if id == 0 {
|
||||
return 1
|
||||
}
|
||||
return id
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue