Initial open source release

This commit is contained in:
A 2026-06-04 01:37:39 +08:00
commit 74992e893f
377 changed files with 118084 additions and 0 deletions

13
internal/loadtest/doc.go Normal file
View file

@ -0,0 +1,13 @@
// Package loadtest 承载 message 模块的压测基线。
//
// 这里只放 env-gated 的压测用例(真实 PostgreSQL + Redis不被生产代码 import。
// 目标与基线方法见 docs/message-module.md 的「Load Baseline」一节。
//
// 运行方式PowerShell
//
// $env:TELESRV_TEST_POSTGRES_DSN = "postgres://telesrv:telesrv@localhost:5432/telesrv?sslmode=disable"
// $env:TELESRV_TEST_REDIS_ADDR = "localhost:6399"
// go test ./internal/loadtest/ -run TestMessageSendBaseline -v -count=1
//
// 未设置上述两个环境变量时用例直接 Skip因此对默认 `go test ./...` 无副作用。
package loadtest

View file

@ -0,0 +1,478 @@
package loadtest
import (
"context"
"crypto/rand"
"encoding/binary"
"fmt"
"math"
"os"
"sort"
"strconv"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
"go.uber.org/zap"
messageapp "telesrv/internal/app/messages"
"telesrv/internal/domain"
"telesrv/internal/mtprotoedge"
"telesrv/internal/rpc"
"telesrv/internal/store/postgres"
"telesrv/internal/store/redisstore"
)
// 第一阶段单机 SLO 目标,来自 docs/message-module.md 的 Next Execution Plan。
// 默认仅作为信息打印;设 TELESRV_LOAD_ENFORCE_SLO=1 时超标会 fail用于回归门禁
const (
sloSendP99 = 150 * time.Millisecond
sloDiffP99 = 100 * time.Millisecond
sloThroughput = 200.0 // msg/s
drainTimeout = 60 * time.Second
diffSampleGoal = 500
)
// TestMessageSendBaseline 用真实 PostgreSQL + Redis 压测私聊文本发送热路径,
// 并发跑 outbox dispatcher 排空在线推送,最后采样 getDifference 读路径。
//
// 这是 closed-loop 饱和压测concurrency 个 worker 各自不停发,直到发满 messages 条。
// 吞吐 = messages / wallclock延迟分位反映该并发下的饱和延迟。
// 固定到达率open-loop的版本留作后续细化见 docs/message-module.md
func TestMessageSendBaseline(t *testing.T) {
dsn := os.Getenv("TELESRV_TEST_POSTGRES_DSN")
redisAddr := os.Getenv("TELESRV_TEST_REDIS_ADDR")
if dsn == "" || redisAddr == "" {
t.Skip("set TELESRV_TEST_POSTGRES_DSN and TELESRV_TEST_REDIS_ADDR to run message load baseline")
}
// 默认用户池取较大值:用户太少会把写集中到少数 dialog/message_box 行造成行锁争用,
// 拉高 send 尾延迟(这是小池假象,生产 20 万用户分散后争用极低)。
users := envInt("TELESRV_LOAD_USERS", 1000)
if users < 2 {
users = 2
}
concurrency := envInt("TELESRV_LOAD_CONCURRENCY", 32)
if concurrency < 1 {
concurrency = 1
}
totalMsgs := envInt("TELESRV_LOAD_MESSAGES", 5000)
if totalMsgs < 1 {
totalMsgs = 1
}
poolConns := envInt("TELESRV_LOAD_POOL_CONNS", 64)
workers := envInt("TELESRV_OUTBOX_WORKERS", 8)
outboxBatch := envInt("TELESRV_OUTBOX_BATCH", 100)
outboxInterval := envDuration("TELESRV_OUTBOX_INTERVAL", 50*time.Millisecond)
leaseTimeout := envDuration("TELESRV_OUTBOX_LEASE_TIMEOUT", 30*time.Second)
enforceSLO := os.Getenv("TELESRV_LOAD_ENFORCE_SLO") == "1"
deferDispatch := os.Getenv("TELESRV_LOAD_DEFER_DISPATCH") == "1"
ctx := context.Background()
if err := postgres.Migrate(dsn); err != nil {
t.Fatalf("migrate: %v", err)
}
pool, err := postgres.Open(ctx, dsn, postgres.WithMaxConns(poolConns))
if err != nil {
t.Fatalf("open postgres: %v", err)
}
t.Cleanup(pool.Close)
rdb, err := redisstore.Open(ctx, redisAddr, os.Getenv("TELESRV_TEST_REDIS_PASSWORD"), 0)
if err != nil {
t.Fatalf("open redis: %v", err)
}
t.Cleanup(func() { _ = rdb.Close() })
// 装配与 main.go 一致的消息热路径Redis 分配器 + PG 消息存储 + transactional outbox。
userStore := postgres.NewUserStore(pool)
updateEventStore := postgres.NewUpdateEventStore(pool)
dispatchOutboxStore := postgres.NewDispatchOutboxStore(pool, postgres.WithLeaseTimeout(leaseTimeout))
dialogStore := postgres.NewDialogStore(pool)
ptsAllocator := redisstore.NewPtsAllocator(rdb, updateEventStore)
boxIDAllocator := redisstore.NewBoxIDAllocator(rdb, postgres.NewMessageBoxCounterSource(pool))
messageStore := postgres.NewMessageStore(pool, postgres.WithMessageAllocators(boxIDAllocator, ptsAllocator))
svc := messageapp.NewService(messageStore, dialogStore)
// 创建独立的测试用户池;用随机 salt 隔离历史残留,结束按 FK 依赖序清理。
ids := seedUsers(t, ctx, userStore, users)
t.Cleanup(func() { cleanup(t, pool, rdb, ids) })
// 在线推送 binder 用真实 SessionManager零连接PushToUserExceptSession 返回 0
// 让 outbox 走完整 claim→ListAfter→MarkDelivered 的 PG 往返,测排空而非网络 fanout。
binder := mtprotoedge.NewSessionManager(zap.NewNop())
metrics := &loadMetrics{}
dispatcher := rpc.NewOutboxDispatcher(updateEventStore, dispatchOutboxStore, binder, zap.NewNop(),
rpc.WithOutboxWorkers(workers),
rpc.WithOutboxBatch(outboxBatch),
rpc.WithOutboxInterval(outboxInterval),
rpc.WithOutboxMetrics(metrics),
)
dispCtx, stopDispatcher := context.WithCancel(ctx)
dispDone := make(chan struct{})
startDispatcher := func() {
go func() {
dispatcher.Run(dispCtx)
close(dispDone)
}()
}
// deferDispatch=1 时先不投递,让发送把积压攒满,再在发送结束后启动 dispatcher
// 以隔离测量「纯排空上限」(否则 dispatcher 实时跟上发送、积压近 0 量不出天花板)。
if !deferDispatch {
startDispatcher()
}
// 后台采样 outbox 积压pending+dispatching记录运行期峰值。
var maxBacklog atomic.Int64
sampleCtx, stopSampler := context.WithCancel(ctx)
sampleDone := make(chan struct{})
go func() {
defer close(sampleDone)
ticker := time.NewTicker(50 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-sampleCtx.Done():
return
case <-ticker.C:
if n := backlog(ctx, pool, ids); n > maxBacklog.Load() {
maxBacklog.Store(n)
}
}
}
}()
// 随机 RandomID 基址,保证 (sender,random_id) 幂等唯一且跨重跑不撞。
randBase := int64(randomUint64(t) & 0x7fff_ffff_ffff)
nowUnix := int(time.Now().Unix())
body := "telesrv load baseline message body"
perWorkerLat := make([][]time.Duration, concurrency)
var sent, dup, sendErr atomic.Int64
var counter atomic.Int64
// 预热先并发发一批不计时的消息热连接池MinConns→MaxConns与 PG plan 缓存,
// 让后续计时窗口反映稳态而非冷启动尾延迟(池过冷时首批 32 并发会挤少量连接)。
warmup := envInt("TELESRV_LOAD_WARMUP", min(users*10, 1000))
if warmup > 0 {
var wwg sync.WaitGroup
for w := 0; w < concurrency; w++ {
wwg.Add(1)
go func() {
defer wwg.Done()
for {
n := counter.Add(1)
if n > int64(warmup) {
return
}
sid := ids[(n-1)%int64(users)]
rid := ids[n%int64(users)]
_, _ = svc.SendPrivateText(ctx, sid, domain.SendPrivateTextRequest{
SenderUserID: sid,
RecipientUserID: rid,
RandomID: randBase + n,
Message: body,
Date: nowUnix,
})
}
}()
}
wwg.Wait()
counter.Store(int64(warmup))
}
start := time.Now()
var wg sync.WaitGroup
for w := 0; w < concurrency; w++ {
wg.Add(1)
go func(w int) {
defer wg.Done()
lat := make([]time.Duration, 0, totalMsgs/concurrency+1)
for {
n := counter.Add(1)
if n > int64(warmup+totalMsgs) {
break
}
senderID := ids[(n-1)%int64(users)]
recipientID := ids[n%int64(users)]
req := domain.SendPrivateTextRequest{
SenderUserID: senderID,
RecipientUserID: recipientID,
RandomID: randBase + n,
Message: body,
Date: nowUnix,
}
t0 := time.Now()
res, err := svc.SendPrivateText(ctx, senderID, req)
lat = append(lat, time.Since(t0))
if err != nil {
sendErr.Add(1)
continue
}
if res.Duplicate {
dup.Add(1)
}
sent.Add(1)
}
perWorkerLat[w] = lat
}(w)
}
wg.Wait()
sendWall := time.Since(start)
deliveredAtSendEnd := metrics.delivered.Load()
if deferDispatch {
// 发送已把全部积压攒满,此刻才启动 dispatcherdrain 阶段即纯排空drainRate 反映排空上限。
startDispatcher()
}
// 等 outbox 排空(积压回 0记录排空耗时再停采样和 dispatcher。
// 用「发送结束后」单独排空的速率隔离 dispatcher 吞吐,去掉发送期对 PG 的争用。
drainStart := time.Now()
drained := waitDrain(ctx, pool, ids, drainTimeout)
drainWall := time.Since(drainStart)
drainRate := float64(metrics.delivered.Load()-deliveredAtSendEnd) / drainWall.Seconds()
stopSampler()
<-sampleDone
stopDispatcher()
<-dispDone
// 合并发送延迟样本并排序。
latencies := make([]time.Duration, 0, totalMsgs)
for _, l := range perWorkerLat {
latencies = append(latencies, l...)
}
sort.Slice(latencies, func(i, j int) bool { return latencies[i] < latencies[j] })
// 采样 getDifference 读路径ListAfter 从 pts=0 拉账号事件)。
diffLat := sampleGetDifference(t, ctx, updateEventStore, ids)
sort.Slice(diffLat, func(i, j int) bool { return diffLat[i] < diffLat[j] })
okSent := sent.Load()
throughput := float64(okSent) / sendWall.Seconds()
sendP99 := percentile(latencies, 99)
diffP99 := percentile(diffLat, 99)
t.Logf("==== message module load baseline ====")
t.Logf("config: users=%d concurrency=%d messages=%d pool=%d outbox(workers=%d batch=%d interval=%s lease=%s)",
users, concurrency, totalMsgs, poolConns, workers, outboxBatch, outboxInterval, leaseTimeout)
t.Logf("send: %d ok, %d dup, %d err in %s -> %.0f msg/s",
okSent, dup.Load(), sendErr.Load(), sendWall.Round(time.Millisecond), throughput)
t.Logf("send.lat p50=%s p90=%s p99=%s max=%s",
percentile(latencies, 50).Round(time.Microsecond),
percentile(latencies, 90).Round(time.Microsecond),
sendP99.Round(time.Microsecond),
percentile(latencies, 100).Round(time.Microsecond))
t.Logf("outbox: delivered=%d failed=%d claimed=%d maxBacklog=%d drain=%s drainRate=%.0f rows/s drained=%v",
metrics.delivered.Load(), metrics.failed.Load(), metrics.claimed.Load(),
maxBacklog.Load(), drainWall.Round(time.Millisecond), drainRate, drained)
t.Logf("getDiff: samples=%d p50=%s p90=%s p99=%s max=%s",
len(diffLat),
percentile(diffLat, 50).Round(time.Microsecond),
percentile(diffLat, 90).Round(time.Microsecond),
diffP99.Round(time.Microsecond),
percentile(diffLat, 100).Round(time.Microsecond))
t.Logf("SLO: send.p99 %s(<%s) %s | getDiff.p99 %s(<%s) %s | throughput %.0f(>=%.0f) %s",
sendP99.Round(time.Millisecond), sloSendP99, pass(sendP99 < sloSendP99),
diffP99.Round(time.Millisecond), sloDiffP99, pass(diffP99 < sloDiffP99),
throughput, sloThroughput, pass(throughput >= sloThroughput))
t.Logf("=======================================")
// 正确性硬断言发送不应出错、不应有意外重复、outbox 必须排空且无终态失败。
if sendErr.Load() != 0 {
t.Fatalf("send errors = %d, want 0", sendErr.Load())
}
if dup.Load() != 0 {
t.Fatalf("duplicates = %d, want 0 (random_id 应唯一)", dup.Load())
}
if okSent != int64(totalMsgs) {
t.Fatalf("sent ok = %d, want %d", okSent, totalMsgs)
}
if !drained {
t.Fatalf("outbox 未在 %s 内排空,残留积压 %d", drainTimeout, backlog(ctx, pool, ids))
}
if metrics.failed.Load() != 0 {
t.Fatalf("outbox failed = %d, want 0", metrics.failed.Load())
}
// 性能 SLO默认信息化门禁模式TELESRV_LOAD_ENFORCE_SLO=1才硬失败。
if enforceSLO {
if sendP99 >= sloSendP99 {
t.Errorf("send p99 %s >= SLO %s", sendP99, sloSendP99)
}
if diffP99 >= sloDiffP99 {
t.Errorf("getDifference p99 %s >= SLO %s", diffP99, sloDiffP99)
}
if throughput < sloThroughput {
t.Errorf("throughput %.0f msg/s < SLO %.0f msg/s", throughput, sloThroughput)
}
}
}
// loadMetrics 实现 rpc.Metrics统计 outbox claim/deliver/fail。
type loadMetrics struct {
claimed atomic.Int64
delivered atomic.Int64
failed atomic.Int64
}
func (m *loadMetrics) MessageSend(time.Duration, bool, error) {}
func (m *loadMetrics) MessageRateLimited(int) {}
func (m *loadMetrics) OutboxClaimed(n int) { m.claimed.Add(int64(n)) }
func (m *loadMetrics) OutboxDelivered(time.Duration) { m.delivered.Add(1) }
func (m *loadMetrics) OutboxFailed(error) { m.failed.Add(1) }
func seedUsers(t *testing.T, ctx context.Context, store *postgres.UserStore, n int) []int64 {
t.Helper()
salt := randomUint64(t) % 1_000_000
ids := make([]int64, n)
errs := make([]error, n)
sem := make(chan struct{}, 16)
var wg sync.WaitGroup
for i := 0; i < n; i++ {
wg.Add(1)
sem <- struct{}{}
go func(i int) {
defer wg.Done()
defer func() { <-sem }()
u, err := store.Create(ctx, domain.User{
AccessHash: int64(i + 1),
Phone: fmt.Sprintf("+1555%06d%05d", salt, i),
FirstName: fmt.Sprintf("Load%05d", i),
})
if err != nil {
errs[i] = err
return
}
ids[i] = u.ID
}(i)
}
wg.Wait()
for i, err := range errs {
if err != nil {
t.Fatalf("create load user %d: %v", i, err)
}
}
return ids
}
// cleanup 按 FK 依赖序删除测试数据outbox→events→boxes→private_messages→dialogs→users
// 再清 Redis pts/box_id 计数。message_boxes.from_user_id 为 ON DELETE RESTRICT必须先删盒子。
// cleanup 在断言之后运行,出错只告警不影响已得结果。
func cleanup(t *testing.T, pool *pgxpool.Pool, rdb *redis.Client, ids []int64) {
t.Helper()
ctx := context.Background()
stmts := []string{
"DELETE FROM dispatch_outbox WHERE target_user_id = ANY($1::bigint[])",
"DELETE FROM user_update_events WHERE user_id = ANY($1::bigint[])",
"DELETE FROM message_boxes WHERE owner_user_id = ANY($1::bigint[])",
"DELETE FROM private_messages WHERE sender_user_id = ANY($1::bigint[])",
"DELETE FROM dialogs WHERE user_id = ANY($1::bigint[])",
"DELETE FROM users WHERE id = ANY($1::bigint[])",
}
for _, sql := range stmts {
if _, err := pool.Exec(ctx, sql, ids); err != nil {
t.Logf("cleanup %q: %v", sql, err)
}
}
keys := make([]string, 0, len(ids)*2)
for _, id := range ids {
keys = append(keys,
fmt.Sprintf("counter:pts:{%d}", id),
fmt.Sprintf("counter:box_id:{%d}", id),
)
}
if err := rdb.Del(ctx, keys...).Err(); err != nil {
t.Logf("cleanup redis counters: %v", err)
}
}
// backlog 返回测试用户集当前未投递pending+dispatching的 outbox 行数。
func backlog(ctx context.Context, pool *pgxpool.Pool, ids []int64) int64 {
var n int64
if err := pool.QueryRow(ctx,
"SELECT count(*) FROM dispatch_outbox WHERE target_user_id = ANY($1::bigint[]) AND status IN ('pending','dispatching')",
ids,
).Scan(&n); err != nil {
return -1
}
return n
}
// waitDrain 轮询 backlog 直到归零或超时,返回是否排空。
func waitDrain(ctx context.Context, pool *pgxpool.Pool, ids []int64, timeout time.Duration) bool {
deadline := time.Now().Add(timeout)
for {
if backlog(ctx, pool, ids) == 0 {
return true
}
if time.Now().After(deadline) {
return false
}
time.Sleep(50 * time.Millisecond)
}
}
func sampleGetDifference(t *testing.T, ctx context.Context, events *postgres.UpdateEventStore, ids []int64) []time.Duration {
t.Helper()
out := make([]time.Duration, 0, diffSampleGoal)
for i := 0; len(out) < diffSampleGoal; i++ {
id := ids[i%len(ids)]
t0 := time.Now()
if _, err := events.ListAfter(ctx, id, 0, 100); err != nil {
t.Fatalf("getDifference ListAfter user %d: %v", id, err)
}
out = append(out, time.Since(t0))
}
return out
}
func percentile(sorted []time.Duration, p float64) time.Duration {
if len(sorted) == 0 {
return 0
}
idx := int(math.Ceil(p/100*float64(len(sorted)))) - 1
if idx < 0 {
idx = 0
}
if idx >= len(sorted) {
idx = len(sorted) - 1
}
return sorted[idx]
}
func pass(ok bool) string {
if ok {
return "PASS"
}
return "WARN"
}
func envInt(key string, def int) int {
if v := os.Getenv(key); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return def
}
func envDuration(key string, def time.Duration) time.Duration {
if v := os.Getenv(key); v != "" {
if d, err := time.ParseDuration(v); err == nil {
return d
}
}
return def
}
func randomUint64(t *testing.T) uint64 {
t.Helper()
var b [8]byte
if _, err := rand.Read(b[:]); err != nil {
t.Fatalf("rand: %v", err)
}
return binary.LittleEndian.Uint64(b[:])
}