perf: sync protocol and core hardening updates
This commit is contained in:
A
parent
152fed3b87
commit
4390ebf5a9
283 changed files with 29231 additions and 2295 deletions
|
|
@ -17,12 +17,49 @@ type TempAuthKeyRetentionStore interface {
|
|||
DeleteExpired(ctx context.Context, expiredBefore int64, limit int) (int, error)
|
||||
}
|
||||
|
||||
// OrphanAuthKeyRetentionStore 回收从未形成授权/temp binding 的旧握手 key。
|
||||
// protected 是当前连接注册表实际使用的 raw auth_key_id 快照。
|
||||
type OrphanAuthKeyRetentionStore interface {
|
||||
DeleteOrphaned(ctx context.Context, olderThan time.Duration, limit int, protected [][8]byte) (int, error)
|
||||
}
|
||||
|
||||
type ActiveRawAuthKeyProvider interface {
|
||||
ActiveRawAuthKeyIDs() [][8]byte
|
||||
}
|
||||
|
||||
// ActiveAuthKeyHeartbeatStore 把本实例仍在使用的 raw auth key 活性持久化。多实例下
|
||||
// orphan GC 不能只看当前进程的 active 快照；其它实例的 heartbeat 会推进数据库
|
||||
// last_used_at，使它们不会被误判为孤儿。
|
||||
type ActiveAuthKeyHeartbeatStore interface {
|
||||
TouchActiveRawAuthKeys(ctx context.Context, ids [][8]byte) error
|
||||
}
|
||||
|
||||
// BotAPIUpdateRetentionStore 回收 Bot API getUpdates 投递队列的死行（性能审计 H1）：
|
||||
// 已确认且超过宽限期的行 + 按消息 date 超过保留期的行（官方 Bot API updates 最多保留 24h）。
|
||||
type BotAPIUpdateRetentionStore interface {
|
||||
DeleteDeliveredOrExpired(ctx context.Context, confirmedGrace, maxAge time.Duration, limit int) (int, error)
|
||||
}
|
||||
|
||||
// UserUpdateEventRetentionStore 只回收所有当前授权设备都明确确认过的账号事件前缀。
|
||||
// 它不是普通 TTL：任一授权缺 state 时确认水位为 0，不得删除该设备可能仍需的事件。
|
||||
type UserUpdateEventRetentionStore interface {
|
||||
DeleteConfirmedPrefix(ctx context.Context, olderThan time.Duration, limit int) (int, error)
|
||||
}
|
||||
|
||||
// ChannelUpdateEventRetentionStore 回收超过保留期的 channel durable update 连续前缀。
|
||||
// 具体 store 必须在同一事务内删除事件并推进 retained floor，低于 floor 的客户端由
|
||||
// updates.getChannelDifference 走 channelDifferenceTooLong 快照恢复。
|
||||
type ChannelUpdateEventRetentionStore interface {
|
||||
DeleteExpiredChannelUpdateEvents(ctx context.Context, olderThan time.Duration, limit int) (int, error)
|
||||
}
|
||||
|
||||
// LoginCodeDeliveryRetentionStore reclaims only compact idempotency receipts
|
||||
// after their associated opaque code lifetime. It must not delete the message,
|
||||
// durable update event, or outbox facts created by the delivery transaction.
|
||||
type LoginCodeDeliveryRetentionStore interface {
|
||||
DeleteExpiredLoginCodeDeliveries(ctx context.Context, expiredBefore time.Time, limit int) (int, error)
|
||||
}
|
||||
|
||||
// botAPIConfirmedGrace 是已确认 Bot API update 行的删除宽限：确认水位之下的行不会再被
|
||||
// getUpdates 读取（fromID 恒 > confirmed），宽限仅防御 offset 回拨调试；回收目标是清堆积。
|
||||
const botAPIConfirmedGrace = 15 * time.Minute
|
||||
|
|
@ -32,23 +69,39 @@ const botAPIConfirmedGrace = 15 * time.Minute
|
|||
// 连接；回收目标是清堆积，晚一天无妨。
|
||||
const tempAuthKeyExpiryGrace = 24 * time.Hour
|
||||
|
||||
const (
|
||||
// terminal failed outbox 只承担短期诊断隔离；它不是 durable update log。
|
||||
// 删除该任务会由 head trigger 立即放行同账号下一 pts，而 user_update_events
|
||||
// 继续保留，在线漏推由正常 difference 路径补偿。
|
||||
defaultOutboxPoisonRetention = time.Minute
|
||||
defaultOutboxPoisonInterval = 15 * time.Second
|
||||
)
|
||||
|
||||
// RetentionWorker 周期性回收存储中的死数据。
|
||||
//
|
||||
// 注意：本 worker 刻意不清理 user_update_events —— pts log 永久保留。原因：TDesktop 不支持
|
||||
// 账号级 updates.differenceTooLong（api_updates.cpp 收到该响应只打一行日志，且漏掉
|
||||
// setRequesting(false)，会永久锁死整个 update 引擎），服务端因此无法让"落后超过保留期"的
|
||||
// 客户端整库重置；一旦裁剪 events，落后客户端的 getDifference 会拿到不完整的事件链而静默
|
||||
// 丢消息。详见 docs/performance-audit.md 与 docs/compatibility-matrix.md。user_update_events
|
||||
// 长期膨胀作为已知 todo。
|
||||
// 注意：TDesktop 不支持账号级 updates.differenceTooLong（api_updates.cpp 收到该响应只
|
||||
// 记录日志且不清 requesting，会永久锁死 update 引擎），因此绝不能按普通 TTL 硬裁剪
|
||||
// user_update_events。本 worker 只允许 store 删除“所有当前授权设备都明确确认”的连续安全
|
||||
// 前缀；落后或缺 state 的任一设备都会把 floor 压回 0。客户端偶然带回已确认前的旧 pts 时，
|
||||
// updates 服务通过普通 differenceSlice checkpoint 推进，不发送 differenceTooLong。
|
||||
type RetentionWorker struct {
|
||||
outbox DispatchOutboxRetentionStore
|
||||
tempKeys TempAuthKeyRetentionStore // 可为 nil（不回收 temp key 绑定）
|
||||
botAPIUpdates BotAPIUpdateRetentionStore // 可为 nil（不回收 Bot API 队列）
|
||||
logger *zap.Logger
|
||||
retention time.Duration
|
||||
botAPIRetention time.Duration
|
||||
interval time.Duration
|
||||
batch int
|
||||
outbox DispatchOutboxRetentionStore
|
||||
tempKeys TempAuthKeyRetentionStore // 可为 nil（不回收 temp key 绑定）
|
||||
botAPIUpdates BotAPIUpdateRetentionStore // 可为 nil（不回收 Bot API 队列）
|
||||
userUpdates UserUpdateEventRetentionStore
|
||||
channelUpdates ChannelUpdateEventRetentionStore
|
||||
loginCodeDeliveries LoginCodeDeliveryRetentionStore
|
||||
orphanAuthKeys OrphanAuthKeyRetentionStore
|
||||
activeAuthKeys ActiveRawAuthKeyProvider
|
||||
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
|
||||
logger *zap.Logger
|
||||
retention time.Duration
|
||||
botAPIRetention time.Duration
|
||||
orphanRetention time.Duration
|
||||
outboxPoisonRetention time.Duration
|
||||
outboxPoisonInterval time.Duration
|
||||
interval time.Duration
|
||||
batch int
|
||||
}
|
||||
|
||||
func NewRetentionWorker(outbox DispatchOutboxRetentionStore, tempKeys TempAuthKeyRetentionStore, logger *zap.Logger, retention, interval time.Duration, batch int) *RetentionWorker {
|
||||
|
|
@ -65,15 +118,32 @@ func NewRetentionWorker(outbox DispatchOutboxRetentionStore, tempKeys TempAuthKe
|
|||
batch = 10000
|
||||
}
|
||||
return &RetentionWorker{
|
||||
outbox: outbox,
|
||||
tempKeys: tempKeys,
|
||||
logger: logger,
|
||||
retention: retention,
|
||||
interval: interval,
|
||||
batch: batch,
|
||||
outbox: outbox,
|
||||
tempKeys: tempKeys,
|
||||
logger: logger,
|
||||
retention: retention,
|
||||
outboxPoisonRetention: defaultOutboxPoisonRetention,
|
||||
outboxPoisonInterval: defaultOutboxPoisonInterval,
|
||||
interval: interval,
|
||||
batch: batch,
|
||||
}
|
||||
}
|
||||
|
||||
// WithDispatchOutboxPoisonPolicy 配置 terminal failed head 的独立短隔离与清理周期。
|
||||
// 该周期不能复用 durable update 的周级保留期，否则一条确定性构造错误会冻结该
|
||||
// 用户整条在线 pts lane。<=0 分别回退到 1m/15s 的安全默认值。
|
||||
func (w *RetentionWorker) WithDispatchOutboxPoisonPolicy(retention, interval time.Duration) *RetentionWorker {
|
||||
if retention <= 0 {
|
||||
retention = defaultOutboxPoisonRetention
|
||||
}
|
||||
if interval <= 0 {
|
||||
interval = defaultOutboxPoisonInterval
|
||||
}
|
||||
w.outboxPoisonRetention = retention
|
||||
w.outboxPoisonInterval = interval
|
||||
return w
|
||||
}
|
||||
|
||||
// WithBotAPIUpdateRetention 启用 bot_api_updates 队列回收；retention <=0 时用官方语义默认 24h。
|
||||
func (w *RetentionWorker) WithBotAPIUpdateRetention(store BotAPIUpdateRetentionStore, retention time.Duration) *RetentionWorker {
|
||||
if retention <= 0 {
|
||||
|
|
@ -84,26 +154,102 @@ func (w *RetentionWorker) WithBotAPIUpdateRetention(store BotAPIUpdateRetentionS
|
|||
return w
|
||||
}
|
||||
|
||||
// WithUserUpdateRetention 启用账号 update 的共同确认安全前缀回收。TDesktop 不支持
|
||||
// account differenceTooLong，具体 store 必须保证未确认前缀永不删除。
|
||||
func (w *RetentionWorker) WithUserUpdateRetention(store UserUpdateEventRetentionStore) *RetentionWorker {
|
||||
w.userUpdates = store
|
||||
return w
|
||||
}
|
||||
|
||||
// WithChannelUpdateRetention 启用 channel durable update 的有界 TTL 回收；复用 worker 的
|
||||
// retention/interval/batch，并由 store 的 retained floor 保证旧 pts 不会读到静默空洞。
|
||||
func (w *RetentionWorker) WithChannelUpdateRetention(store ChannelUpdateEventRetentionStore) *RetentionWorker {
|
||||
w.channelUpdates = store
|
||||
return w
|
||||
}
|
||||
|
||||
// WithLoginCodeDeliveryRetention enables bounded seek cleanup for compact
|
||||
// phone_code_hash receipts. Each row carries its own expiry derived from the
|
||||
// code TTL, so this cleanup intentionally does not reuse update-log retention.
|
||||
func (w *RetentionWorker) WithLoginCodeDeliveryRetention(store LoginCodeDeliveryRetentionStore) *RetentionWorker {
|
||||
w.loginCodeDeliveries = store
|
||||
return w
|
||||
}
|
||||
|
||||
// WithOrphanAuthKeyRetention 启用未授权握手 key 的有界回收。active 必须提供 raw key，
|
||||
// 不能提供 temp→perm business key；否则未登录或 PFS 连接会被误判为 orphan。
|
||||
func (w *RetentionWorker) WithOrphanAuthKeyRetention(store OrphanAuthKeyRetentionStore, active ActiveRawAuthKeyProvider, retention time.Duration) *RetentionWorker {
|
||||
w.orphanAuthKeys = store
|
||||
w.activeAuthKeys = active
|
||||
w.activeAuthKeyHeartbeat, _ = store.(ActiveAuthKeyHeartbeatStore)
|
||||
w.orphanRetention = retention
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) Run(ctx context.Context) {
|
||||
w.runOnce(ctx)
|
||||
ticker := time.NewTicker(w.interval)
|
||||
defer ticker.Stop()
|
||||
retentionTicker := time.NewTicker(w.interval)
|
||||
defer retentionTicker.Stop()
|
||||
poisonTicker := time.NewTicker(w.outboxPoisonInterval)
|
||||
defer poisonTicker.Stop()
|
||||
var (
|
||||
heartbeatTicker *time.Ticker
|
||||
heartbeatC <-chan time.Time
|
||||
)
|
||||
if interval := w.orphanHeartbeatInterval(); interval > 0 {
|
||||
heartbeatTicker = time.NewTicker(interval)
|
||||
heartbeatC = heartbeatTicker.C
|
||||
defer heartbeatTicker.Stop()
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
w.runOnce(ctx)
|
||||
case <-retentionTicker.C:
|
||||
w.runRetentionOnce(ctx)
|
||||
case <-poisonTicker.C:
|
||||
w.runOutboxPoisonOnce(ctx)
|
||||
case <-heartbeatC:
|
||||
w.heartbeatActiveAuthKeys(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) runOnce(ctx context.Context) {
|
||||
outboxDeleted, err := w.outbox.DeleteFailed(ctx, w.retention, w.batch)
|
||||
w.runOutboxPoisonOnce(ctx)
|
||||
w.runRetentionOnce(ctx)
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) runOutboxPoisonOnce(ctx context.Context) {
|
||||
if w.outbox == nil {
|
||||
return
|
||||
}
|
||||
outboxDeleted, err := w.outbox.DeleteFailed(ctx, w.outboxPoisonRetention, w.batch)
|
||||
if err != nil {
|
||||
w.logger.Warn("清理 failed dispatch_outbox 失败", zap.Error(err))
|
||||
w.logger.Error("清理 terminal failed dispatch_outbox 失败",
|
||||
zap.String("signal", "dispatch_outbox_poison_cleanup_failed"),
|
||||
zap.Duration("quarantine", w.outboxPoisonRetention),
|
||||
zap.Error(err),
|
||||
)
|
||||
} else if outboxDeleted > 0 {
|
||||
w.logger.Info("清理 failed dispatch_outbox 完成", zap.Int("deleted", outboxDeleted))
|
||||
// Error 级结构化信号刻意保留：发生 terminal failed 代表确定性编码、事件缺失
|
||||
// 或其它不可自动重试故障。任务删除只解冻在线 lane，不会删除 durable event。
|
||||
w.logger.Error("terminal failed dispatch_outbox 已结束隔离并释放用户 lane",
|
||||
zap.String("signal", "dispatch_outbox_poison_released"),
|
||||
zap.Int("deleted", outboxDeleted),
|
||||
zap.Duration("quarantine", w.outboxPoisonRetention),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) runRetentionOnce(ctx context.Context) {
|
||||
if w.loginCodeDeliveries != nil {
|
||||
deleted, err := w.loginCodeDeliveries.DeleteExpiredLoginCodeDeliveries(ctx, time.Now(), w.batch)
|
||||
if err != nil {
|
||||
w.logger.Warn("回收过期 login-code delivery 回执失败", zap.Error(err))
|
||||
} else if deleted > 0 {
|
||||
w.logger.Info("回收过期 login-code delivery 回执完成", zap.Int("deleted", deleted))
|
||||
}
|
||||
}
|
||||
if w.tempKeys != nil {
|
||||
expiredBefore := time.Now().Add(-tempAuthKeyExpiryGrace).Unix()
|
||||
|
|
@ -114,6 +260,24 @@ func (w *RetentionWorker) runOnce(ctx context.Context) {
|
|||
w.logger.Info("回收过期 temp auth key 绑定完成", zap.Int("deleted", tempDeleted))
|
||||
}
|
||||
}
|
||||
if w.orphanAuthKeys != nil && w.orphanRetention > 0 {
|
||||
var protected [][8]byte
|
||||
if w.activeAuthKeys != nil {
|
||||
protected = w.activeAuthKeys.ActiveRawAuthKeyIDs()
|
||||
}
|
||||
if !w.touchActiveAuthKeys(ctx, protected) {
|
||||
// Fail safe: if this instance cannot publish its own active set, deleting against a
|
||||
// stale database heartbeat could evict keys used by another instance too. Keep all
|
||||
// candidates for this pass and retry after the next heartbeat.
|
||||
} else {
|
||||
orphanDeleted, err := w.orphanAuthKeys.DeleteOrphaned(ctx, w.orphanRetention, w.batch, protected)
|
||||
if err != nil {
|
||||
w.logger.Warn("回收未授权 orphan auth key 失败", zap.Error(err))
|
||||
} else if orphanDeleted > 0 {
|
||||
w.logger.Info("回收未授权 orphan auth key 完成", zap.Int("deleted", orphanDeleted))
|
||||
}
|
||||
}
|
||||
}
|
||||
if w.botAPIUpdates != nil {
|
||||
botAPIDeleted, err := w.botAPIUpdates.DeleteDeliveredOrExpired(ctx, botAPIConfirmedGrace, w.botAPIRetention, w.batch)
|
||||
if err != nil {
|
||||
|
|
@ -122,4 +286,63 @@ func (w *RetentionWorker) runOnce(ctx context.Context) {
|
|||
w.logger.Info("回收 bot_api_updates 队列完成", zap.Int("deleted", botAPIDeleted))
|
||||
}
|
||||
}
|
||||
if w.userUpdates != nil {
|
||||
userDeleted, err := w.userUpdates.DeleteConfirmedPrefix(ctx, w.retention, w.batch)
|
||||
if err != nil {
|
||||
w.logger.Warn("回收已共同确认的 user_update_events 前缀失败", zap.Error(err))
|
||||
} else if userDeleted > 0 {
|
||||
w.logger.Info("回收已共同确认的 user_update_events 前缀完成", zap.Int("deleted", userDeleted))
|
||||
}
|
||||
}
|
||||
if w.channelUpdates != nil {
|
||||
channelDeleted, err := w.channelUpdates.DeleteExpiredChannelUpdateEvents(ctx, w.retention, w.batch)
|
||||
if err != nil {
|
||||
// store 会逐频道隔离坏 gap 后继续本轮；deleted 可能非零，必须同时记录，
|
||||
// 既不能把全局 pass 伪装成完全失败，也不能吞掉不变量错误。
|
||||
w.logger.Warn("回收过期 channel_update_events 存在隔离频道",
|
||||
zap.Int("deleted", channelDeleted),
|
||||
zap.Error(err),
|
||||
)
|
||||
} else if channelDeleted > 0 {
|
||||
w.logger.Info("回收过期 channel_update_events 连续前缀完成", zap.Int("deleted", channelDeleted))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) orphanHeartbeatInterval() time.Duration {
|
||||
if w.activeAuthKeyHeartbeat == nil || w.activeAuthKeys == nil || w.orphanRetention <= 0 {
|
||||
return 0
|
||||
}
|
||||
interval := w.orphanRetention / 3
|
||||
if interval <= 0 {
|
||||
interval = time.Nanosecond
|
||||
}
|
||||
if w.interval > 0 && w.interval < interval {
|
||||
interval = w.interval
|
||||
}
|
||||
return interval
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) heartbeatActiveAuthKeys(ctx context.Context) {
|
||||
if w.activeAuthKeys == nil {
|
||||
return
|
||||
}
|
||||
w.touchActiveAuthKeys(ctx, w.activeAuthKeys.ActiveRawAuthKeyIDs())
|
||||
}
|
||||
|
||||
// touchActiveAuthKeys returns false only when a configured durable heartbeat failed. A store that
|
||||
// predates the optional heartbeat interface keeps single-instance behavior.
|
||||
func (w *RetentionWorker) touchActiveAuthKeys(ctx context.Context, protected [][8]byte) bool {
|
||||
if w.activeAuthKeyHeartbeat == nil {
|
||||
return true
|
||||
}
|
||||
if err := w.activeAuthKeyHeartbeat.TouchActiveRawAuthKeys(ctx, protected); err != nil {
|
||||
w.logger.Error("刷新 active raw auth key heartbeat 失败，本轮跳过 orphan GC",
|
||||
zap.String("signal", "auth_key_heartbeat_failed"),
|
||||
zap.Int("active_keys", len(protected)),
|
||||
zap.Error(err),
|
||||
)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,19 +2,57 @@ package maintenance
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
"go.uber.org/zap/zaptest/observer"
|
||||
)
|
||||
|
||||
type fakeOutboxRetention struct {
|
||||
calls int
|
||||
calls int
|
||||
olderThan time.Duration
|
||||
limit int
|
||||
deleted int
|
||||
}
|
||||
|
||||
func (f *fakeOutboxRetention) DeleteFailed(context.Context, time.Duration, int) (int, error) {
|
||||
func (f *fakeOutboxRetention) DeleteFailed(_ context.Context, olderThan time.Duration, limit int) (int, error) {
|
||||
f.calls++
|
||||
return 0, nil
|
||||
f.olderThan = olderThan
|
||||
f.limit = limit
|
||||
return f.deleted, nil
|
||||
}
|
||||
|
||||
func TestRetentionWorkerUsesIndependentOutboxPoisonPolicyAndSignalsRelease(t *testing.T) {
|
||||
core, logs := observer.New(zapcore.ErrorLevel)
|
||||
outbox := &fakeOutboxRetention{deleted: 2}
|
||||
w := NewRetentionWorker(outbox, nil, zap.New(core), 7*24*time.Hour, time.Hour, 73).
|
||||
WithDispatchOutboxPoisonPolicy(2*time.Minute, 7*time.Second)
|
||||
|
||||
w.runOnce(context.Background())
|
||||
|
||||
if outbox.calls != 1 || outbox.olderThan != 2*time.Minute || outbox.limit != 73 {
|
||||
t.Fatalf("outbox poison calls/args = %d/%v/%d, want 1/2m/73", outbox.calls, outbox.olderThan, outbox.limit)
|
||||
}
|
||||
entries := logs.FilterMessage("terminal failed dispatch_outbox 已结束隔离并释放用户 lane").All()
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("poison release error signals = %d, want 1", len(entries))
|
||||
}
|
||||
if got := entries[0].ContextMap()["signal"]; got != "dispatch_outbox_poison_released" {
|
||||
t.Fatalf("poison signal = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetentionWorkerOutboxPoisonPolicyDefaultsAreShort(t *testing.T) {
|
||||
outbox := &fakeOutboxRetention{}
|
||||
w := NewRetentionWorker(outbox, nil, zap.NewNop(), 168*time.Hour, time.Hour, 100).
|
||||
WithDispatchOutboxPoisonPolicy(0, 0)
|
||||
w.runOutboxPoisonOnce(context.Background())
|
||||
if outbox.olderThan != defaultOutboxPoisonRetention || w.outboxPoisonInterval != defaultOutboxPoisonInterval {
|
||||
t.Fatalf("default poison policy = %v/%v, want %v/%v", outbox.olderThan, w.outboxPoisonInterval, defaultOutboxPoisonRetention, defaultOutboxPoisonInterval)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeTempKeyRetention struct {
|
||||
|
|
@ -65,6 +103,34 @@ type fakeBotAPIRetention struct {
|
|||
limit int
|
||||
}
|
||||
|
||||
type fakeLoginCodeDeliveryRetention struct {
|
||||
calls int
|
||||
expiredBefore time.Time
|
||||
limit int
|
||||
}
|
||||
|
||||
func (f *fakeLoginCodeDeliveryRetention) DeleteExpiredLoginCodeDeliveries(_ context.Context, expiredBefore time.Time, limit int) (int, error) {
|
||||
f.calls++
|
||||
f.expiredBefore = expiredBefore
|
||||
f.limit = limit
|
||||
return 4, nil
|
||||
}
|
||||
|
||||
func TestRetentionWorkerReclaimsExpiredLoginCodeDeliveryReceipts(t *testing.T) {
|
||||
loginCodes := &fakeLoginCodeDeliveryRetention{}
|
||||
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), 168*time.Hour, time.Hour, 83).
|
||||
WithLoginCodeDeliveryRetention(loginCodes)
|
||||
before := time.Now()
|
||||
w.runRetentionOnce(context.Background())
|
||||
after := time.Now()
|
||||
if loginCodes.calls != 1 || loginCodes.limit != 83 {
|
||||
t.Fatalf("login-code retention calls/limit = %d/%d, want 1/83", loginCodes.calls, loginCodes.limit)
|
||||
}
|
||||
if loginCodes.expiredBefore.Before(before) || loginCodes.expiredBefore.After(after) {
|
||||
t.Fatalf("login-code expiry boundary = %v, want within [%v,%v]", loginCodes.expiredBefore, before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeBotAPIRetention) DeleteDeliveredOrExpired(_ context.Context, confirmedGrace, maxAge time.Duration, limit int) (int, error) {
|
||||
f.calls++
|
||||
f.confirmedGrace = confirmedGrace
|
||||
|
|
@ -99,3 +165,151 @@ func TestRetentionWorkerBotAPIRetentionDefaultsTo24h(t *testing.T) {
|
|||
t.Fatalf("default bot api retention = %v, want 24h", botAPI.maxAge)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeUserUpdateRetention struct {
|
||||
calls int
|
||||
olderThan time.Duration
|
||||
limit int
|
||||
}
|
||||
|
||||
func (f *fakeUserUpdateRetention) DeleteConfirmedPrefix(_ context.Context, olderThan time.Duration, limit int) (int, error) {
|
||||
f.calls++
|
||||
f.olderThan = olderThan
|
||||
f.limit = limit
|
||||
return 9, nil
|
||||
}
|
||||
|
||||
func TestRetentionWorkerReclaimsOnlyConfirmedUserUpdatePrefix(t *testing.T) {
|
||||
const retention = 7 * 24 * time.Hour
|
||||
store := &fakeUserUpdateRetention{}
|
||||
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), retention, time.Hour, 91).
|
||||
WithUserUpdateRetention(store)
|
||||
w.runOnce(context.Background())
|
||||
if store.calls != 1 || store.olderThan != retention || store.limit != 91 {
|
||||
t.Fatalf("user update retention calls/args = %d/%v/%d, want 1/%v/91", store.calls, store.olderThan, store.limit, retention)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeChannelUpdateRetention struct {
|
||||
calls int
|
||||
olderThan time.Duration
|
||||
limit int
|
||||
}
|
||||
|
||||
func (f *fakeChannelUpdateRetention) DeleteExpiredChannelUpdateEvents(_ context.Context, olderThan time.Duration, limit int) (int, error) {
|
||||
f.calls++
|
||||
f.olderThan = olderThan
|
||||
f.limit = limit
|
||||
return 7, nil
|
||||
}
|
||||
|
||||
func TestRetentionWorkerReclaimsChannelUpdates(t *testing.T) {
|
||||
const (
|
||||
retention = 14 * 24 * time.Hour
|
||||
batch = 321
|
||||
)
|
||||
channelUpdates := &fakeChannelUpdateRetention{}
|
||||
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), retention, time.Hour, batch).
|
||||
WithChannelUpdateRetention(channelUpdates)
|
||||
|
||||
w.runOnce(context.Background())
|
||||
|
||||
if channelUpdates.calls != 1 {
|
||||
t.Fatalf("channel update retention calls = %d, want 1", channelUpdates.calls)
|
||||
}
|
||||
if channelUpdates.olderThan != retention || channelUpdates.limit != batch {
|
||||
t.Fatalf("channel update retention args = (%v, %d), want (%v, %d)",
|
||||
channelUpdates.olderThan, channelUpdates.limit, retention, batch)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeOrphanAuthKeyRetention struct {
|
||||
calls int
|
||||
olderThan time.Duration
|
||||
limit int
|
||||
protected [][8]byte
|
||||
}
|
||||
|
||||
func (f *fakeOrphanAuthKeyRetention) DeleteOrphaned(_ context.Context, olderThan time.Duration, limit int, protected [][8]byte) (int, error) {
|
||||
f.calls++
|
||||
f.olderThan = olderThan
|
||||
f.limit = limit
|
||||
f.protected = append([][8]byte(nil), protected...)
|
||||
return 2, nil
|
||||
}
|
||||
|
||||
type fakeActiveRawAuthKeys struct{ ids [][8]byte }
|
||||
|
||||
func (f fakeActiveRawAuthKeys) ActiveRawAuthKeyIDs() [][8]byte {
|
||||
return append([][8]byte(nil), f.ids...)
|
||||
}
|
||||
|
||||
func TestRetentionWorkerProtectsActiveRawAuthKeysFromOrphanGC(t *testing.T) {
|
||||
store := &fakeOrphanAuthKeyRetention{}
|
||||
active := fakeActiveRawAuthKeys{ids: [][8]byte{{1}, {2}}}
|
||||
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 73).
|
||||
WithOrphanAuthKeyRetention(store, active, 24*time.Hour)
|
||||
|
||||
w.runOnce(context.Background())
|
||||
|
||||
if store.calls != 1 || store.olderThan != 24*time.Hour || store.limit != 73 {
|
||||
t.Fatalf("orphan retention calls/args = %d/%v/%d, want 1/24h/73", store.calls, store.olderThan, store.limit)
|
||||
}
|
||||
if len(store.protected) != 2 || store.protected[0] != ([8]byte{1}) || store.protected[1] != ([8]byte{2}) {
|
||||
t.Fatalf("protected raw auth keys = %v, want {1},{2}", store.protected)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeHeartbeatOrphanRetention struct {
|
||||
fakeOrphanAuthKeyRetention
|
||||
heartbeatCalls int
|
||||
heartbeatIDs [][8]byte
|
||||
heartbeatErr error
|
||||
}
|
||||
|
||||
func (f *fakeHeartbeatOrphanRetention) TouchActiveRawAuthKeys(_ context.Context, ids [][8]byte) error {
|
||||
f.heartbeatCalls++
|
||||
f.heartbeatIDs = append([][8]byte(nil), ids...)
|
||||
return f.heartbeatErr
|
||||
}
|
||||
|
||||
func TestRetentionWorkerHeartbeatsActiveKeysBeforeOrphanDelete(t *testing.T) {
|
||||
store := &fakeHeartbeatOrphanRetention{}
|
||||
active := fakeActiveRawAuthKeys{ids: [][8]byte{{3}, {4}}}
|
||||
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, 2*time.Hour, 19).
|
||||
WithOrphanAuthKeyRetention(store, active, 3*time.Hour)
|
||||
|
||||
w.runRetentionOnce(context.Background())
|
||||
|
||||
if store.heartbeatCalls != 1 || store.calls != 1 {
|
||||
t.Fatalf("heartbeat/delete calls = %d/%d, want 1/1", store.heartbeatCalls, store.calls)
|
||||
}
|
||||
if len(store.heartbeatIDs) != 2 || store.heartbeatIDs[0] != ([8]byte{3}) || store.heartbeatIDs[1] != ([8]byte{4}) {
|
||||
t.Fatalf("heartbeat ids = %v, want {3},{4}", store.heartbeatIDs)
|
||||
}
|
||||
// min(retention worker interval=2h, orphan retention/3=1h)
|
||||
if got := w.orphanHeartbeatInterval(); got != time.Hour {
|
||||
t.Fatalf("heartbeat interval = %v, want 1h", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetentionWorkerSkipsOrphanDeleteWhenHeartbeatFails(t *testing.T) {
|
||||
core, logs := observer.New(zapcore.ErrorLevel)
|
||||
store := &fakeHeartbeatOrphanRetention{heartbeatErr: errors.New("db unavailable")}
|
||||
active := fakeActiveRawAuthKeys{ids: [][8]byte{{5}}}
|
||||
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.New(core), time.Hour, 30*time.Minute, 11).
|
||||
WithOrphanAuthKeyRetention(store, active, 24*time.Hour)
|
||||
if got := w.orphanHeartbeatInterval(); got != 30*time.Minute {
|
||||
t.Fatalf("heartbeat interval = %v, want worker interval 30m", got)
|
||||
}
|
||||
|
||||
w.runRetentionOnce(context.Background())
|
||||
|
||||
if store.heartbeatCalls != 1 || store.calls != 0 {
|
||||
t.Fatalf("heartbeat/delete calls = %d/%d, want 1/0", store.heartbeatCalls, store.calls)
|
||||
}
|
||||
entries := logs.FilterMessage("刷新 active raw auth key heartbeat 失败，本轮跳过 orphan GC").All()
|
||||
if len(entries) != 1 || entries[0].ContextMap()["signal"] != "auth_key_heartbeat_failed" {
|
||||
t.Fatalf("heartbeat failure signals = %+v", entries)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue