feat: sync durable moderation and appeals
This commit is contained in:
iamxvbaba
parent
e1a95c7318
commit
9f467f4be7
140 changed files with 13730 additions and 316 deletions
|
|
@ -67,6 +67,19 @@ type LoginCodeDeliveryRetentionStore interface {
|
|||
DeleteExpiredLoginCodeDeliveries(ctx context.Context, expiredBefore time.Time, limit int) (int, error)
|
||||
}
|
||||
|
||||
type ClientTelemetryRetentionStore interface {
|
||||
DeleteExpiredClientTelemetry(ctx context.Context, olderThan time.Time, limit int) (int, error)
|
||||
}
|
||||
|
||||
type AuthDeliveryReportRetentionStore interface {
|
||||
DeleteExpiredAuthDeliveryReports(ctx context.Context, olderThan time.Time, limit int) (int, error)
|
||||
}
|
||||
|
||||
type ModerationRetentionStore interface {
|
||||
DeleteExpiredSponsoredMessageImpressions(ctx context.Context, olderThan time.Time, limit int) (int, error)
|
||||
DeleteExpiredModerationAppealLinks(ctx context.Context, olderThan time.Time, limit int) (int, error)
|
||||
}
|
||||
|
||||
// botAPIConfirmedGrace 是已确认 Bot API update 行的删除宽限：确认水位之下的行不会再被
|
||||
// getUpdates 读取（fromID 恒 > confirmed），宽限仅防御 offset 回拨调试；回收目标是清堆积。
|
||||
const botAPIConfirmedGrace = 15 * time.Minute
|
||||
|
|
@ -92,24 +105,29 @@ const (
|
|||
// 前缀；落后或缺 state 的任一设备都会把 floor 压回 0。客户端偶然带回已确认前的旧 pts 时，
|
||||
// updates 服务通过普通 differenceSlice checkpoint 推进，不发送 differenceTooLong。
|
||||
type RetentionWorker struct {
|
||||
outbox DispatchOutboxRetentionStore
|
||||
tempKeys TempAuthKeyRetentionStore // 可为 nil（不回收 temp key 绑定）
|
||||
authKeySessionLayers AuthKeySessionLayerRetentionStore
|
||||
botAPIUpdates BotAPIUpdateRetentionStore // 可为 nil（不回收 Bot API 队列）
|
||||
userUpdates UserUpdateEventRetentionStore
|
||||
channelUpdates ChannelUpdateEventRetentionStore
|
||||
loginCodeDeliveries LoginCodeDeliveryRetentionStore
|
||||
orphanAuthKeys OrphanAuthKeyRetentionStore
|
||||
activeAuthKeys ActiveRawAuthKeyProvider
|
||||
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
|
||||
logger *zap.Logger
|
||||
retention time.Duration
|
||||
botAPIRetention time.Duration
|
||||
orphanRetention time.Duration
|
||||
outboxPoisonRetention time.Duration
|
||||
outboxPoisonInterval time.Duration
|
||||
interval time.Duration
|
||||
batch int
|
||||
outbox DispatchOutboxRetentionStore
|
||||
tempKeys TempAuthKeyRetentionStore // 可为 nil（不回收 temp key 绑定）
|
||||
authKeySessionLayers AuthKeySessionLayerRetentionStore
|
||||
botAPIUpdates BotAPIUpdateRetentionStore // 可为 nil（不回收 Bot API 队列）
|
||||
userUpdates UserUpdateEventRetentionStore
|
||||
channelUpdates ChannelUpdateEventRetentionStore
|
||||
loginCodeDeliveries LoginCodeDeliveryRetentionStore
|
||||
clientTelemetry ClientTelemetryRetentionStore
|
||||
authDeliveryReports AuthDeliveryReportRetentionStore
|
||||
moderation ModerationRetentionStore
|
||||
orphanAuthKeys OrphanAuthKeyRetentionStore
|
||||
activeAuthKeys ActiveRawAuthKeyProvider
|
||||
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
|
||||
logger *zap.Logger
|
||||
retention time.Duration
|
||||
botAPIRetention time.Duration
|
||||
orphanRetention time.Duration
|
||||
clientTelemetryRetention time.Duration
|
||||
authDeliveryReportRetention time.Duration
|
||||
outboxPoisonRetention time.Duration
|
||||
outboxPoisonInterval time.Duration
|
||||
interval time.Duration
|
||||
batch int
|
||||
}
|
||||
|
||||
func NewRetentionWorker(outbox DispatchOutboxRetentionStore, tempKeys TempAuthKeyRetentionStore, logger *zap.Logger, retention, interval time.Duration, batch int) *RetentionWorker {
|
||||
|
|
@ -191,6 +209,29 @@ func (w *RetentionWorker) WithAuthKeySessionLayerRetention(store AuthKeySessionL
|
|||
return w
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) WithClientTelemetryRetention(store ClientTelemetryRetentionStore, retention time.Duration) *RetentionWorker {
|
||||
if retention <= 0 {
|
||||
retention = 30 * 24 * time.Hour
|
||||
}
|
||||
w.clientTelemetry = store
|
||||
w.clientTelemetryRetention = retention
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) WithAuthDeliveryReportRetention(store AuthDeliveryReportRetentionStore, retention time.Duration) *RetentionWorker {
|
||||
if retention <= 0 {
|
||||
retention = 30 * 24 * time.Hour
|
||||
}
|
||||
w.authDeliveryReports = store
|
||||
w.authDeliveryReportRetention = retention
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) WithModerationRetention(store ModerationRetentionStore) *RetentionWorker {
|
||||
w.moderation = store
|
||||
return w
|
||||
}
|
||||
|
||||
// WithOrphanAuthKeyRetention 启用未授权握手 key 的有界回收。active 必须提供 raw key，
|
||||
// 不能提供 temp→perm business key；否则未登录或 PFS 连接会被误判为 orphan。
|
||||
func (w *RetentionWorker) WithOrphanAuthKeyRetention(store OrphanAuthKeyRetentionStore, active ActiveRawAuthKeyProvider, retention time.Duration) *RetentionWorker {
|
||||
|
|
@ -274,6 +315,45 @@ func (w *RetentionWorker) runRetentionOnce(ctx context.Context) {
|
|||
w.logger.Info("回收过期 login-code delivery 回执完成", zap.Int("deleted", deleted))
|
||||
}
|
||||
}
|
||||
if w.clientTelemetry != nil {
|
||||
deleted, err := w.clientTelemetry.DeleteExpiredClientTelemetry(
|
||||
ctx, time.Now().Add(-w.clientTelemetryRetention), w.batch,
|
||||
)
|
||||
if err != nil {
|
||||
w.logger.Warn("回收过期客户端 telemetry 失败", zap.Error(err))
|
||||
} else if deleted > 0 {
|
||||
w.logger.Info("回收过期客户端 telemetry 完成", zap.Int("deleted", deleted))
|
||||
}
|
||||
}
|
||||
if w.authDeliveryReports != nil {
|
||||
deleted, err := w.authDeliveryReports.DeleteExpiredAuthDeliveryReports(
|
||||
ctx, time.Now().Add(-w.authDeliveryReportRetention), w.batch,
|
||||
)
|
||||
if err != nil {
|
||||
w.logger.Warn("回收过期验证码投递诊断失败", zap.Error(err))
|
||||
} else if deleted > 0 {
|
||||
w.logger.Info("回收过期验证码投递诊断完成", zap.Int("deleted", deleted))
|
||||
}
|
||||
}
|
||||
if w.moderation != nil {
|
||||
now := time.Now()
|
||||
impressions, err := w.moderation.DeleteExpiredSponsoredMessageImpressions(
|
||||
ctx, now, w.batch,
|
||||
)
|
||||
if err != nil {
|
||||
w.logger.Warn("回收过期 sponsored impression 失败", zap.Error(err))
|
||||
} else if impressions > 0 {
|
||||
w.logger.Info("回收过期 sponsored impression 完成", zap.Int("deleted", impressions))
|
||||
}
|
||||
links, err := w.moderation.DeleteExpiredModerationAppealLinks(
|
||||
ctx, now, w.batch,
|
||||
)
|
||||
if err != nil {
|
||||
w.logger.Warn("回收过期审核申诉链接失败", zap.Error(err))
|
||||
} else if links > 0 {
|
||||
w.logger.Info("回收过期审核申诉链接完成", zap.Int("deleted", links))
|
||||
}
|
||||
}
|
||||
if w.tempKeys != nil {
|
||||
expiredBefore := time.Now().Add(-tempAuthKeyExpiryGrace).Unix()
|
||||
tempDeleted, err := w.tempKeys.DeleteExpired(ctx, expiredBefore, w.batch)
|
||||
|
|
|
|||
|
|
@ -131,6 +131,86 @@ func TestRetentionWorkerReclaimsExpiredLoginCodeDeliveryReceipts(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
type fakeReportRetention struct {
|
||||
telemetryBefore time.Time
|
||||
authBefore time.Time
|
||||
sponsoredBefore time.Time
|
||||
appealBefore time.Time
|
||||
telemetryCalls int
|
||||
authCalls int
|
||||
sponsoredCalls int
|
||||
appealCalls int
|
||||
limit int
|
||||
}
|
||||
|
||||
func (f *fakeReportRetention) DeleteExpiredClientTelemetry(_ context.Context, before time.Time, limit int) (int, error) {
|
||||
f.telemetryCalls++
|
||||
f.telemetryBefore = before
|
||||
f.limit = limit
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (f *fakeReportRetention) DeleteExpiredAuthDeliveryReports(_ context.Context, before time.Time, limit int) (int, error) {
|
||||
f.authCalls++
|
||||
f.authBefore = before
|
||||
f.limit = limit
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (f *fakeReportRetention) DeleteExpiredSponsoredMessageImpressions(_ context.Context, before time.Time, limit int) (int, error) {
|
||||
f.sponsoredCalls++
|
||||
f.sponsoredBefore = before
|
||||
f.limit = limit
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (f *fakeReportRetention) DeleteExpiredModerationAppealLinks(_ context.Context, before time.Time, limit int) (int, error) {
|
||||
f.appealCalls++
|
||||
f.appealBefore = before
|
||||
f.limit = limit
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func TestRetentionWorkerSeparatesTelemetryDiagnosticsAndModerationCapabilities(t *testing.T) {
|
||||
const (
|
||||
telemetryTTL = 7 * 24 * time.Hour
|
||||
authTTL = 14 * 24 * time.Hour
|
||||
batch = 47
|
||||
)
|
||||
store := &fakeReportRetention{}
|
||||
w := NewRetentionWorker(
|
||||
&fakeOutboxRetention{}, nil, zap.NewNop(),
|
||||
168*time.Hour, time.Hour, batch,
|
||||
).WithClientTelemetryRetention(store, telemetryTTL).
|
||||
WithAuthDeliveryReportRetention(store, authTTL).
|
||||
WithModerationRetention(store)
|
||||
before := time.Now()
|
||||
w.runRetentionOnce(context.Background())
|
||||
after := time.Now()
|
||||
if store.telemetryCalls != 1 || store.authCalls != 1 ||
|
||||
store.sponsoredCalls != 1 || store.appealCalls != 1 ||
|
||||
store.limit != batch {
|
||||
t.Fatalf("calls telemetry/auth/sponsored/appeal=%d/%d/%d/%d limit=%d",
|
||||
store.telemetryCalls, store.authCalls,
|
||||
store.sponsoredCalls, store.appealCalls, store.limit)
|
||||
}
|
||||
if store.telemetryBefore.Before(before.Add(-telemetryTTL)) ||
|
||||
store.telemetryBefore.After(after.Add(-telemetryTTL)) {
|
||||
t.Fatalf("telemetry boundary=%v", store.telemetryBefore)
|
||||
}
|
||||
if store.authBefore.Before(before.Add(-authTTL)) ||
|
||||
store.authBefore.After(after.Add(-authTTL)) {
|
||||
t.Fatalf("auth boundary=%v", store.authBefore)
|
||||
}
|
||||
if store.sponsoredBefore.Before(before) ||
|
||||
store.sponsoredBefore.After(after) ||
|
||||
store.appealBefore.Before(before) ||
|
||||
store.appealBefore.After(after) {
|
||||
t.Fatalf("moderation capability boundaries sponsored=%v appeal=%v",
|
||||
store.sponsoredBefore, store.appealBefore)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeBotAPIRetention) DeleteDeliveredOrExpired(_ context.Context, confirmedGrace, maxAge time.Duration, limit int) (int, error) {
|
||||
f.calls++
|
||||
f.confirmedGrace = confirmedGrace
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue