feat: sync multilayer td integration
This commit is contained in:
A
parent
20a310f6ca
commit
766c5db992
491 changed files with 26235 additions and 35340 deletions
|
|
@ -1,11 +1,476 @@
|
|||
package mtprotoedge
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRPCResultCacheFullSessionDoesNotBlockAnotherAuth(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithFairCapacity(func() time.Time { return now }, rpcResultCacheCapacity{
|
||||
maxPending: 8, maxPendingPerAuth: 6,
|
||||
globalMaxEntries: 8, globalMaxBytes: 64,
|
||||
authMaxEntries: 6, authMaxBytes: 48,
|
||||
sessionMaxEntries: 2, sessionMaxBytes: 16,
|
||||
})
|
||||
authA := [8]byte{0xa1}
|
||||
authB := [8]byte{0xb1}
|
||||
const sessionA = int64(77)
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
msgID := int64(1000 + i)
|
||||
claim, err := cache.Acquire(authA, sessionA, msgID)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("same-session admission %d = %#v, %v", i, claim, err)
|
||||
}
|
||||
cache.Put(authA, sessionA, msgID, &encodedOutboundMessage{body: []byte{1}})
|
||||
}
|
||||
if _, err := cache.Acquire(authA, sessionA, 2000); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("admission beyond session entry limit = %v, want capacity", err)
|
||||
}
|
||||
otherAuth, err := cache.Acquire(authB, 88, 3000)
|
||||
if err != nil || otherAuth.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("other auth blocked by full session: %#v, %v", otherAuth, err)
|
||||
}
|
||||
if !otherAuth.owner.Abort() {
|
||||
t.Fatal("other-auth owner did not abort")
|
||||
}
|
||||
if _, ok := cache.Get(authA, sessionA, 1000); !ok {
|
||||
t.Fatal("session capacity pressure evicted an unexpired result")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheFullAuthDoesNotBlockAnotherAuth(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
|
||||
maxPending: 8, maxPendingPerAuth: 4,
|
||||
globalMaxEntries: 8, globalMaxBytes: 64,
|
||||
authMaxEntries: 2, authMaxBytes: 32,
|
||||
sessionMaxEntries: 2, sessionMaxBytes: 16,
|
||||
})
|
||||
authA := [8]byte{0xa2}
|
||||
authB := [8]byte{0xb2}
|
||||
for i := 0; i < 2; i++ {
|
||||
claim, err := cache.Acquire(authA, int64(10+i), int64(100+i))
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("auth A admission %d = %#v, %v", i, claim, err)
|
||||
}
|
||||
cache.Put(authA, int64(10+i), int64(100+i), &encodedOutboundMessage{body: []byte{1}})
|
||||
}
|
||||
if _, err := cache.Acquire(authA, 12, 102); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("same-auth new session at auth limit = %v, want capacity", err)
|
||||
}
|
||||
other, err := cache.Acquire(authB, 20, 200)
|
||||
if err != nil || other.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("other auth blocked by full auth A: %#v, %v", other, err)
|
||||
}
|
||||
other.owner.Abort()
|
||||
}
|
||||
|
||||
func TestRPCResultCacheAuthAndSessionByteLimitsAreIndependent(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
|
||||
maxPending: 8, maxPendingPerAuth: 6,
|
||||
globalMaxEntries: 10, globalMaxBytes: 10,
|
||||
authMaxEntries: 8, authMaxBytes: 4,
|
||||
sessionMaxEntries: 6, sessionMaxBytes: 2,
|
||||
})
|
||||
authA := [8]byte{0xa4}
|
||||
authB := [8]byte{0xb4}
|
||||
first, err := cache.Acquire(authA, 1, 101)
|
||||
if err != nil || first.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("first owner = %#v, %v", first, err)
|
||||
}
|
||||
cache.Put(authA, 1, 101, &encodedOutboundMessage{body: []byte{1, 2}})
|
||||
if _, err := cache.Acquire(authA, 1, 102); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("same session beyond byte limit = %v, want capacity", err)
|
||||
}
|
||||
second, err := cache.Acquire(authA, 2, 201)
|
||||
if err != nil || second.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("second session owner = %#v, %v", second, err)
|
||||
}
|
||||
cache.Put(authA, 2, 201, &encodedOutboundMessage{body: []byte{3, 4}})
|
||||
if _, err := cache.Acquire(authA, 3, 301); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("same auth beyond byte limit = %v, want capacity", err)
|
||||
}
|
||||
other, err := cache.Acquire(authB, 3, 302)
|
||||
if err != nil || other.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("other auth blocked by auth A byte limit: %#v, %v", other, err)
|
||||
}
|
||||
other.owner.Abort()
|
||||
}
|
||||
|
||||
func TestRPCResultCachePerAuthPendingLimitIsAdditional(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
|
||||
maxPending: 6, maxPendingPerAuth: 2,
|
||||
globalMaxEntries: 12, globalMaxBytes: 64,
|
||||
authMaxEntries: 6, authMaxBytes: 32,
|
||||
sessionMaxEntries: 4, sessionMaxBytes: 16,
|
||||
})
|
||||
authA := [8]byte{0xa3}
|
||||
authB := [8]byte{0xb3}
|
||||
owners := make([]*rpcResultOwnerLease, 0, 3)
|
||||
for i := 0; i < 2; i++ {
|
||||
claim, err := cache.Acquire(authA, int64(i+1), int64(100+i))
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("pending auth A %d = %#v, %v", i, claim, err)
|
||||
}
|
||||
owners = append(owners, claim.owner)
|
||||
}
|
||||
if _, err := cache.Acquire(authA, 3, 103); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("third pending owner for auth A = %v, want capacity", err)
|
||||
}
|
||||
other, err := cache.Acquire(authB, 4, 104)
|
||||
if err != nil || other.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("auth B blocked by auth A pending limit: %#v, %v", other, err)
|
||||
}
|
||||
owners = append(owners, other.owner)
|
||||
for _, owner := range owners {
|
||||
if !owner.Abort() {
|
||||
t.Fatal("pending owner did not abort")
|
||||
}
|
||||
}
|
||||
if usage := cache.fairBudget.authSnapshot(authA); usage != (rpcResultBudgetUsage{}) {
|
||||
t.Fatalf("auth A budget after abort = %#v", usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheFairReservationLifecycleReturnsEveryScope(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithFairCapacity(func() time.Time { return now }, rpcResultCacheCapacity{
|
||||
maxPending: 4, maxPendingPerAuth: 3,
|
||||
globalMaxEntries: 6, globalMaxBytes: 10,
|
||||
authMaxEntries: 5, authMaxBytes: 8,
|
||||
sessionMaxEntries: 3, sessionMaxBytes: 6,
|
||||
})
|
||||
auth := [8]byte{0xc1}
|
||||
|
||||
aborted, err := cache.Acquire(auth, 1, 101)
|
||||
if err != nil || aborted.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("aborted owner = %#v, %v", aborted, err)
|
||||
}
|
||||
if usage := cache.fairBudget.authSnapshot(auth); usage.entries != 1 || usage.bytes != 1 || usage.pending != 1 {
|
||||
t.Fatalf("pending auth reservation = %#v", usage)
|
||||
}
|
||||
if !aborted.owner.Abort() {
|
||||
t.Fatal("owner Abort lost")
|
||||
}
|
||||
if usage := cache.fairBudget.authSnapshot(auth); usage != (rpcResultBudgetUsage{}) {
|
||||
t.Fatalf("Abort leaked auth reservation %#v", usage)
|
||||
}
|
||||
|
||||
body, err := cache.Acquire(auth, 1, 102)
|
||||
if err != nil || body.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("body owner = %#v, %v", body, err)
|
||||
}
|
||||
cache.Put(auth, 1, 102, &encodedOutboundMessage{body: make([]byte, 4)})
|
||||
if usage := cache.fairBudget.sessionSnapshot(auth, 1); usage.entries != 1 || usage.bytes != 4 || usage.pending != 0 {
|
||||
t.Fatalf("body session reservation = %#v", usage)
|
||||
}
|
||||
|
||||
tombstone, err := cache.Acquire(auth, 2, 201)
|
||||
if err != nil || tombstone.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("tombstone owner = %#v, %v", tombstone, err)
|
||||
}
|
||||
// This cannot fit the 10-byte global or 8-byte auth ceiling. Put must not
|
||||
// panic or lose ownership; it transfers the one-byte token to a tombstone.
|
||||
cache.Put(auth, 2, 201, &encodedOutboundMessage{body: make([]byte, 20)})
|
||||
if usage := cache.fairBudget.sessionSnapshot(auth, 2); usage.entries != 1 || usage.bytes != 1 || usage.pending != 0 {
|
||||
t.Fatalf("tombstone session reservation = %#v", usage)
|
||||
}
|
||||
if got := cache.completedEntries.snapshot(); got != 2 {
|
||||
t.Fatalf("global entries after body+tombstone = %d, want 2", got)
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != 5 {
|
||||
t.Fatalf("global bytes after body+tombstone = %d, want 5", got)
|
||||
}
|
||||
|
||||
cache.Put(auth, 1, 102, &encodedOutboundMessage{body: make([]byte, 2)})
|
||||
if got := cache.completedBytes.snapshot(); got != 3 {
|
||||
t.Fatalf("replacement did not resize global bytes: %d", got)
|
||||
}
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
_, _ = cache.Get(auth, 1, 102)
|
||||
_, _ = cache.Get(auth, 2, 201)
|
||||
if got := cache.completedEntries.snapshot(); got != 0 {
|
||||
t.Fatalf("TTL leaked global entries %d", got)
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != 0 {
|
||||
t.Fatalf("TTL leaked global bytes %d", got)
|
||||
}
|
||||
if usage := cache.fairBudget.authSnapshot(auth); usage != (rpcResultBudgetUsage{}) {
|
||||
t.Fatalf("TTL leaked auth reservation %#v", usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheFullKeyMaphashSpreadsOneSession(t *testing.T) {
|
||||
first := newRPCResultCacheWithFlightLimit(time.Now, 64)
|
||||
second := newRPCResultCacheWithFlightLimit(time.Now, 64)
|
||||
auth := [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
|
||||
const sessionID = int64(99)
|
||||
seen := make(map[uint64]struct{})
|
||||
differentInstance := false
|
||||
for msgID := int64(1); msgID <= 256; msgID++ {
|
||||
key := rpcResultCacheKey{authKeyID: auth, sessionID: sessionID, reqMsgID: msgID}
|
||||
firstIndex := first.shardIndex(key)
|
||||
seen[firstIndex] = struct{}{}
|
||||
if firstIndex != second.shardIndex(key) {
|
||||
differentInstance = true
|
||||
}
|
||||
}
|
||||
if len(seen) < rpcResultCacheShards/2 {
|
||||
t.Fatalf("one session used only %d/%d full-key shards", len(seen), rpcResultCacheShards)
|
||||
}
|
||||
if !differentInstance {
|
||||
t.Fatal("two cache instances produced an identical shard stream; seed is not instance-random")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheConcurrentFairReservationsNeverOvercommit(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFairCapacity(time.Now, rpcResultCacheCapacity{
|
||||
maxPending: 24, maxPendingPerAuth: 4,
|
||||
globalMaxEntries: 24, globalMaxBytes: 24,
|
||||
authMaxEntries: 8, authMaxBytes: 8,
|
||||
sessionMaxEntries: 3, sessionMaxBytes: 3,
|
||||
})
|
||||
const callers = 256
|
||||
start := make(chan struct{})
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
owners []*rpcResultOwnerLease
|
||||
)
|
||||
for i := 0; i < callers; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
auth := [8]byte{byte(i % 4)}
|
||||
claim, err := cache.Acquire(auth, int64(i%8), int64(1000+i))
|
||||
if errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
return
|
||||
}
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Errorf("Acquire %d = %#v, %v", i, claim, err)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
owners = append(owners, claim.owner)
|
||||
mu.Unlock()
|
||||
}(i)
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
if got := cache.completedEntries.snapshot(); got > 24 || got != int64(len(owners)) {
|
||||
t.Fatalf("global entry usage=%d owners=%d limit=24", got, len(owners))
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got > 24 || got != int64(len(owners)) {
|
||||
t.Fatalf("global byte usage=%d owners=%d limit=24", got, len(owners))
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
auth := [8]byte{byte(i)}
|
||||
usage := cache.fairBudget.authSnapshot(auth)
|
||||
if usage.entries > 8 || usage.bytes > 8 || usage.pending > 4 {
|
||||
t.Fatalf("auth %d overcommitted: %#v", i, usage)
|
||||
}
|
||||
for sessionID := int64(0); sessionID < 8; sessionID++ {
|
||||
session := cache.fairBudget.sessionSnapshot(auth, sessionID)
|
||||
if session.entries > 3 || session.bytes > 3 {
|
||||
t.Fatalf("auth %d session %d overcommitted: %#v", i, sessionID, session)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, owner := range owners {
|
||||
if !owner.Abort() {
|
||||
t.Fatal("concurrent owner did not abort")
|
||||
}
|
||||
}
|
||||
if cache.completedEntries.snapshot() != 0 || cache.completedBytes.snapshot() != 0 {
|
||||
t.Fatal("concurrent Abort leaked global fair budget")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheConcurrentOwnerPublicationAcrossShards(t *testing.T) {
|
||||
const publications = 256
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithFairCapacity(func() time.Time { return now }, rpcResultCacheCapacity{
|
||||
maxPending: publications, maxPendingPerAuth: 4,
|
||||
globalMaxEntries: publications, globalMaxBytes: publications * 4,
|
||||
authMaxEntries: 4, authMaxBytes: 16,
|
||||
sessionMaxEntries: 1, sessionMaxBytes: 4,
|
||||
})
|
||||
|
||||
type publication struct {
|
||||
auth [8]byte
|
||||
session int64
|
||||
msgID int64
|
||||
owner *rpcResultOwnerLease
|
||||
}
|
||||
publicationsByKey := make([]publication, 0, publications)
|
||||
for i := 0; i < publications; i++ {
|
||||
auth := [8]byte{byte(i), byte(i >> 8), 0xa5}
|
||||
sessionID := int64(10_000 + i)
|
||||
msgID := int64(20_000 + i)
|
||||
claim, err := cache.Acquire(auth, sessionID, msgID)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("Acquire %d = %#v, %v", i, claim, err)
|
||||
}
|
||||
publicationsByKey = append(publicationsByKey, publication{
|
||||
auth: auth, session: sessionID, msgID: msgID, owner: claim.owner,
|
||||
})
|
||||
}
|
||||
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for i := range publicationsByKey {
|
||||
item := publicationsByKey[i]
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
if !item.owner.CompleteExecution(true) {
|
||||
t.Errorf("CompleteExecution(%d) lost owner", item.msgID)
|
||||
return
|
||||
}
|
||||
cache.Put(item.auth, item.session, item.msgID, &encodedOutboundMessage{body: []byte{1, 2, 3, 4}})
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
|
||||
for _, item := range publicationsByKey {
|
||||
encoded, ok := cache.Get(item.auth, item.session, item.msgID)
|
||||
if !ok || encoded == nil || len(encoded.body) != 4 {
|
||||
t.Fatalf("completed publication %d missing: ok=%v encoded=%#v", item.msgID, ok, encoded)
|
||||
}
|
||||
}
|
||||
if got := cache.completedEntries.snapshot(); got != publications {
|
||||
t.Fatalf("completed entries=%d, want %d", got, publications)
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != publications*4 {
|
||||
t.Fatalf("completed bytes=%d, want %d", got, publications*4)
|
||||
}
|
||||
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
cache.expireCompletedResults()
|
||||
if cache.completedEntries.snapshot() != 0 || cache.completedBytes.snapshot() != 0 {
|
||||
t.Fatal("parallel publications leaked fair-budget reservations after TTL")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRPCResultCacheParallelShardPut(b *testing.B) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, rpcResultFlightDefaultMaxPending)
|
||||
var nextWorker atomic.Uint64
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
id := nextWorker.Add(1)
|
||||
auth := [8]byte{
|
||||
byte(id), byte(id >> 8), byte(id >> 16), byte(id >> 24),
|
||||
byte(id >> 32), byte(id >> 40), byte(id >> 48), byte(id >> 56),
|
||||
}
|
||||
sessionID := int64(id)
|
||||
msgID := int64(1_000_000 + id)
|
||||
encoded := &encodedOutboundMessage{body: []byte{1, 2, 3, 4}}
|
||||
for pb.Next() {
|
||||
cache.Put(auth, sessionID, msgID, encoded)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRPCResultCacheEntryReservationTransfersAndReturns(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithCapacity(func() time.Time { return now }, 4, 2, 2)
|
||||
authKeyID := [8]byte{0xa2}
|
||||
|
||||
first, err := cache.Acquire(authKeyID, 1, 101)
|
||||
if err != nil || first.state != rpcResultAcquireOwner || cache.completedEntries.snapshot() != 1 {
|
||||
t.Fatalf("first pending reservation = %#v entries=%d err=%v", first, cache.completedEntries.snapshot(), err)
|
||||
}
|
||||
cache.Put(authKeyID, 1, 101, &encodedOutboundMessage{body: []byte{1}})
|
||||
if got := cache.completedEntries.snapshot(); got != 1 {
|
||||
t.Fatalf("pending -> body changed entry count to %d", got)
|
||||
}
|
||||
|
||||
second, err := cache.Acquire(authKeyID, 2, 202)
|
||||
if err != nil || second.state != rpcResultAcquireOwner || cache.completedEntries.snapshot() != 2 {
|
||||
t.Fatalf("second pending reservation = %#v entries=%d err=%v", second, cache.completedEntries.snapshot(), err)
|
||||
}
|
||||
// The byte budget has only the second owner's one-byte token remaining.
|
||||
// Publication therefore leaves an identity tombstone, which still owns its
|
||||
// real process-wide entry slot.
|
||||
cache.Put(authKeyID, 2, 202, &encodedOutboundMessage{body: []byte{2, 2, 2}})
|
||||
if got := cache.completedEntries.snapshot(); got != 2 {
|
||||
t.Fatalf("pending -> tombstone changed entry count to %d", got)
|
||||
}
|
||||
if _, err := cache.Acquire(authKeyID, 3, 303); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("third admission at entry limit = %v, want capacity", err)
|
||||
}
|
||||
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
firstShard := cache.shardIndex(rpcResultCacheKey{authKeyID: authKeyID, sessionID: 1, reqMsgID: 101})
|
||||
secondShard := cache.shardIndex(rpcResultCacheKey{authKeyID: authKeyID, sessionID: 2, reqMsgID: 202})
|
||||
thirdMsgID := rpcResultTestMsgIDOutsideShards(t, cache, authKeyID, 3, 303, firstShard, secondShard)
|
||||
third, err := cache.Acquire(authKeyID, 3, thirdMsgID)
|
||||
if err != nil || third.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("admission after global expiry reap = %#v, %v", third, err)
|
||||
}
|
||||
if got := cache.completedEntries.snapshot(); got != 1 {
|
||||
t.Fatalf("expired entries were not returned before new owner: %d", got)
|
||||
}
|
||||
if !third.owner.Abort() || cache.completedEntries.snapshot() != 0 {
|
||||
t.Fatalf("Abort did not return entry reservation: entries=%d", cache.completedEntries.snapshot())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheConcurrentGlobalEntryReservationNeverOvercommits(t *testing.T) {
|
||||
const limit = 8
|
||||
cache := newRPCResultCacheWithCapacity(time.Now, 128, 1<<20, limit)
|
||||
authKeyID := [8]byte{0xa3}
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
owners []*rpcResultOwnerLease
|
||||
)
|
||||
for i := 0; i < 64; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
claim, err := cache.Acquire(authKeyID, int64(i+1), int64(1000+i))
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Errorf("Acquire %d: %v", i, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if claim.state != rpcResultAcquireOwner {
|
||||
t.Errorf("Acquire %d state = %d", i, claim.state)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
owners = append(owners, claim.owner)
|
||||
mu.Unlock()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
if len(owners) != limit || cache.completedEntries.snapshot() != limit {
|
||||
t.Fatalf("concurrent owners=%d entries=%d, want %d", len(owners), cache.completedEntries.snapshot(), limit)
|
||||
}
|
||||
for _, owner := range owners {
|
||||
if !owner.Abort() {
|
||||
t.Fatal("reserved owner failed to abort")
|
||||
}
|
||||
}
|
||||
if got := cache.completedEntries.snapshot(); got != 0 {
|
||||
t.Fatalf("entry reservations after abort = %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheRoundTripAndTTL(t *testing.T) {
|
||||
if rpcResultCacheTTL != 331*time.Second {
|
||||
t.Fatalf("replay TTL = %v, want full 300s past + 30s future window + 1s", rpcResultCacheTTL)
|
||||
}
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCache(func() time.Time { return now })
|
||||
|
||||
|
|
@ -17,6 +482,12 @@ func TestRPCResultCacheRoundTripAndTTL(t *testing.T) {
|
|||
t.Fatal("unexpected hit on empty cache")
|
||||
}
|
||||
cache.Put(keyID, 5, 7, encoded)
|
||||
if got := cache.completedEntries.snapshot(); got != 1 {
|
||||
t.Fatalf("direct Put entry reservation = %d, want 1", got)
|
||||
}
|
||||
if usage := cache.fairBudget.sessionSnapshot(keyID, 5); usage.entries != 1 || usage.bytes != 4 || usage.pending != 0 {
|
||||
t.Fatalf("direct Put session reservation = %#v", usage)
|
||||
}
|
||||
|
||||
got, ok := cache.Get(keyID, 5, 7)
|
||||
if !ok {
|
||||
|
|
@ -40,29 +511,320 @@ func TestRPCResultCacheRoundTripAndTTL(t *testing.T) {
|
|||
if _, ok := cache.Get(keyID, 5, 7); ok {
|
||||
t.Fatal("expected expiry after TTL")
|
||||
}
|
||||
if got := cache.completedEntries.snapshot(); got != 0 {
|
||||
t.Fatalf("direct Put expiry left %d entry reservations", got)
|
||||
}
|
||||
if usage := cache.fairBudget.authSnapshot(keyID); usage != (rpcResultBudgetUsage{}) {
|
||||
t.Fatalf("direct Put expiry leaked auth reservation %#v", usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheShardTrim(t *testing.T) {
|
||||
func TestRPCResultCacheDuplicatePutPreservesCompletedExecutionMetadata(t *testing.T) {
|
||||
cache := newRPCResultCacheWithFlightLimit(time.Now, 1)
|
||||
keyID := [8]byte{1, 9, 8, 4}
|
||||
const sessionID, reqMsgID = int64(11), int64(12)
|
||||
claim, err := cache.Acquire(keyID, sessionID, reqMsgID)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("Acquire owner = %#v, %v", claim, err)
|
||||
}
|
||||
if !claim.owner.CompleteExecution(true) || !claim.owner.HandOff() {
|
||||
t.Fatal("complete owner metadata")
|
||||
}
|
||||
first := &encodedOutboundMessage{body: []byte{1, 2, 3, 4}, typeID: 42, reqMsgID: reqMsgID}
|
||||
cache.Put(keyID, sessionID, reqMsgID, first)
|
||||
second := &encodedOutboundMessage{body: []byte{5, 6, 7, 8}, typeID: 42, reqMsgID: reqMsgID}
|
||||
cache.Put(keyID, sessionID, reqMsgID, second)
|
||||
|
||||
replay, err := cache.Acquire(keyID, sessionID, reqMsgID)
|
||||
if err != nil || replay.state != rpcResultAcquireCompleted || replay.encoded != second ||
|
||||
!replay.executionKnown || !replay.executionOK {
|
||||
t.Fatalf("duplicate Put metadata = %#v, err=%v", replay, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheShardCapacityNeverEvictsUnexpiredResult(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCache(func() time.Time { return now })
|
||||
|
||||
var keyID [8]byte
|
||||
// 同一 (auth_key, session) 固定落在同一 shard；塞超过单 shard 条数上限，最旧的被逐出。
|
||||
perShard := rpcResultCacheMaxEntries / rpcResultCacheShards
|
||||
for i := 0; i < perShard+1; i++ {
|
||||
cache.Put(keyID, 1, int64(100+i), &encodedOutboundMessage{body: []byte{byte(i)}})
|
||||
firstKey := rpcResultCacheKey{authKeyID: keyID, sessionID: 1, reqMsgID: 100}
|
||||
shard := cache.shard(firstKey)
|
||||
shard.mu.Lock()
|
||||
shard.maxEntries = 1
|
||||
shard.mu.Unlock()
|
||||
|
||||
claim, err := cache.Acquire(keyID, 1, 100)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("first admission = %#v, %v", claim, err)
|
||||
}
|
||||
if _, ok := cache.Get(keyID, 1, 100); ok {
|
||||
t.Fatal("oldest entry should have been evicted by per-shard entry limit")
|
||||
first := &encodedOutboundMessage{body: []byte{1}}
|
||||
cache.Put(keyID, 1, 100, first)
|
||||
secondMsgID := rpcResultTestMsgIDForShard(t, cache, keyID, 1, 101, cache.shardIndex(firstKey))
|
||||
if _, err := cache.Acquire(keyID, 1, secondMsgID); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("full-shard admission = %v, want capacity", err)
|
||||
}
|
||||
if _, ok := cache.Get(keyID, 1, int64(100+perShard)); !ok {
|
||||
t.Fatal("newest entry should survive")
|
||||
if got, ok := cache.Get(keyID, 1, 100); !ok || got != first {
|
||||
t.Fatalf("unexpired first result was displaced: got=%p ok=%v", got, ok)
|
||||
}
|
||||
|
||||
// 单条超过单 shard 字节预算的结果不入缓存。
|
||||
huge := &encodedOutboundMessage{body: make([]byte, rpcResultCacheMaxBytes/rpcResultCacheShards+1)}
|
||||
cache.Put(keyID, 2, 999, huge)
|
||||
if _, ok := cache.Get(keyID, 2, 999); ok {
|
||||
t.Fatal("oversized entry should be rejected")
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
claim, err = cache.Acquire(keyID, 1, secondMsgID)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("admission after expiry = %#v, %v", claim, err)
|
||||
}
|
||||
claim.owner.Abort()
|
||||
}
|
||||
|
||||
func TestRPCResultCacheGlobalByteCapacityNeverEvictsUnexpiredResults(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithLimits(func() time.Time { return now }, 32, 10)
|
||||
var keyID [8]byte
|
||||
|
||||
// Five two-byte results consume the global budget. The sixth admission must
|
||||
// fail bounded; none of the retained results may be sacrificed for it.
|
||||
for sessionID := int64(1); sessionID <= 5; sessionID++ {
|
||||
claim, err := cache.Acquire(keyID, sessionID, 100+sessionID)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("admission %d = %#v, %v", sessionID, claim, err)
|
||||
}
|
||||
cache.Put(keyID, sessionID, 100+sessionID, &encodedOutboundMessage{body: []byte{1, 2}})
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != 10 {
|
||||
t.Fatalf("completed bytes at capacity = %d, want 10", got)
|
||||
}
|
||||
if _, err := cache.Acquire(keyID, 6, 106); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("byte-full admission = %v, want capacity", err)
|
||||
}
|
||||
for sessionID := int64(1); sessionID <= 5; sessionID++ {
|
||||
if _, ok := cache.Get(keyID, sessionID, 100+sessionID); !ok {
|
||||
t.Fatalf("unexpired result %d was evicted", sessionID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheByteBudgetReturnsOnReplaceExpiryAndCapacity(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithLimits(func() time.Time { return now }, 32, 32)
|
||||
var keyID [8]byte
|
||||
|
||||
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 4)})
|
||||
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 7)})
|
||||
if got := cache.completedBytes.snapshot(); got != 7 {
|
||||
t.Fatalf("completed bytes after growing replacement = %d, want 7", got)
|
||||
}
|
||||
if usage := cache.fairBudget.sessionSnapshot(keyID, 1); usage.entries != 1 || usage.bytes != 7 {
|
||||
t.Fatalf("replacement fair reservation after growth = %#v", usage)
|
||||
}
|
||||
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: make([]byte, 2)})
|
||||
if got := cache.completedBytes.snapshot(); got != 2 {
|
||||
t.Fatalf("completed bytes after shrinking replacement = %d, want 2", got)
|
||||
}
|
||||
if usage := cache.fairBudget.sessionSnapshot(keyID, 1); usage.entries != 1 || usage.bytes != 2 {
|
||||
t.Fatalf("replacement fair reservation after shrink = %#v", usage)
|
||||
}
|
||||
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
if _, ok := cache.Get(keyID, 1, 101); ok {
|
||||
t.Fatal("replacement should expire")
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != 0 {
|
||||
t.Fatalf("completed bytes after expiry = %d, want 0", got)
|
||||
}
|
||||
|
||||
key := rpcResultCacheKey{authKeyID: keyID, sessionID: 2, reqMsgID: 201}
|
||||
shard := cache.shard(key)
|
||||
shard.mu.Lock()
|
||||
shard.maxEntries = 1
|
||||
shard.mu.Unlock()
|
||||
claim, err := cache.Acquire(keyID, 2, 201)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("entry-capacity first admission = %#v, %v", claim, err)
|
||||
}
|
||||
cache.Put(keyID, 2, 201, &encodedOutboundMessage{body: make([]byte, 3)})
|
||||
secondMsgID := rpcResultTestMsgIDForShard(t, cache, keyID, 2, 202, cache.shardIndex(key))
|
||||
if _, err := cache.Acquire(keyID, 2, secondMsgID); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("entry-capacity second admission = %v", err)
|
||||
}
|
||||
if got := cache.completedBytes.snapshot(); got != 3 {
|
||||
t.Fatalf("completed bytes after capacity rejection = %d, want 3", got)
|
||||
}
|
||||
if _, ok := cache.Get(keyID, 2, 201); !ok {
|
||||
t.Fatal("capacity rejection displaced the first result")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCachePublicationOverflowLeavesReplayCapacityTombstone(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithLimits(func() time.Time { return now }, 32, 4)
|
||||
var keyID [8]byte
|
||||
claim, err := cache.Acquire(keyID, 1, 101)
|
||||
if err != nil || claim.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("owner admission = %#v, %v", claim, err)
|
||||
}
|
||||
claim.owner.CompleteExecution(true)
|
||||
tooLarge := &encodedOutboundMessage{body: make([]byte, 5)}
|
||||
cache.Put(keyID, 1, 101, tooLarge)
|
||||
if got := cache.completedBytes.snapshot(); got != 1 {
|
||||
t.Fatalf("tombstone bytes = %d, want 1", got)
|
||||
}
|
||||
if _, ok := cache.Get(keyID, 1, 101); ok {
|
||||
t.Fatal("capacity tombstone must not masquerade as a replayable body")
|
||||
}
|
||||
if _, err := cache.Acquire(keyID, 1, 101); !errors.Is(err, ErrRPCResultFlightCapacity) {
|
||||
t.Fatalf("duplicate after publication overflow = %v, want capacity", err)
|
||||
}
|
||||
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
retry, err := cache.Acquire(keyID, 1, 101)
|
||||
if err != nil || retry.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("admission after tombstone expiry = %#v, %v", retry, err)
|
||||
}
|
||||
retry.owner.Abort()
|
||||
}
|
||||
|
||||
func TestRPCResultCacheByteCapacityReclaimsExpiredAcrossShards(t *testing.T) {
|
||||
now := time.Unix(1000, 0)
|
||||
cache := newRPCResultCacheWithLimits(func() time.Time { return now }, 32, 2)
|
||||
var keyID [8]byte
|
||||
|
||||
first, err := cache.Acquire(keyID, 1, 101)
|
||||
if err != nil || first.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("first admission = %#v, %v", first, err)
|
||||
}
|
||||
cache.Put(keyID, 1, 101, &encodedOutboundMessage{body: []byte{1, 2}})
|
||||
if got := cache.completedBytes.snapshot(); got != 2 {
|
||||
t.Fatalf("full budget = %d, want 2", got)
|
||||
}
|
||||
|
||||
// Select a key in another full-key shard. Its failed one-byte reservation
|
||||
// must trigger the cold-path global expiry reap before returning capacity.
|
||||
now = now.Add(rpcResultCacheTTL + time.Second)
|
||||
firstKey := rpcResultCacheKey{authKeyID: keyID, sessionID: 1, reqMsgID: 101}
|
||||
secondMsgID := rpcResultTestMsgIDOutsideShard(t, cache, keyID, 2, 202, cache.shardIndex(firstKey))
|
||||
second, err := cache.Acquire(keyID, 2, secondMsgID)
|
||||
if err != nil || second.state != rpcResultAcquireOwner {
|
||||
t.Fatalf("cross-shard admission after expiry = %#v, %v", second, err)
|
||||
}
|
||||
second.owner.Abort()
|
||||
if got := cache.completedBytes.snapshot(); got != 0 {
|
||||
t.Fatalf("bytes after expired reap and abort = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheServerOptionsPropagateFairLimits(t *testing.T) {
|
||||
sessionBytes := int64(maxOutboundBodyBytes)
|
||||
s := New(Options{
|
||||
RPCGlobalMaxTasks: 6,
|
||||
RPCResultCacheMaxEntries: 12,
|
||||
RPCResultCacheMaxBytes: sessionBytes + 2048,
|
||||
RPCResultCacheAuthMaxEntries: 8,
|
||||
RPCResultCacheAuthMaxBytes: sessionBytes + 1024,
|
||||
RPCResultCacheSessionMaxEntries: 4,
|
||||
RPCResultCacheSessionMaxBytes: sessionBytes,
|
||||
RPCResultPendingPerAuth: 3,
|
||||
})
|
||||
if s.rpcResults.completedEntries.max != 12 || s.rpcResults.completedBytes.max != sessionBytes+2048 {
|
||||
t.Fatalf("global option propagation = %d/%d", s.rpcResults.completedEntries.max, s.rpcResults.completedBytes.max)
|
||||
}
|
||||
budget := s.rpcResults.fairBudget
|
||||
if budget.authLimit.entries != 8 || budget.authLimit.bytes != sessionBytes+1024 ||
|
||||
budget.sessionLimit.entries != 4 || budget.sessionLimit.bytes != sessionBytes || budget.pendingPerAuth != 3 {
|
||||
t.Fatalf("fair option propagation = auth:%#v session:%#v pending:%d",
|
||||
budget.authLimit, budget.sessionLimit, budget.pendingPerAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPCResultCacheServerOptionsFailFast(t *testing.T) {
|
||||
base := Options{
|
||||
RPCGlobalMaxTasks: 6,
|
||||
RPCResultCacheMaxEntries: 12,
|
||||
RPCResultCacheMaxBytes: 64 << 20,
|
||||
RPCResultCacheAuthMaxEntries: 8,
|
||||
RPCResultCacheAuthMaxBytes: 32 << 20,
|
||||
RPCResultCacheSessionMaxEntries: 4,
|
||||
RPCResultCacheSessionMaxBytes: 16 << 20,
|
||||
RPCResultPendingPerAuth: 3,
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*Options)
|
||||
}{
|
||||
{name: "entry hierarchy", mutate: func(o *Options) { o.RPCResultCacheAuthMaxEntries = 13 }},
|
||||
{name: "body does not fit session", mutate: func(o *Options) { o.RPCResultCacheSessionMaxBytes = maxOutboundBodyBytes - 1 }},
|
||||
{name: "pending hierarchy", mutate: func(o *Options) { o.RPCResultPendingPerAuth = 7 }},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
opts := base
|
||||
test.mutate(&opts)
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("New accepted invalid rpc_result cache options")
|
||||
}
|
||||
}()
|
||||
_ = New(opts)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func rpcResultTestMsgIDForShard(
|
||||
t *testing.T,
|
||||
cache *rpcResultCache,
|
||||
authKeyID [8]byte,
|
||||
sessionID, start int64,
|
||||
target uint64,
|
||||
) int64 {
|
||||
t.Helper()
|
||||
for msgID := start; msgID < start+1_000_000; msgID++ {
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: msgID}
|
||||
if cache.shardIndex(key) == target {
|
||||
return msgID
|
||||
}
|
||||
}
|
||||
t.Fatal("failed to find rpc_result key for target shard")
|
||||
return 0
|
||||
}
|
||||
|
||||
func rpcResultTestMsgIDOutsideShard(
|
||||
t *testing.T,
|
||||
cache *rpcResultCache,
|
||||
authKeyID [8]byte,
|
||||
sessionID, start int64,
|
||||
excluded uint64,
|
||||
) int64 {
|
||||
t.Helper()
|
||||
for msgID := start; msgID < start+1_000_000; msgID++ {
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: msgID}
|
||||
if cache.shardIndex(key) != excluded {
|
||||
return msgID
|
||||
}
|
||||
}
|
||||
t.Fatal("failed to find rpc_result key outside excluded shard")
|
||||
return 0
|
||||
}
|
||||
|
||||
func rpcResultTestMsgIDOutsideShards(
|
||||
t *testing.T,
|
||||
cache *rpcResultCache,
|
||||
authKeyID [8]byte,
|
||||
sessionID, start int64,
|
||||
excluded ...uint64,
|
||||
) int64 {
|
||||
t.Helper()
|
||||
for msgID := start; msgID < start+1_000_000; msgID++ {
|
||||
key := rpcResultCacheKey{authKeyID: authKeyID, sessionID: sessionID, reqMsgID: msgID}
|
||||
index := cache.shardIndex(key)
|
||||
allowed := true
|
||||
for _, blocked := range excluded {
|
||||
if index == blocked {
|
||||
allowed = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allowed {
|
||||
return msgID
|
||||
}
|
||||
}
|
||||
t.Fatal("failed to find rpc_result key outside excluded shards")
|
||||
return 0
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue