astra/plc-mirror
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Detect CAR files with zero blocks and handle them accordingly

Browse source
This commit is contained in:
Max Ignatenko 2024-03-17 17:42:09 +00:00
parent 638bdcf515
commit 4c41389e9b
2 changed files with 12 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

8
cmd/record-indexer/workerpool.go
View file

@ -173,7 +173,8 @@ retry:
}
// TODO: add a configuration knob for switching between full and partial fetch.
b, err := comatproto.SyncGetRepo(ctx, client, work.Repo.DID, work.Repo.LastIndexedRev)
sinceRev := work.Repo.LastIndexedRev
b, err := comatproto.SyncGetRepo(ctx, client, work.Repo.DID, sinceRev)
if err != nil {
if err, ok := errors.As[*xrpc.Error](err); ok {
if err.IsThrottled() && err.Ratelimit != nil {
@ -204,7 +205,10 @@ retry:
}
newRev, err := repo.GetRev(ctx, bytes.NewReader(b))
if err != nil {
if sinceRev != "" && errors.Is(err, repo.ErrZeroBlocks) {
// No new records since the rev we requested above.
return nil
} else if err != nil {
l := 25
if len(b) < l {
l = len(b)