Handle repos with unknown PDS

cmd/record-indexer/scheduler.go

@@ -106,6 +106,9 @@ func (s *Scheduler) fillQueue(ctx context.Context) error {
 		perPDSLimit = maxQueueLen * 2 / len(remotes)
 	}
 
+	// Fake remote to account for repos we didn't have a PDS for yet.
+	remotes = append(remotes, pds.PDS{ID: pds.Unknown})
+
 	for _, remote := range remotes {
 		repos := []repo.Repo{}
 

cmd/record-indexer/workerpool.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/uabluerail/bsky-tools/xrpcauth"
 	"github.com/uabluerail/indexer/models"
+	"github.com/uabluerail/indexer/pds"
 	"github.com/uabluerail/indexer/repo"
 	"github.com/uabluerail/indexer/util/resolver"
 )
@@ -192,6 +193,17 @@ retry:
 	}
 	reposFetched.WithLabelValues(u.String(), "true").Inc()
 
+	if work.Repo.PDS == pds.Unknown {
+		remote, err := pds.EnsureExists(ctx, p.db, u.String())
+		if err != nil {
+			return err
+		}
+		work.Repo.PDS = remote.ID
+		if err := p.db.Model(&work.Repo).Where(&repo.Repo{ID: work.Repo.ID}).Updates(&repo.Repo{PDS: work.Repo.PDS}).Error; err != nil {
+			return fmt.Errorf("failed to set repo's PDS: %w", err)
+		}
+	}
+
 	newRev, err := repo.GetRev(ctx, bytes.NewReader(b))
 	if err != nil {
 		return fmt.Errorf("failed to read 'rev' from the fetched repo: %w", err)

pds/pds.go

@@ -1,6 +1,8 @@
 package pds
 
 import (
+	"context"
+	"fmt"
 	"time"
 
 	"gorm.io/gorm"
@@ -8,6 +10,8 @@ import (
 	"github.com/uabluerail/indexer/models"
 )
 
+const Unknown models.ID = 0
+
 type PDS struct {
 	ID                    models.ID `gorm:"primarykey"`
 	CreatedAt             time.Time
@@ -22,3 +26,11 @@ type PDS struct {
 func AutoMigrate(db *gorm.DB) error {
 	return db.AutoMigrate(&PDS{})
 }
+
+func EnsureExists(ctx context.Context, db *gorm.DB, host string) (*PDS, error) {
+	remote := PDS{Host: host}
+	if err := db.Model(&remote).Where(&PDS{Host: host}).FirstOrCreate(&remote).Error; err != nil {
+		return nil, fmt.Errorf("failed to get PDS record from DB for %q: %w", remote.Host, err)
+	}
+	return &remote, nil
+}

repo/repo.go

@@ -88,8 +88,8 @@ func EnsureExists(ctx context.Context, db *gorm.DB, did string) (*Repo, bool, er
 	if u.Host == "" {
 		return nil, false, fmt.Errorf("PDS endpoint (%q) doesn't have a host part", pdsHost)
 	}
-	remote := pds.PDS{Host: u.String()}
+	remote, err := pds.EnsureExists(ctx, db, u.String())
-	if err := db.Model(&remote).Where(&pds.PDS{Host: remote.Host}).FirstOrCreate(&remote).Error; err != nil {
+	if err != nil {
 		return nil, false, fmt.Errorf("failed to get PDS record from DB for %q: %w", remote.Host, err)
 	}
 	r = Repo{DID: did, PDS: models.ID(remote.ID)}