Import

2024-02-15 16:10:39 +00:00 · 2024-02-15 16:10:39 +00:00 · 63a767d890
parent 2b6abac607
commit 63a767d890
25 changed files with 3027 additions and 0 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -0,0 +1,5 @@
+*
+**/*
+!go.mod
+!go.sum
+!**/*.go
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,2 @@
+go.work*
+.env
--- a/cmd/consumer/Dockerfile
+++ b/cmd/consumer/Dockerfile
@ -0,0 +1,14 @@
+FROM golang:1.21.1 as builder
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . ./
+RUN go build -trimpath ./cmd/consumer
+
+FROM alpine:latest as certs
+RUN apk --update add ca-certificates
+
+FROM debian:stable-slim
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /app/consumer .
+ENTRYPOINT ["./consumer"]
--- a/cmd/consumer/consumer.go
+++ b/cmd/consumer/consumer.go
@ -0,0 +1,431 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+
+	comatproto "github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/xrpc"
+	"github.com/ipld/go-ipld-prime/codec/dagcbor"
+	"github.com/ipld/go-ipld-prime/datamodel"
+	"github.com/ipld/go-ipld-prime/node/basicnode"
+
+	"github.com/uabluerail/indexer/models"
+	"github.com/uabluerail/indexer/pds"
+	"github.com/uabluerail/indexer/repo"
+)
+
+type Consumer struct {
+	db     *gorm.DB
+	remote pds.PDS
+
+	lastCursorPersist time.Time
+}
+
+func NewConsumer(ctx context.Context, remote *pds.PDS, db *gorm.DB) (*Consumer, error) {
+	return &Consumer{
+		db:     db,
+		remote: *remote,
+	}, nil
+}
+
+func (c *Consumer) Start(ctx context.Context) error {
+	go c.run(ctx)
+	return nil
+}
+
+func (c *Consumer) run(ctx context.Context) {
+	log := zerolog.Ctx(ctx).With().Str("pds", c.remote.Host).Logger()
+	ctx = log.WithContext(ctx)
+
+	for {
+		if err := c.runOnce(ctx); err != nil {
+			log.Error().Err(err).Msgf("Consumer of %q failed (will be restarted): %s", c.remote.Host, err)
+		}
+		time.Sleep(time.Second)
+	}
+}
+
+func (c *Consumer) runOnce(ctx context.Context) error {
+	log := zerolog.Ctx(ctx)
+
+	addr, err := url.Parse(c.remote.Host)
+	if err != nil {
+		return fmt.Errorf("parsing URL %q: %s", c.remote.Host, err)
+	}
+	addr.Scheme = "wss"
+	addr.Path = path.Join(addr.Path, "xrpc/com.atproto.sync.subscribeRepos")
+
+	if c.remote.Cursor > 0 {
+		params := url.Values{"cursor": []string{fmt.Sprint(c.remote.Cursor)}}
+		addr.RawQuery = params.Encode()
+	}
+
+	conn, _, err := websocket.DefaultDialer.DialContext(ctx, addr.String(), http.Header{})
+	if err != nil {
+		return fmt.Errorf("establishing websocker connection: %w", err)
+	}
+	defer conn.Close()
+
+	ch := make(chan bool)
+	defer close(ch)
+	go func() {
+		t := time.NewTicker(time.Minute)
+		for {
+			select {
+			case <-ch:
+				return
+			case <-t.C:
+				if err := conn.WriteControl(websocket.PingMessage, []byte("ping"), time.Now().Add(time.Minute)); err != nil {
+					log.Error().Err(err).Msgf("Failed to send ping: %s", err)
+				}
+			}
+		}
+	}()
+
+	first := true
+	for {
+		_, b, err := conn.ReadMessage()
+		if err != nil {
+			return fmt.Errorf("websocket.ReadMessage: %w", err)
+		}
+
+		r := bytes.NewReader(b)
+		proto := basicnode.Prototype.Any
+		headerNode := proto.NewBuilder()
+		if err := (&dagcbor.DecodeOptions{DontParseBeyondEnd: true}).Decode(headerNode, r); err != nil {
+			return fmt.Errorf("unmarshaling message header: %w", err)
+		}
+		header, err := parseHeader(headerNode.Build())
+		if err != nil {
+			return fmt.Errorf("parsing message header: %w", err)
+		}
+		switch header.Op {
+		case 1:
+			if err := c.processMessage(ctx, header.Type, r, first); err != nil {
+				return err
+			}
+		case -1:
+			bodyNode := proto.NewBuilder()
+			if err := (&dagcbor.DecodeOptions{DontParseBeyondEnd: true, AllowLinks: true}).Decode(bodyNode, r); err != nil {
+				return fmt.Errorf("unmarshaling message body: %w", err)
+			}
+			body, err := parseError(bodyNode.Build())
+			if err != nil {
+				return fmt.Errorf("parsing error payload: %w", err)
+			}
+			return &body
+		default:
+			log.Warn().Msgf("Unknown 'op' value received: %d", header.Op)
+		}
+		first = false
+	}
+}
+
+func (c *Consumer) checkForCursorReset(ctx context.Context, seq int64) error {
+	// hack to detect cursor resets upon connection for implementations
+	// that don't emit an explicit #info when connecting with an outdated cursor.
+
+	if seq == c.remote.Cursor+1 {
+		// No reset.
+		return nil
+	}
+
+	return c.resetCursor(ctx, seq)
+}
+
+func (c *Consumer) resetCursor(ctx context.Context, seq int64) error {
+	zerolog.Ctx(ctx).Warn().Str("pds", c.remote.Host).Msgf("Cursor reset: %d -> %d", c.remote.Cursor, seq)
+	err := c.db.Model(&c.remote).
+		Where(&pds.PDS{Model: gorm.Model{ID: c.remote.ID}}).
+		Updates(&pds.PDS{FirstCursorSinceReset: seq}).Error
+	if err != nil {
+		return fmt.Errorf("updating FirstCursorSinceReset: %w", err)
+	}
+	c.remote.FirstCursorSinceReset = seq
+	return nil
+}
+
+func (c *Consumer) updateCursor(ctx context.Context, seq int64) error {
+	if math.Abs(float64(seq-c.remote.Cursor)) < 100 && time.Since(c.lastCursorPersist) < 5*time.Second {
+		c.remote.Cursor = seq
+		return nil
+	}
+
+	err := c.db.Model(&c.remote).
+		Where(&pds.PDS{Model: gorm.Model{ID: c.remote.ID}}).
+		Updates(&pds.PDS{Cursor: seq}).Error
+	if err != nil {
+		return fmt.Errorf("updating Cursor: %w", err)
+	}
+	c.remote.Cursor = seq
+	return nil
+
+}
+
+func (c *Consumer) processMessage(ctx context.Context, typ string, r io.Reader, first bool) error {
+	log := zerolog.Ctx(ctx)
+
+	switch typ {
+	case "#commit":
+		payload := &comatproto.SyncSubscribeRepos_Commit{}
+		if err := payload.UnmarshalCBOR(r); err != nil {
+			return fmt.Errorf("failed to unmarshal commit: %w", err)
+		}
+
+		if c.remote.FirstCursorSinceReset == 0 {
+			if err := c.resetCursor(ctx, payload.Seq); err != nil {
+				return fmt.Errorf("handling cursor reset: %w", err)
+			}
+		}
+		if first {
+			if err := c.checkForCursorReset(ctx, payload.Seq); err != nil {
+				return err
+			}
+		}
+
+		repoInfo, err := repo.EnsureExists(ctx, c.db, payload.Repo)
+		if err != nil {
+			return fmt.Errorf("repo.EnsureExists(%q): %w", payload.Repo, err)
+		}
+		if repoInfo.PDS != models.ID(c.remote.ID) {
+			log.Error().Str("did", payload.Repo).Str("rev", payload.Rev).
+				Msgf("Commit from an incorrect PDS, skipping")
+			return nil
+		}
+
+		// TODO: verify signature
+
+		expectRecords := false
+		deletions := []string{}
+		for _, op := range payload.Ops {
+			switch op.Action {
+			case "create":
+				expectRecords = true
+			case "update":
+				expectRecords = true
+			case "delete":
+				deletions = append(deletions, op.Path)
+			}
+		}
+		for _, d := range deletions {
+			parts := strings.SplitN(d, "/", 2)
+			if len(parts) != 2 {
+				continue
+			}
+			err := c.db.Model(&repo.Record{}).
+				Where(&repo.Record{
+					Repo:       models.ID(repoInfo.ID),
+					Collection: parts[0],
+					Rkey:       parts[1]}).
+				Updates(&repo.Record{Deleted: true}).Error
+			if err != nil {
+				return fmt.Errorf("failed to mark %s/%s as deleted: %w", payload.Repo, d, err)
+			}
+		}
+
+		newRecs, err := repo.ExtractRecords(ctx, bytes.NewReader(payload.Blocks))
+		if err != nil {
+			return fmt.Errorf("failed to extract records: %w", err)
+		}
+		recs := []repo.Record{}
+		for k, v := range newRecs {
+			parts := strings.SplitN(k, "/", 2)
+			if len(parts) != 2 {
+				log.Warn().Msgf("Unexpected key format: %q", k)
+				continue
+			}
+			recs = append(recs, repo.Record{
+				Repo:       models.ID(repoInfo.ID),
+				Collection: parts[0],
+				Rkey:       parts[1],
+				Content:    v,
+			})
+		}
+		if len(recs) == 0 && expectRecords {
+			log.Debug().Int64("seq", payload.Seq).Str("pds", c.remote.Host).Msgf("len(recs) == 0")
+		}
+		if len(recs) > 0 || expectRecords {
+			err = c.db.Model(&repo.Record{}).
+				Clauses(clause.OnConflict{DoUpdates: clause.AssignmentColumns([]string{"content"}),
+					Columns: []clause.Column{{Name: "repo"}, {Name: "collection"}, {Name: "rkey"}}}).
+				Create(recs).Error
+			if err != nil {
+				return fmt.Errorf("inserting records into the database: %w", err)
+			}
+		}
+
+		if payload.TooBig {
+			// Just trigger a re-index by resetting rev.
+			err := c.db.Model(r).Where(&repo.Repo{Model: gorm.Model{ID: repoInfo.ID}}).
+				Updates(&repo.Repo{
+					FirstCursorSinceReset: c.remote.FirstCursorSinceReset,
+					FirstRevSinceReset:    payload.Rev,
+				}).Error
+			if err != nil {
+				return fmt.Errorf("failed to update repo info after cursor reset: %w", err)
+			}
+		}
+
+		if repoInfo.FirstCursorSinceReset != c.remote.FirstCursorSinceReset {
+			err := c.db.Model(r).Where(&repo.Repo{Model: gorm.Model{ID: repoInfo.ID}}).
+				Updates(&repo.Repo{
+					FirstCursorSinceReset: c.remote.FirstCursorSinceReset,
+					FirstRevSinceReset:    payload.Rev,
+				}).Error
+			if err != nil {
+				return fmt.Errorf("failed to update repo info after cursor reset: %w", err)
+			}
+		}
+
+		if err := c.updateCursor(ctx, payload.Seq); err != nil {
+			return err
+		}
+	case "#handle":
+		payload := &comatproto.SyncSubscribeRepos_Handle{}
+		if err := payload.UnmarshalCBOR(r); err != nil {
+			return fmt.Errorf("failed to unmarshal commit: %w", err)
+		}
+
+		if c.remote.FirstCursorSinceReset == 0 {
+			if err := c.resetCursor(ctx, payload.Seq); err != nil {
+				return fmt.Errorf("handling cursor reset: %w", err)
+			}
+		}
+		if first {
+			if err := c.checkForCursorReset(ctx, payload.Seq); err != nil {
+				return err
+			}
+		}
+		// No-op, we don't store handles.
+		if err := c.updateCursor(ctx, payload.Seq); err != nil {
+			return err
+		}
+	case "#migrate":
+		payload := &comatproto.SyncSubscribeRepos_Migrate{}
+		if err := payload.UnmarshalCBOR(r); err != nil {
+			return fmt.Errorf("failed to unmarshal commit: %w", err)
+		}
+
+		if c.remote.FirstCursorSinceReset == 0 {
+			if err := c.resetCursor(ctx, payload.Seq); err != nil {
+				return fmt.Errorf("handling cursor reset: %w", err)
+			}
+		}
+		if first {
+			if err := c.checkForCursorReset(ctx, payload.Seq); err != nil {
+				return err
+			}
+		}
+
+		log.Debug().Interface("payload", payload).Str("did", payload.Did).Msgf("MIGRATION")
+		// TODO
+		if err := c.updateCursor(ctx, payload.Seq); err != nil {
+			return err
+		}
+	case "#tombstone":
+		payload := &comatproto.SyncSubscribeRepos_Tombstone{}
+		if err := payload.UnmarshalCBOR(r); err != nil {
+			return fmt.Errorf("failed to unmarshal commit: %w", err)
+		}
+
+		if c.remote.FirstCursorSinceReset == 0 {
+			if err := c.resetCursor(ctx, payload.Seq); err != nil {
+				return fmt.Errorf("handling cursor reset: %w", err)
+			}
+		}
+		if first {
+			if err := c.checkForCursorReset(ctx, payload.Seq); err != nil {
+				return err
+			}
+		}
+		// TODO
+		if err := c.updateCursor(ctx, payload.Seq); err != nil {
+			return err
+		}
+	case "#info":
+		payload := &comatproto.SyncSubscribeRepos_Info{}
+		if err := payload.UnmarshalCBOR(r); err != nil {
+			return fmt.Errorf("failed to unmarshal commit: %w", err)
+		}
+		switch payload.Name {
+		case "OutdatedCursor":
+			if !first {
+				log.Warn().Msgf("Received cursor reset notification in the middle of a stream: %+v", payload)
+			}
+			c.remote.FirstCursorSinceReset = 0
+		default:
+			log.Error().Msgf("Unknown #info message %q: %+v", payload.Name, payload)
+		}
+	default:
+		log.Warn().Msgf("Unknown message type received: %s", typ)
+	}
+	return nil
+}
+
+type Header struct {
+	Op   int64
+	Type string
+}
+
+func parseHeader(node datamodel.Node) (Header, error) {
+	r := Header{}
+	op, err := node.LookupByString("op")
+	if err != nil {
+		return r, fmt.Errorf("missing 'op': %w", err)
+	}
+	r.Op, err = op.AsInt()
+	if err != nil {
+		return r, fmt.Errorf("op.AsInt(): %w", err)
+	}
+	if r.Op == -1 {
+		// Error frame, type should not be present
+		return r, nil
+	}
+	t, err := node.LookupByString("t")
+	if err != nil {
+		return r, fmt.Errorf("missing 't': %w", err)
+	}
+	r.Type, err = t.AsString()
+	if err != nil {
+		return r, fmt.Errorf("t.AsString(): %w", err)
+	}
+	return r, nil
+}
+
+func parseError(node datamodel.Node) (xrpc.XRPCError, error) {
+	r := xrpc.XRPCError{}
+	e, err := node.LookupByString("error")
+	if err != nil {
+		return r, fmt.Errorf("missing 'error': %w", err)
+	}
+	r.ErrStr, err = e.AsString()
+	if err != nil {
+		return r, fmt.Errorf("error.AsString(): %w", err)
+	}
+	m, err := node.LookupByString("message")
+	if err == nil {
+		r.Message, err = m.AsString()
+		if err != nil {
+			return r, fmt.Errorf("message.AsString(): %w", err)
+		}
+	} else if !errors.Is(err, datamodel.ErrNotExists{}) {
+		return r, fmt.Errorf("looking up 'message': %w", err)
+	}
+
+	return r, nil
+}
--- a/cmd/consumer/main.go
+++ b/cmd/consumer/main.go
@ -0,0 +1,176 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"runtime/debug"
+	"strings"
+	"syscall"
+	"time"
+
+	_ "github.com/joho/godotenv/autoload"
+	"github.com/kelseyhightower/envconfig"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/rs/zerolog"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+
+	"github.com/uabluerail/indexer/pds"
+	"github.com/uabluerail/indexer/util/gormzerolog"
+)
+
+type Config struct {
+	LogFile     string
+	LogFormat   string `default:"text"`
+	LogLevel    int64  `default:"1"`
+	MetricsPort string `split_words:"true"`
+	DBUrl       string `envconfig:"POSTGRES_URL"`
+}
+
+var config Config
+
+func runMain(ctx context.Context) error {
+	ctx = setupLogging(ctx)
+	log := zerolog.Ctx(ctx)
+	log.Debug().Msgf("Starting up...")
+	db, err := gorm.Open(postgres.Open(config.DBUrl), &gorm.Config{
+		Logger: gormzerolog.New(&logger.Config{
+			SlowThreshold:             3 * time.Second,
+			IgnoreRecordNotFoundError: true,
+		}, nil),
+	})
+	if err != nil {
+		return fmt.Errorf("connecting to the database: %w", err)
+	}
+	log.Debug().Msgf("DB connection established")
+
+	remotes := []pds.PDS{}
+	if err := db.Find(&remotes).Error; err != nil {
+		return fmt.Errorf("listing known PDSs: %w", err)
+	}
+	// TODO: check for changes and start/stop consumers as needed
+	for _, remote := range remotes {
+		c, err := NewConsumer(ctx, &remote, db)
+		if err != nil {
+			return fmt.Errorf("failed to create a consumer for %q: %w", remote.Host, err)
+		}
+		if err := c.Start(ctx); err != nil {
+			return fmt.Errorf("failed ot start a consumer for %q: %w", remote.Host, err)
+		}
+	}
+
+	log.Info().Msgf("Starting HTTP listener on %q...", config.MetricsPort)
+	http.Handle("/metrics", promhttp.Handler())
+	srv := &http.Server{Addr: fmt.Sprintf(":%s", config.MetricsPort)}
+	errCh := make(chan error)
+	go func() {
+		errCh <- srv.ListenAndServe()
+	}()
+	select {
+	case <-ctx.Done():
+		if err := srv.Shutdown(context.Background()); err != nil {
+			return fmt.Errorf("HTTP server shutdown failed: %w", err)
+		}
+	}
+	return <-errCh
+}
+
+func main() {
+	flag.StringVar(&config.LogFile, "log", "", "Path to the log file. If empty, will log to stderr")
+	flag.StringVar(&config.LogFormat, "log-format", "text", "Logging format. 'text' or 'json'")
+	flag.Int64Var(&config.LogLevel, "log-level", 1, "Log level. -1 - trace, 0 - debug, 1 - info, 5 - panic")
+
+	if err := envconfig.Process("consumer", &config); err != nil {
+		log.Fatalf("envconfig.Process: %s", err)
+	}
+
+	flag.Parse()
+
+	ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	if err := runMain(ctx); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func setupLogging(ctx context.Context) context.Context {
+	logFile := os.Stderr
+
+	if config.LogFile != "" {
+		f, err := os.OpenFile(config.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			log.Fatalf("Failed to open the specified log file %q: %s", config.LogFile, err)
+		}
+		logFile = f
+	}
+
+	var output io.Writer
+
+	switch config.LogFormat {
+	case "json":
+		output = logFile
+	case "text":
+		prefixList := []string{}
+		info, ok := debug.ReadBuildInfo()
+		if ok {
+			prefixList = append(prefixList, info.Path+"/")
+		}
+
+		basedir := ""
+		_, sourceFile, _, ok := runtime.Caller(0)
+		if ok {
+			basedir = filepath.Dir(sourceFile)
+		}
+
+		if basedir != "" && strings.HasPrefix(basedir, "/") {
+			prefixList = append(prefixList, basedir+"/")
+			head, _ := filepath.Split(basedir)
+			for head != "/" {
+				prefixList = append(prefixList, head)
+				head, _ = filepath.Split(strings.TrimSuffix(head, "/"))
+			}
+		}
+
+		output = zerolog.ConsoleWriter{
+			Out:        logFile,
+			NoColor:    true,
+			TimeFormat: time.RFC3339,
+			PartsOrder: []string{
+				zerolog.LevelFieldName,
+				zerolog.TimestampFieldName,
+				zerolog.CallerFieldName,
+				zerolog.MessageFieldName,
+			},
+			FormatFieldName:  func(i interface{}) string { return fmt.Sprintf("%s:", i) },
+			FormatFieldValue: func(i interface{}) string { return fmt.Sprintf("%s", i) },
+			FormatCaller: func(i interface{}) string {
+				s := i.(string)
+				for _, p := range prefixList {
+					s = strings.TrimPrefix(s, p)
+				}
+				return s
+			},
+		}
+	default:
+		log.Fatalf("Invalid log format specified: %q", config.LogFormat)
+	}
+
+	logger := zerolog.New(output).Level(zerolog.Level(config.LogLevel)).With().Caller().Timestamp().Logger()
+
+	ctx = logger.WithContext(ctx)
+
+	zerolog.DefaultContextLogger = &logger
+	log.SetOutput(logger)
+
+	return ctx
+}
--- a/cmd/lister/Dockerfile
+++ b/cmd/lister/Dockerfile
@ -0,0 +1,14 @@
+FROM golang:1.21.1 as builder
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . ./
+RUN go build -trimpath ./cmd/lister
+
+FROM alpine:latest as certs
+RUN apk --update add ca-certificates
+
+FROM debian:stable-slim
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /app/lister .
+ENTRYPOINT ["./lister"]
--- a/cmd/lister/lister.go
+++ b/cmd/lister/lister.go
@ -0,0 +1,108 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/rs/zerolog"
+	"gorm.io/gorm"
+
+	comatproto "github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/did"
+
+	"github.com/uabluerail/bsky-tools/pagination"
+	"github.com/uabluerail/bsky-tools/xrpcauth"
+	"github.com/uabluerail/indexer/pds"
+	"github.com/uabluerail/indexer/repo"
+	"github.com/uabluerail/indexer/util/resolver"
+)
+
+type Lister struct {
+	db       *gorm.DB
+	resolver did.Resolver
+
+	pollInterval        time.Duration
+	listRefreshInterval time.Duration
+}
+
+func NewLister(ctx context.Context, db *gorm.DB) (*Lister, error) {
+	return &Lister{
+		db:                  db,
+		resolver:            resolver.Resolver,
+		pollInterval:        5 * time.Minute,
+		listRefreshInterval: 24 * time.Hour,
+	}, nil
+}
+
+func (l *Lister) Start(ctx context.Context) error {
+	go l.run(ctx)
+	return nil
+}
+
+func (l *Lister) run(ctx context.Context) {
+	log := zerolog.Ctx(ctx)
+	ticker := time.NewTicker(l.pollInterval)
+
+	log.Info().Msgf("Lister starting...")
+	t := make(chan time.Time, 1)
+	t <- time.Now()
+	for {
+		select {
+		case <-ctx.Done():
+			log.Info().Msgf("Lister stopped (context expired)")
+			return
+		case <-t:
+			db := l.db.WithContext(ctx)
+
+			remote := pds.PDS{}
+			if err := db.Model(&remote).
+				Where("last_list is null or last_list < ?", time.Now().Add(-l.listRefreshInterval)).
+				Take(&remote).Error; err != nil {
+				if !errors.Is(err, gorm.ErrRecordNotFound) {
+					log.Error().Err(err).Msgf("Failed to query DB for a PDS to list repos from: %s", err)
+				}
+				break
+			}
+			client := xrpcauth.NewAnonymousClient(ctx)
+			client.Host = remote.Host
+
+			log.Info().Msgf("Listing repos from %q...", remote.Host)
+			dids, err := pagination.Reduce(
+				func(cursor string) (resp *comatproto.SyncListRepos_Output, nextCursor string, err error) {
+					resp, err = comatproto.SyncListRepos(ctx, client, cursor, 200)
+					if err == nil && resp.Cursor != nil {
+						nextCursor = *resp.Cursor
+					}
+					return
+				},
+				func(resp *comatproto.SyncListRepos_Output, acc []string) ([]string, error) {
+					for _, repo := range resp.Repos {
+						if repo == nil {
+							continue
+						}
+						acc = append(acc, repo.Did)
+					}
+					return acc, nil
+				})
+
+			if err != nil {
+				log.Error().Err(err).Msgf("Failed to list repos from %q: %s", remote.Host, err)
+				break
+			}
+			log.Info().Msgf("Received %d DIDs from %q", len(dids), remote.Host)
+
+			for _, did := range dids {
+				if _, err := repo.EnsureExists(ctx, l.db, did); err != nil {
+					log.Error().Err(err).Msgf("Failed to ensure that we have a record for the repo %q: %s", did, err)
+				}
+			}
+
+			if err := db.Model(&remote).Updates(&pds.PDS{LastList: time.Now()}).Error; err != nil {
+				log.Error().Err(err).Msgf("Failed to update the timestamp of last list for %q: %s", remote.Host, err)
+			}
+		case v := <-ticker.C:
+			t <- v
+		}
+	}
+}
--- a/cmd/lister/main.go
+++ b/cmd/lister/main.go
@ -0,0 +1,180 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"runtime/debug"
+	"strings"
+	"syscall"
+	"time"
+
+	_ "github.com/joho/godotenv/autoload"
+	"github.com/kelseyhightower/envconfig"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/rs/zerolog"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+
+	"github.com/uabluerail/indexer/pds"
+	"github.com/uabluerail/indexer/repo"
+	"github.com/uabluerail/indexer/util/gormzerolog"
+)
+
+type Config struct {
+	LogFile     string
+	LogFormat   string `default:"text"`
+	LogLevel    int64  `default:"1"`
+	MetricsPort string `split_words:"true"`
+	DBUrl       string `envconfig:"POSTGRES_URL"`
+}
+
+var config Config
+
+func runMain(ctx context.Context) error {
+	ctx = setupLogging(ctx)
+	log := zerolog.Ctx(ctx)
+	log.Debug().Msgf("Starting up...")
+	db, err := gorm.Open(postgres.Open(config.DBUrl), &gorm.Config{
+		Logger: gormzerolog.New(&logger.Config{
+			SlowThreshold:             1 * time.Second,
+			IgnoreRecordNotFoundError: true,
+		}, nil),
+	})
+	if err != nil {
+		return fmt.Errorf("connecting to the database: %w", err)
+	}
+	log.Debug().Msgf("DB connection established")
+
+	for _, f := range []func(*gorm.DB) error{
+		pds.AutoMigrate,
+		repo.AutoMigrate,
+	} {
+		if err := f(db); err != nil {
+			return fmt.Errorf("auto-migrating DB schema: %w", err)
+		}
+	}
+	log.Debug().Msgf("DB schema updated")
+
+	lister, err := NewLister(ctx, db)
+	if err != nil {
+		return fmt.Errorf("failed to create lister: %w", err)
+	}
+	if err := lister.Start(ctx); err != nil {
+		return fmt.Errorf("failed to start lister: %w", err)
+	}
+
+	log.Info().Msgf("Starting HTTP listener on %q...", config.MetricsPort)
+	http.Handle("/metrics", promhttp.Handler())
+	srv := &http.Server{Addr: fmt.Sprintf(":%s", config.MetricsPort)}
+	errCh := make(chan error)
+	go func() {
+		errCh <- srv.ListenAndServe()
+	}()
+	select {
+	case <-ctx.Done():
+		if err := srv.Shutdown(context.Background()); err != nil {
+			return fmt.Errorf("HTTP server shutdown failed: %w", err)
+		}
+	}
+	return <-errCh
+}
+
+func main() {
+	flag.StringVar(&config.LogFile, "log", "", "Path to the log file. If empty, will log to stderr")
+	flag.StringVar(&config.LogFormat, "log-format", "text", "Logging format. 'text' or 'json'")
+	flag.Int64Var(&config.LogLevel, "log-level", 1, "Log level. -1 - trace, 0 - debug, 1 - info, 5 - panic")
+
+	if err := envconfig.Process("lister", &config); err != nil {
+		log.Fatalf("envconfig.Process: %s", err)
+	}
+
+	flag.Parse()
+
+	ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	if err := runMain(ctx); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func setupLogging(ctx context.Context) context.Context {
+	logFile := os.Stderr
+
+	if config.LogFile != "" {
+		f, err := os.OpenFile(config.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			log.Fatalf("Failed to open the specified log file %q: %s", config.LogFile, err)
+		}
+		logFile = f
+	}
+
+	var output io.Writer
+
+	switch config.LogFormat {
+	case "json":
+		output = logFile
+	case "text":
+		prefixList := []string{}
+		info, ok := debug.ReadBuildInfo()
+		if ok {
+			prefixList = append(prefixList, info.Path+"/")
+		}
+
+		basedir := ""
+		_, sourceFile, _, ok := runtime.Caller(0)
+		if ok {
+			basedir = filepath.Dir(sourceFile)
+		}
+
+		if basedir != "" && strings.HasPrefix(basedir, "/") {
+			prefixList = append(prefixList, basedir+"/")
+			head, _ := filepath.Split(basedir)
+			for head != "/" {
+				prefixList = append(prefixList, head)
+				head, _ = filepath.Split(strings.TrimSuffix(head, "/"))
+			}
+		}
+
+		output = zerolog.ConsoleWriter{
+			Out:        logFile,
+			NoColor:    true,
+			TimeFormat: time.RFC3339,
+			PartsOrder: []string{
+				zerolog.LevelFieldName,
+				zerolog.TimestampFieldName,
+				zerolog.CallerFieldName,
+				zerolog.MessageFieldName,
+			},
+			FormatFieldName:  func(i interface{}) string { return fmt.Sprintf("%s:", i) },
+			FormatFieldValue: func(i interface{}) string { return fmt.Sprintf("%s", i) },
+			FormatCaller: func(i interface{}) string {
+				s := i.(string)
+				for _, p := range prefixList {
+					s = strings.TrimPrefix(s, p)
+				}
+				return s
+			},
+		}
+	default:
+		log.Fatalf("Invalid log format specified: %q", config.LogFormat)
+	}
+
+	logger := zerolog.New(output).Level(zerolog.Level(config.LogLevel)).With().Caller().Timestamp().Logger()
+
+	ctx = logger.WithContext(ctx)
+
+	zerolog.DefaultContextLogger = &logger
+	log.SetOutput(logger)
+
+	return ctx
+}
--- a/cmd/record-indexer/Dockerfile
+++ b/cmd/record-indexer/Dockerfile
@ -0,0 +1,14 @@
+FROM golang:1.21.1 as builder
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . ./
+RUN go build -trimpath ./cmd/record-indexer
+
+FROM alpine:latest as certs
+RUN apk --update add ca-certificates
+
+FROM debian:stable-slim
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /app/record-indexer .
+ENTRYPOINT ["./record-indexer"]
--- a/cmd/record-indexer/admin.go
+++ b/cmd/record-indexer/admin.go
@ -0,0 +1,77 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"golang.org/x/time/rate"
+)
+
+func AddAdminHandlers(limiter *Limiter, pool *WorkerPool) {
+	http.HandleFunc("/rate/set", handleRateSet(limiter))
+	http.HandleFunc("/rate/setAll", handleRateSetAll(limiter))
+	http.HandleFunc("/pool/resize", handlePoolResize(pool))
+}
+
+func handlePoolResize(pool *WorkerPool) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		s := r.FormValue("size")
+		if s == "" {
+			http.Error(w, "need size", http.StatusBadRequest)
+			return
+		}
+
+		size, err := strconv.Atoi(s)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		pool.Resize(context.Background(), size)
+		fmt.Fprintln(w, "OK")
+	}
+}
+
+func handleRateSet(limiter *Limiter) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		s := r.FormValue("limit")
+		if s == "" {
+			http.Error(w, "need limit", http.StatusBadRequest)
+			return
+		}
+		name := r.FormValue("name")
+		if name == "" {
+			http.Error(w, "need name", http.StatusBadRequest)
+			return
+		}
+
+		limit, err := strconv.Atoi(s)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		limiter.SetLimit(context.Background(), name, rate.Limit(limit))
+		fmt.Fprintln(w, "OK")
+	}
+}
+
+func handleRateSetAll(limiter *Limiter) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		s := r.FormValue("limit")
+		if s == "" {
+			http.Error(w, "need limit", http.StatusBadRequest)
+			return
+		}
+		limit, err := strconv.Atoi(s)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		limiter.SetAllLimits(context.Background(), rate.Limit(limit))
+		fmt.Fprintln(w, "OK")
+	}
+}
--- a/cmd/record-indexer/main.go
+++ b/cmd/record-indexer/main.go
@ -0,0 +1,179 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"runtime/debug"
+	"strings"
+	"syscall"
+	"time"
+
+	_ "github.com/joho/godotenv/autoload"
+	"github.com/kelseyhightower/envconfig"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/rs/zerolog"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+
+	"github.com/uabluerail/indexer/util/gormzerolog"
+)
+
+type Config struct {
+	LogFile     string
+	LogFormat   string `default:"text"`
+	LogLevel    int64  `default:"1"`
+	MetricsPort string `split_words:"true"`
+	DBUrl       string `envconfig:"POSTGRES_URL"`
+	Workers     int    `default:"2"`
+}
+
+var config Config
+
+func runMain(ctx context.Context) error {
+	ctx = setupLogging(ctx)
+	log := zerolog.Ctx(ctx)
+	log.Debug().Msgf("Starting up...")
+	db, err := gorm.Open(postgres.Open(config.DBUrl), &gorm.Config{
+		Logger: gormzerolog.New(&logger.Config{
+			SlowThreshold:             3 * time.Second,
+			IgnoreRecordNotFoundError: true,
+		}, nil),
+	})
+	if err != nil {
+		return fmt.Errorf("connecting to the database: %w", err)
+	}
+	log.Debug().Msgf("DB connection established")
+
+	limiter, err := NewLimiter(db)
+	if err != nil {
+		return fmt.Errorf("failed to create limiter: %w", err)
+	}
+
+	ch := make(chan WorkItem)
+	pool := NewWorkerPool(ch, db, config.Workers, limiter)
+	if err := pool.Start(ctx); err != nil {
+		return fmt.Errorf("failed to start worker pool: %w", err)
+	}
+
+	scheduler := NewScheduler(ch, db)
+	if err := scheduler.Start(ctx); err != nil {
+		return fmt.Errorf("failed to start scheduler: %w", err)
+	}
+
+	log.Info().Msgf("Starting HTTP listener on %q...", config.MetricsPort)
+	AddAdminHandlers(limiter, pool)
+	http.Handle("/metrics", promhttp.Handler())
+	srv := &http.Server{Addr: fmt.Sprintf(":%s", config.MetricsPort)}
+	errCh := make(chan error)
+	go func() {
+		errCh <- srv.ListenAndServe()
+	}()
+	select {
+	case <-ctx.Done():
+		if err := srv.Shutdown(context.Background()); err != nil {
+			return fmt.Errorf("HTTP server shutdown failed: %w", err)
+		}
+	}
+	return <-errCh
+}
+
+func main() {
+	flag.StringVar(&config.LogFile, "log", "", "Path to the log file. If empty, will log to stderr")
+	flag.StringVar(&config.LogFormat, "log-format", "text", "Logging format. 'text' or 'json'")
+	flag.Int64Var(&config.LogLevel, "log-level", 1, "Log level. -1 - trace, 0 - debug, 1 - info, 5 - panic")
+	flag.IntVar(&config.Workers, "workers", 2, "Number of workers to start with")
+
+	if err := envconfig.Process("indexer", &config); err != nil {
+		log.Fatalf("envconfig.Process: %s", err)
+	}
+
+	flag.Parse()
+
+	ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	if err := runMain(ctx); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func setupLogging(ctx context.Context) context.Context {
+	logFile := os.Stderr
+
+	if config.LogFile != "" {
+		f, err := os.OpenFile(config.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			log.Fatalf("Failed to open the specified log file %q: %s", config.LogFile, err)
+		}
+		logFile = f
+	}
+
+	var output io.Writer
+
+	switch config.LogFormat {
+	case "json":
+		output = logFile
+	case "text":
+		prefixList := []string{}
+		info, ok := debug.ReadBuildInfo()
+		if ok {
+			prefixList = append(prefixList, info.Path+"/")
+		}
+
+		basedir := ""
+		_, sourceFile, _, ok := runtime.Caller(0)
+		if ok {
+			basedir = filepath.Dir(sourceFile)
+		}
+
+		if basedir != "" && strings.HasPrefix(basedir, "/") {
+			prefixList = append(prefixList, basedir+"/")
+			head, _ := filepath.Split(basedir)
+			for head != "/" {
+				prefixList = append(prefixList, head)
+				head, _ = filepath.Split(strings.TrimSuffix(head, "/"))
+			}
+		}
+
+		output = zerolog.ConsoleWriter{
+			Out:        logFile,
+			NoColor:    true,
+			TimeFormat: time.RFC3339,
+			PartsOrder: []string{
+				zerolog.LevelFieldName,
+				zerolog.TimestampFieldName,
+				zerolog.CallerFieldName,
+				zerolog.MessageFieldName,
+			},
+			FormatFieldName:  func(i interface{}) string { return fmt.Sprintf("%s:", i) },
+			FormatFieldValue: func(i interface{}) string { return fmt.Sprintf("%s", i) },
+			FormatCaller: func(i interface{}) string {
+				s := i.(string)
+				for _, p := range prefixList {
+					s = strings.TrimPrefix(s, p)
+				}
+				return s
+			},
+		}
+	default:
+		log.Fatalf("Invalid log format specified: %q", config.LogFormat)
+	}
+
+	logger := zerolog.New(output).Level(zerolog.Level(config.LogLevel)).With().Caller().Timestamp().Logger()
+
+	ctx = logger.WithContext(ctx)
+
+	zerolog.DefaultContextLogger = &logger
+	log.SetOutput(logger)
+
+	return ctx
+}
--- a/cmd/record-indexer/metrics.go
+++ b/cmd/record-indexer/metrics.go
@ -0,0 +1,41 @@
+package main
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var reposQueued = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "indexer_repos_queued_count",
+	Help: "Number of repos added to the queue",
+})
+
+var queueLenght = promauto.NewGaugeVec(prometheus.GaugeOpts{
+	Name: "indexer_queue_length",
+	Help: "Current length of indexing queue",
+}, []string{"state"})
+
+var reposFetched = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "indexer_repos_fetched_count",
+	Help: "Number of repos fetched",
+}, []string{"remote", "success"})
+
+var reposIndexed = promauto.NewCounterVec(prometheus.CounterOpts{
+	Name: "indexer_repos_indexed_count",
+	Help: "Number of repos indexed",
+}, []string{"success"})
+
+var recordsFetched = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "indexer_records_fetched_count",
+	Help: "Number of records fetched",
+})
+
+var recordsInserted = promauto.NewCounter(prometheus.CounterOpts{
+	Name: "indexer_records_inserted_count",
+	Help: "Number of records inserted into DB",
+})
+
+var workerPoolSize = promauto.NewGauge(prometheus.GaugeOpts{
+	Name: "indexer_workers_count",
+	Help: "Current number of workers running",
+})
--- a/cmd/record-indexer/ratelimits.go
+++ b/cmd/record-indexer/ratelimits.go
@ -0,0 +1,82 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/rs/zerolog"
+	"github.com/uabluerail/indexer/pds"
+	"golang.org/x/time/rate"
+	"gorm.io/gorm"
+)
+
+const defaultRateLimit = 10
+
+type Limiter struct {
+	mu      sync.RWMutex
+	db      *gorm.DB
+	limiter map[string]*rate.Limiter
+}
+
+func NewLimiter(db *gorm.DB) (*Limiter, error) {
+	remotes := []pds.PDS{}
+
+	if err := db.Find(&remotes).Error; err != nil {
+		return nil, fmt.Errorf("failed to get the list of known PDSs: %w", err)
+	}
+
+	l := &Limiter{
+		db:      db,
+		limiter: map[string]*rate.Limiter{},
+	}
+
+	for _, remote := range remotes {
+		limit := remote.CrawlLimit
+		if limit == 0 {
+			limit = defaultRateLimit
+		}
+		l.limiter[remote.Host] = rate.NewLimiter(rate.Limit(limit), limit*2)
+	}
+	return l, nil
+}
+
+func (l *Limiter) getLimiter(name string) *rate.Limiter {
+	l.mu.RLock()
+	limiter := l.limiter[name]
+	l.mu.RUnlock()
+
+	if limiter != nil {
+		return limiter
+	}
+
+	limiter = rate.NewLimiter(defaultRateLimit, defaultRateLimit*2)
+	l.mu.Lock()
+	l.limiter[name] = limiter
+	l.mu.Unlock()
+	return limiter
+}
+
+func (l *Limiter) Wait(ctx context.Context, name string) error {
+	return l.getLimiter(name).Wait(ctx)
+}
+
+func (l *Limiter) SetLimit(ctx context.Context, name string, limit rate.Limit) {
+	l.getLimiter(name).SetLimit(limit)
+	err := l.db.Model(&pds.PDS{}).Where(&pds.PDS{Host: name}).Updates(&pds.PDS{CrawlLimit: int(limit)}).Error
+	if err != nil {
+		zerolog.Ctx(ctx).Error().Err(err).Msgf("Failed to persist rate limit change for %q: %s", name, err)
+	}
+}
+
+func (l *Limiter) SetAllLimits(ctx context.Context, limit rate.Limit) {
+	l.mu.RLock()
+	for name, limiter := range l.limiter {
+		limiter.SetLimit(limit)
+		err := l.db.Model(&pds.PDS{}).Where(&pds.PDS{Host: name}).Updates(&pds.PDS{CrawlLimit: int(limit)}).Error
+		if err != nil {
+			zerolog.Ctx(ctx).Error().Err(err).Msgf("Failed to persist rate limit change for %q: %s", name, err)
+		}
+	}
+	l.mu.RUnlock()
+}
--- a/cmd/record-indexer/scheduler.go
+++ b/cmd/record-indexer/scheduler.go
@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/rs/zerolog"
+	"github.com/uabluerail/indexer/pds"
+	"github.com/uabluerail/indexer/repo"
+	"gorm.io/gorm"
+)
+
+type Scheduler struct {
+	db     *gorm.DB
+	output chan<- WorkItem
+
+	queue      map[string]*repo.Repo
+	inProgress map[string]*repo.Repo
+}
+
+func NewScheduler(output chan<- WorkItem, db *gorm.DB) *Scheduler {
+	return &Scheduler{
+		db:         db,
+		output:     output,
+		queue:      map[string]*repo.Repo{},
+		inProgress: map[string]*repo.Repo{},
+	}
+}
+
+func (s *Scheduler) Start(ctx context.Context) error {
+	go s.run(ctx)
+	return nil
+}
+
+func (s *Scheduler) run(ctx context.Context) {
+	log := zerolog.Ctx(ctx)
+	t := time.NewTicker(time.Minute)
+	defer t.Stop()
+
+	if err := s.fillQueue(ctx); err != nil {
+		log.Error().Err(err).Msgf("Failed to get more tasks for the queue: %s", err)
+	}
+
+	done := make(chan string)
+	for {
+		if len(s.queue) > 0 {
+			next := WorkItem{signal: make(chan struct{})}
+			for _, r := range s.queue {
+				next.Repo = r
+				break
+			}
+
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				if err := s.fillQueue(ctx); err != nil {
+					log.Error().Err(err).Msgf("Failed to get more tasks for the queue: %s", err)
+				}
+			case s.output <- next:
+				delete(s.queue, next.Repo.DID)
+				s.inProgress[next.Repo.DID] = next.Repo
+				go func(did string, ch chan struct{}) {
+					select {
+					case <-ch:
+					case <-ctx.Done():
+					}
+					done <- did
+				}(next.Repo.DID, next.signal)
+				s.updateQueueLenMetrics()
+			case did := <-done:
+				delete(s.inProgress, did)
+				s.updateQueueLenMetrics()
+			}
+		} else {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				if err := s.fillQueue(ctx); err != nil {
+					log.Error().Err(err).Msgf("Failed to get more tasks for the queue: %s", err)
+				}
+			case did := <-done:
+				delete(s.inProgress, did)
+				s.updateQueueLenMetrics()
+			}
+		}
+	}
+}
+
+func (s *Scheduler) fillQueue(ctx context.Context) error {
+	const maxQueueLen = 10000
+
+	if len(s.queue)+len(s.inProgress) >= maxQueueLen {
+		return nil
+	}
+
+	remotes := []pds.PDS{}
+	if err := s.db.Find(&remotes).Error; err != nil {
+		return fmt.Errorf("failed to get the list of PDSs: %w", err)
+	}
+
+	perPDSLimit := maxQueueLen * 2 / len(remotes)
+
+	for _, remote := range remotes {
+		repos := []repo.Repo{}
+
+		err := s.db.Model(&repos).
+			Where(`pds = ? AND (
+				last_indexed_rev is null OR last_indexed_rev = ''
+				 OR (first_rev_since_reset is not null AND first_rev_since_reset <> '' AND last_indexed_rev < first_rev_since_reset))`,
+				remote.ID).
+			Limit(perPDSLimit).Find(&repos).Error
+
+		if err != nil {
+			return fmt.Errorf("querying DB: %w", err)
+		}
+		for _, r := range repos {
+			if s.queue[r.DID] != nil || s.inProgress[r.DID] != nil {
+				continue
+			}
+			copied := r
+			s.queue[r.DID] = &copied
+			reposQueued.Inc()
+		}
+		s.updateQueueLenMetrics()
+	}
+
+	return nil
+}
+
+func (s *Scheduler) updateQueueLenMetrics() {
+	queueLenght.WithLabelValues("queued").Set(float64(len(s.queue)))
+	queueLenght.WithLabelValues("inProgress").Set(float64(len(s.inProgress)))
+}
--- a/cmd/record-indexer/workerpool.go
+++ b/cmd/record-indexer/workerpool.go
@ -0,0 +1,238 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/imax9000/errors"
+	"github.com/rs/zerolog"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+
+	comatproto "github.com/bluesky-social/indigo/api/atproto"
+	"github.com/bluesky-social/indigo/xrpc"
+
+	"github.com/uabluerail/bsky-tools/xrpcauth"
+	"github.com/uabluerail/indexer/models"
+	"github.com/uabluerail/indexer/repo"
+	"github.com/uabluerail/indexer/util/resolver"
+)
+
+type WorkItem struct {
+	Repo   *repo.Repo
+	signal chan struct{}
+}
+
+type WorkerPool struct {
+	db      *gorm.DB
+	input   <-chan WorkItem
+	limiter *Limiter
+
+	workerSignals []chan struct{}
+	resize        chan int
+}
+
+func NewWorkerPool(input <-chan WorkItem, db *gorm.DB, size int, limiter *Limiter) *WorkerPool {
+	r := &WorkerPool{
+		db:      db,
+		input:   input,
+		limiter: limiter,
+		resize:  make(chan int),
+	}
+	r.workerSignals = make([]chan struct{}, size)
+	for i := range r.workerSignals {
+		r.workerSignals[i] = make(chan struct{})
+	}
+	return r
+}
+
+func (p *WorkerPool) Start(ctx context.Context) error {
+	go p.run(ctx)
+	return nil
+}
+
+func (p *WorkerPool) Resize(ctx context.Context, size int) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case p.resize <- size:
+		return nil
+	}
+}
+
+func (p *WorkerPool) run(ctx context.Context) {
+	for _, ch := range p.workerSignals {
+		go p.worker(ctx, ch)
+	}
+	workerPoolSize.Set(float64(len(p.workerSignals)))
+
+	for {
+		select {
+		case <-ctx.Done():
+			for _, ch := range p.workerSignals {
+				close(ch)
+			}
+			// also wait for all workers to stop?
+		case newSize := <-p.resize:
+			switch {
+			case newSize > len(p.workerSignals):
+				ch := make([]chan struct{}, newSize-len(p.workerSignals))
+				for i := range ch {
+					ch[i] = make(chan struct{})
+					go p.worker(ctx, ch[i])
+				}
+				p.workerSignals = append(p.workerSignals, ch...)
+				workerPoolSize.Set(float64(len(p.workerSignals)))
+			case newSize < len(p.workerSignals) && newSize > 0:
+				for _, ch := range p.workerSignals[newSize:] {
+					close(ch)
+				}
+				p.workerSignals = p.workerSignals[:newSize]
+				workerPoolSize.Set(float64(len(p.workerSignals)))
+			}
+		}
+	}
+}
+
+func (p *WorkerPool) worker(ctx context.Context, signal chan struct{}) {
+	log := zerolog.Ctx(ctx)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-signal:
+			return
+		case work := <-p.input:
+			updates := &repo.Repo{}
+			if err := p.doWork(ctx, work); err != nil {
+				log.Error().Err(err).Msgf("Work task %q failed: %s", work.Repo.DID, err)
+				updates.LastError = err.Error()
+				reposIndexed.WithLabelValues("false").Inc()
+			} else {
+				reposIndexed.WithLabelValues("true").Inc()
+			}
+			updates.LastIndexAttempt = time.Now()
+			err := p.db.Model(&repo.Repo{}).
+				Where(&repo.Repo{Model: gorm.Model{ID: work.Repo.ID}}).
+				Updates(updates).Error
+			if err != nil {
+				log.Error().Err(err).Msgf("Failed to update repo info for %q: %s", work.Repo.DID, err)
+			}
+		}
+	}
+}
+
+func (p *WorkerPool) doWork(ctx context.Context, work WorkItem) error {
+	log := zerolog.Ctx(ctx)
+	defer close(work.signal)
+
+	doc, err := resolver.GetDocument(ctx, work.Repo.DID)
+	if err != nil {
+		return fmt.Errorf("resolving did %q: %w", work.Repo.DID, err)
+	}
+
+	pdsHost := ""
+	for _, srv := range doc.Service {
+		if srv.Type != "AtprotoPersonalDataServer" {
+			continue
+		}
+		pdsHost = srv.ServiceEndpoint
+	}
+	if pdsHost == "" {
+		return fmt.Errorf("did not find any PDS in DID Document")
+	}
+	u, err := url.Parse(pdsHost)
+	if err != nil {
+		return fmt.Errorf("PDS endpoint (%q) is an invalid URL: %w", pdsHost, err)
+	}
+	if u.Host == "" {
+		return fmt.Errorf("PDS endpoint (%q) doesn't have a host part", pdsHost)
+	}
+
+	client := xrpcauth.NewAnonymousClient(ctx)
+	client.Host = u.String()
+
+retry:
+	if p.limiter != nil {
+		if err := p.limiter.Wait(ctx, u.String()); err != nil {
+			return fmt.Errorf("failed to wait on rate limiter: %w", err)
+		}
+	}
+
+	b, err := comatproto.SyncGetRepo(ctx, client, work.Repo.DID, "")
+	if err != nil {
+		if err, ok := errors.As[*xrpc.Error](err); ok {
+			if err.IsThrottled() && err.Ratelimit != nil {
+				log.Debug().Str("pds", u.String()).Msgf("Hit a rate limit (%s), sleeping until %s", err.Ratelimit.Policy, err.Ratelimit.Reset)
+				time.Sleep(time.Until(err.Ratelimit.Reset))
+				goto retry
+			}
+		}
+
+		reposFetched.WithLabelValues(u.String(), "false").Inc()
+		return fmt.Errorf("failed to fetch repo: %w", err)
+	}
+	reposFetched.WithLabelValues(u.String(), "true").Inc()
+
+	newRev, err := repo.GetRev(ctx, bytes.NewReader(b))
+	if err != nil {
+		return fmt.Errorf("failed to read 'rev' from the fetched repo: %w", err)
+	}
+
+	newRecs, err := repo.ExtractRecords(ctx, bytes.NewReader(b))
+	if err != nil {
+		return fmt.Errorf("failed to extract records: %w", err)
+	}
+	recs := []repo.Record{}
+	for k, v := range newRecs {
+		parts := strings.SplitN(k, "/", 2)
+		if len(parts) != 2 {
+			log.Warn().Msgf("Unexpected key format: %q", k)
+			continue
+		}
+		recs = append(recs, repo.Record{
+			Repo:       models.ID(work.Repo.ID),
+			Collection: parts[0],
+			Rkey:       parts[1],
+			Content:    v,
+		})
+	}
+	recordsFetched.Add(float64(len(recs)))
+	if len(recs) > 0 {
+		for _, batch := range splitInBatshes(recs, 500) {
+			result := p.db.Model(&repo.Record{}).
+				Clauses(clause.OnConflict{DoUpdates: clause.AssignmentColumns([]string{"content"}),
+					Columns: []clause.Column{{Name: "repo"}, {Name: "collection"}, {Name: "rkey"}}}).
+				Create(batch)
+			if err := result.Error; err != nil {
+				return fmt.Errorf("inserting records into the database: %w", err)
+			}
+			recordsInserted.Add(float64(result.RowsAffected))
+
+		}
+	}
+
+	err = p.db.Model(&repo.Repo{}).Where(&repo.Repo{Model: gorm.Model{ID: work.Repo.ID}}).
+		Updates(&repo.Repo{LastIndexedRev: newRev}).Error
+	if err != nil {
+		return fmt.Errorf("updating repo rev: %w", err)
+	}
+
+	return nil
+}
+
+func splitInBatshes[T any](s []T, batchSize int) [][]T {
+	var r [][]T
+	for i := 0; i < len(s); i += batchSize {
+		if i+batchSize < len(s) {
+			r = append(r, s[i:i+batchSize])
+		} else {
+			r = append(r, s[i:])
+		}
+	}
+	return r
+}
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,79 @@
+version: '3.8'
+
+services:
+  postgres:
+    image: "postgres:16"
+    volumes:
+      - "${DATA_DIR:?specify data dir in .env file}/postgres:/var/lib/postgresql/data:rw"
+    restart: always
+    environment:
+      POSTGRES_DB: bluesky
+      POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:?specify password in .env file}"
+  lister:
+    build:
+      context: .
+      dockerfile: cmd/lister/Dockerfile
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    restart: always
+    image: uabluerail/repo-lister
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+    links:
+      - postgres:db
+    environment:
+      LISTER_METRICS_PORT: '8080'
+      LISTER_POSTGRES_URL: "postgres://postgres:${POSTGRES_PASSWORD}@db/bluesky?sslmode=disable"
+      ATP_PLC_ADDR: "${PLC_ADDRESS:-https://plc.directory}"
+    ports:
+      - "0.0.0.0:11001:8080"
+    command: ["--log-level=0"]
+  consumer:
+    build:
+      context: .
+      dockerfile: cmd/consumer/Dockerfile
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    restart: always
+    image: uabluerail/firehose-consumer
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+    links:
+      - postgres:db
+    environment:
+      CONSUMER_METRICS_PORT: '8080'
+      CONSUMER_POSTGRES_URL: "postgres://postgres:${POSTGRES_PASSWORD}@db/bluesky?sslmode=disable"
+      ATP_PLC_ADDR: "${PLC_ADDRESS:-https://plc.directory}"
+    ports:
+      - "0.0.0.0:11002:8080"
+    command: ["--log-level=0"]
+
+  record-indexer:
+    build:
+      context: .
+      dockerfile: cmd/record-indexer/Dockerfile
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    restart: always
+    image: uabluerail/record-indexer
+    deploy:
+      resources:
+        limits:
+          memory: 4G
+    links:
+      - postgres:db
+    dns:
+      - 1.1.1.1
+      - 8.8.8.8
+    environment:
+      INDEXER_METRICS_PORT: '8080'
+      INDEXER_POSTGRES_URL: "postgres://postgres:${POSTGRES_PASSWORD}@db/bluesky?sslmode=disable"
+      INDEXER_WORKERS: 50
+      ATP_PLC_ADDR: "${PLC_ADDRESS:-https://plc.directory}"
+    ports:
+      - "0.0.0.0:11003:8080"
+    command: ["--log-level=0"]
--- a/example.env
+++ b/example.env
@ -0,0 +1,4 @@
+POSTGRES_PASSWORD='some password'
+DATA_DIR=/path/to/storage/dir
+# If you have a PLC mirror (and you do want to have it), put it's address here
+#PLC_ADDRESS=http://host.docker.internal:8097
--- a/go.mod
+++ b/go.mod
@ -0,0 +1,110 @@
+module github.com/uabluerail/indexer
+
+go 1.21.0
+
+require (
+	github.com/bluesky-social/indigo v0.0.0-20240213052310-89516fdbfe38
+	github.com/ipfs/go-cid v0.4.1
+	github.com/ipld/go-car v0.6.2
+	github.com/ipld/go-ipld-prime v0.21.0
+	github.com/joho/godotenv v1.5.1
+	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/prometheus/client_golang v1.18.0
+	github.com/rs/zerolog v1.32.0
+	github.com/uabluerail/bsky-tools v0.0.0-20240210190127-668df5f36070
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
+	gorm.io/driver/postgres v1.5.6
+	gorm.io/gorm v1.25.7
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/carlmjohnson/versioninfo v0.22.5 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/uuid v1.4.0 // indirect
+	github.com/gorilla/websocket v1.5.1 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
+	github.com/hashicorp/golang-lru v1.0.2 // indirect
+	github.com/hashicorp/golang-lru/arc/v2 v2.0.6 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/imax9000/errors v1.0.0 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-block-format v0.2.0 // indirect
+	github.com/ipfs/go-blockservice v0.5.2 // indirect
+	github.com/ipfs/go-car v0.0.4 // indirect
+	github.com/ipfs/go-datastore v0.6.0 // indirect
+	github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect
+	github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect
+	github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.3 // indirect
+	github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
+	github.com/ipfs/go-ipld-format v0.6.0 // indirect
+	github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-log/v2 v2.5.1 // indirect
+	github.com/ipfs/go-merkledag v0.11.0 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-verifcid v0.0.3 // indirect
+	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgx/v5 v5.5.0 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
+	github.com/lestrrat-go/blackmagic v1.0.1 // indirect
+	github.com/lestrrat-go/httpcc v1.0.1 // indirect
+	github.com/lestrrat-go/httprc v1.0.4 // indirect
+	github.com/lestrrat-go/iter v1.0.2 // indirect
+	github.com/lestrrat-go/jwx/v2 v2.0.12 // indirect
+	github.com/lestrrat-go/option v1.0.1 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multihash v0.2.3 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/whyrusleeping/cbor-gen v0.0.0-20240201211319-bf2168ca937c // indirect
+	github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6 // indirect
+	gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect
+	gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
+	go.opentelemetry.io/otel v1.21.0 // indirect
+	go.opentelemetry.io/otel/metric v1.21.0 // indirect
+	go.opentelemetry.io/otel/trace v1.21.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
+	golang.org/x/oauth2 v0.12.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	lukechampine.com/blake3 v1.2.1 // indirect
+)
--- a/go.sum
+++ b/go.sum
@ -0,0 +1,566 @@
+github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
+github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bluesky-social/indigo v0.0.0-20240213052310-89516fdbfe38 h1:KKXEauaiIkqW9HfChxp/t2tvHSQm8Pbbv/mottywm6g=
+github.com/bluesky-social/indigo v0.0.0-20240213052310-89516fdbfe38/go.mod h1:N3Fv7QoBtarvhoHtGLP2U+my6ZbAetbPMwAFTtRAUrI=
+github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
+github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
+github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
+github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
+github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
+github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
+github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
+github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
+github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
+github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
+github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc=
+github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis=
+github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
+github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
+github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
+github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
+github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
+github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M=
+github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
+github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/golang-lru/arc/v2 v2.0.6 h1:4NU7uP5vSoK6TbaMj3NtY478TTAWLso/vL1gpNrInHg=
+github.com/hashicorp/golang-lru/arc/v2 v2.0.6/go.mod h1:cfdDIX05DWvYV6/shsxDfa/OVcRieOt+q4FnM8x+Xno=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
+github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
+github.com/imax9000/errors v1.0.0 h1:nscXEeqLi+yiCRN70brkdx/34vbrer1PNNMxA8U19q4=
+github.com/imax9000/errors v1.0.0/go.mod h1:wmjAf/T6cGLoTLiu8XGaoRMXBcqgXjQaJ3jNMYCBN/8=
+github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI=
+github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
+github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
+github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0=
+github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc=
+github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
+github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
+github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
+github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
+github.com/ipfs/go-blockservice v0.5.2 h1:in9Bc+QcXwd1apOVM7Un9t8tixPKdaHQFdLSUM1Xgk8=
+github.com/ipfs/go-blockservice v0.5.2/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk=
+github.com/ipfs/go-car v0.0.4 h1:zLhxykvk4SFU4oIpgcIoiolVL3jqcK0hjqcQfUSs4dk=
+github.com/ipfs/go-car v0.0.4/go.mod h1:eZX0EppfsvSQN8IsJnx57bheogWMgQjJVWU/fDA7ySQ=
+github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
+github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=
+github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk=
+github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
+github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
+github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk=
+github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8=
+github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
+github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
+github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
+github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
+github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
+github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ=
+github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE=
+github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk=
+github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
+github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
+github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo=
+github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw=
+github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo=
+github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM=
+github.com/ipfs/go-ipfs-exchange-interface v0.2.1 h1:jMzo2VhLKSHbVe+mHNzYgs95n0+t0Q69GQ5WhRDZV/s=
+github.com/ipfs/go-ipfs-exchange-interface v0.2.1/go.mod h1:MUsYn6rKbG6CTtsDp+lKJPmVt3ZrCViNyH3rfPGsZ2E=
+github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0=
+github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=
+github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY=
+github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
+github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0=
+github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs=
+github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc=
+github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs=
+github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk=
+github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms=
+github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k=
+github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U=
+github.com/ipfs/go-ipld-format v0.6.0/go.mod h1:g4QVMTn3marU3qXchwjpKPKgJv+zF+OlaKMyhJ4LHPg=
+github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk=
+github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM=
+github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
+github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8=
+github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo=
+github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g=
+github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY=
+github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI=
+github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
+github.com/ipfs/go-merkledag v0.11.0 h1:DgzwK5hprESOzS4O1t/wi6JDpyVQdvm9Bs59N/jqfBY=
+github.com/ipfs/go-merkledag v0.11.0/go.mod h1:Q4f/1ezvBiJV0YCIXvt51W/9/kqJGH4I1LsA7+djsM4=
+github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
+github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
+github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U=
+github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
+github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs=
+github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw=
+github.com/ipld/go-car v0.0.4/go.mod h1:kgp82Cpd0TgoxgCsr7yndUXMkMgNP/aAwVBtJtsNR6g=
+github.com/ipld/go-car v0.6.1 h1:blWbEHf1j62JMWFIqWE//YR0m7k5ZMw0AuUOU5hjrH8=
+github.com/ipld/go-car v0.6.1/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8=
+github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc=
+github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8=
+github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
+github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
+github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E=
+github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.5.0 h1:NxstgwndsTRy7eq9/kqYc/BZh5w2hHJV86wjvO+1xPw=
+github.com/jackc/pgx/v5 v5.5.0/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
+github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs=
+github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
+github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs=
+github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
+github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
+github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
+github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
+github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
+github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
+github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
+github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80=
+github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
+github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
+github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
+github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8=
+github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo=
+github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
+github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
+github.com/lestrrat-go/jwx/v2 v2.0.12 h1:3d589+5w/b9b7S3DneICPW16AqTyYXB7VRjgluSDWeA=
+github.com/lestrrat-go/jwx/v2 v2.0.12/go.mod h1:Mq4KN1mM7bp+5z/W5HS8aCNs5RKZ911G/0y2qUjAQuQ=
+github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
+github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
+github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
+github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
+github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ=
+github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
+github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc=
+github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8=
+github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM=
+github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8=
+github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro=
+github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8=
+github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco=
+github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco=
+github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI=
+github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g=
+github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90=
+github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo=
+github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE=
+github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY=
+github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU=
+github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY=
+github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY=
+github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
+github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
+github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4=
+github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
+github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
+github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA=
+github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8=
+github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q=
+github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0=
+github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU=
+github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ=
+github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI=
+github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA=
+github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs=
+github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14=
+github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc=
+github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc=
+github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc=
+github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo=
+github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
+github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
+github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
+github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
+github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
+github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
+github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
+github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
+github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE=
+github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI=
+github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
+github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0=
+github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
+github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
+github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
+github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
+github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
+github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
+github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q=
+github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU=
+github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
+github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
+github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
+github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
+github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
+github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
+github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
+github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
+github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
+github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
+github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
+github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
+github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
+github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f h1:VXTQfuJj9vKR4TCkEuWIckKvdHFeJH/huIFJ9/cXOB0=
+github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
+github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
+github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
+github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
+github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
+github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
+github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
+github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
+github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
+github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0=
+github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+github.com/uabluerail/bsky-tools v0.0.0-20240210190127-668df5f36070 h1:TVLWGAHXd4gCRZFXZcRfdZkDh1qRnrnQBHROHyqxhP8=
+github.com/uabluerail/bsky-tools v0.0.0-20240210190127-668df5f36070/go.mod h1:tSTIBNNt/jFXbcfyNs3y6tsBuJ/x3KIBghCW9sNPEio=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
+github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
+github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
+github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
+github.com/whyrusleeping/cbor-gen v0.0.0-20240201211319-bf2168ca937c h1:QNbN8SzRc40MGwnd2op/l3E32M445kVJqvgt7NagF4c=
+github.com/whyrusleeping/cbor-gen v0.0.0-20240201211319-bf2168ca937c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
+github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6 h1:yJ9/LwIGIk/c0CdoavpC9RNSGSruIspSZtxG3Nnldic=
+github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6/go.mod h1:39U9RRVr4CKbXpXYopWn+FSH5s+vWu6+RmguSPWAq5s=
+github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc=
+github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
+github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8=
+github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA=
+github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4=
+github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA=
+gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8=
+gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAFgcfgTnfJrmYKWhHnci3GjDqcZp1M3Q=
+gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02/go.mod h1:JTnUj0mpYiAsuZLmKjTx/ex3AtMowcCgnE7YNyCEP0I=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo=
+go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
+go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
+go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
+go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
+go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
+go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
+go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
+go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
+go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
+go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
+go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
+golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
+golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
+golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
+golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/postgres v1.5.6 h1:ydr9xEd5YAM0vxVDY0X139dyzNz10spDiDlC7+ibLeU=
+gorm.io/driver/postgres v1.5.6/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA=
+gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI=
+lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
--- a/models/models.go
+++ b/models/models.go
@ -0,0 +1,3 @@
+package models
+
+type ID uint // should be same as type of gorm.Model.ID
--- a/pds/pds.go
+++ b/pds/pds.go
@ -0,0 +1,20 @@
+package pds
+
+import (
+	"time"
+
+	"gorm.io/gorm"
+)
+
+type PDS struct {
+	gorm.Model
+	Host                  string `gorm:"uniqueIndex"`
+	Cursor                int64
+	FirstCursorSinceReset int64
+	LastList              time.Time
+	CrawlLimit            int
+}
+
+func AutoMigrate(db *gorm.DB) error {
+	return db.AutoMigrate(&PDS{})
+}
--- a/repo/mst.go
+++ b/repo/mst.go
@ -0,0 +1,296 @@
+package repo
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/ipfs/go-cid"
+	"github.com/ipld/go-car"
+	"github.com/ipld/go-ipld-prime/codec/dagcbor"
+	"github.com/ipld/go-ipld-prime/codec/dagjson"
+	"github.com/ipld/go-ipld-prime/datamodel"
+	"github.com/ipld/go-ipld-prime/node/basicnode"
+)
+
+func ExtractRecords(ctx context.Context, b io.Reader) (map[string]json.RawMessage, error) {
+	r, err := car.NewCarReader(b)
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct CAR reader: %w", err)
+	}
+
+	blocks := map[cid.Cid][]byte{}
+	for {
+		block, err := r.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("reading next block: %w", err)
+		}
+		c, err := block.Cid().Prefix().Sum(block.RawData())
+		if err != nil {
+			return nil, fmt.Errorf("failed to calculate CID from content")
+		}
+		if c.Equals(block.Cid()) {
+			blocks[block.Cid()] = block.RawData()
+		}
+	}
+
+	records := map[string]cid.Cid{}
+	for _, root := range r.Header.Roots {
+		// TODO: verify that a root is a commit record and validate signature
+
+		cids, err := findRecords(blocks, root, nil, nil, 0)
+		if err != nil {
+			return nil, err
+		}
+		for k, v := range cids {
+			records[k] = v
+		}
+	}
+
+	res := map[string]json.RawMessage{}
+	for k, c := range records {
+		builder := basicnode.Prototype.Any.NewBuilder()
+		if err := (&dagcbor.DecodeOptions{AllowLinks: true}).Decode(builder, bytes.NewReader(blocks[c])); err != nil {
+			return nil, fmt.Errorf("unmarshaling %q: %w", c.String(), err)
+		}
+		w := bytes.NewBuffer(nil)
+		if err := (dagjson.EncodeOptions{EncodeLinks: true, EncodeBytes: true}).Encode(builder.Build(), w); err != nil {
+			return nil, fmt.Errorf("marshaling %q as JSON: %w", c.String(), err)
+		}
+		res[k] = w.Bytes()
+	}
+	return res, nil
+}
+
+const maxDepth = 128
+
+func findRecords(blocks map[cid.Cid][]byte, root cid.Cid, key []byte, visited map[cid.Cid]bool, depth int) (map[string]cid.Cid, error) {
+	if depth > maxDepth {
+		return nil, fmt.Errorf("reached maximum depth at %q", root.String())
+	}
+
+	if visited == nil {
+		visited = map[cid.Cid]bool{}
+	}
+
+	visited[root] = true
+
+	builder := basicnode.Prototype.Any.NewBuilder()
+	if err := (&dagcbor.DecodeOptions{AllowLinks: true}).Decode(builder, bytes.NewReader(blocks[root])); err != nil {
+		return nil, fmt.Errorf("unmarshaling %q: %w", root.String(), err)
+	}
+	node := builder.Build()
+
+	if node.Kind() != datamodel.Kind_Map {
+		return nil, nil
+	}
+
+	m, err := parseMap(node)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, ok := m["$type"]; ok {
+		return map[string]cid.Cid{string(key): root}, nil
+	}
+
+	if d, ok := m["data"]; ok {
+		// Commit record
+		if d.Kind() == datamodel.Kind_Link {
+			l, _ := d.AsLink()
+			if l != nil {
+				c, err := cid.Parse([]byte(l.Binary()))
+				if err != nil {
+					return nil, fmt.Errorf("failed to parse %q as CID: %w", l.String(), err)
+				}
+				if _, ok := blocks[c]; ok && !visited[c] {
+					return findRecords(blocks, c, nil, visited, depth+1)
+				}
+			}
+		}
+		return nil, nil
+	}
+
+	if entries, ok := m["e"]; ok {
+		// MST node
+		r := map[string]cid.Cid{}
+		iter := entries.ListIterator()
+		key = []byte{}
+		for !iter.Done() {
+			_, item, err := iter.Next()
+			if err != nil {
+				return nil, fmt.Errorf("failed to read the next list item in block %q: %w", root.String(), err)
+			}
+			if item.Kind() != datamodel.Kind_Map {
+				continue
+			}
+
+			m, err := parseMap(item)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, field := range []string{"k", "p", "v", "t"} {
+				if _, ok := m[field]; !ok {
+					return nil, fmt.Errorf("TreeEntry is missing field %q", field)
+				}
+			}
+			prefixLen, err := m["p"].AsInt()
+			if err != nil {
+				return nil, fmt.Errorf("m[\"p\"].AsInt(): %w", err)
+			}
+			prefixPart, err := m["k"].AsBytes()
+			if err != nil {
+				return nil, fmt.Errorf("m[\"k\"].AsBytes(): %w", err)
+			}
+			val, err := m["v"].AsLink()
+			if err != nil {
+				return nil, fmt.Errorf("m[\"v\"].AsLink(): %w", err)
+			}
+			c, err := cid.Parse([]byte(val.Binary()))
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse %q as CID: %w", val.String(), err)
+			}
+
+			if len(key) == 0 {
+				// First entry, must have a full key.
+				if prefixLen != 0 {
+					return nil, fmt.Errorf("incomplete key in the first entry")
+				}
+				key = prefixPart
+			}
+
+			if prefixLen > int64(len(key)) {
+				return nil, fmt.Errorf("specified prefix length is larger than the key length: %d > %d", prefixLen, len(key))
+			}
+			key = append(key[:prefixLen], prefixPart...)
+
+			if _, ok := blocks[c]; ok && !visited[c] {
+				results, err := findRecords(blocks, c, key, visited, depth+1)
+				if err != nil {
+					return nil, err
+				}
+				for k, v := range results {
+					r[k] = v
+				}
+			}
+
+			if m["t"] != nil && m["t"].Kind() == datamodel.Kind_Link {
+				subtree, err := m["t"].AsLink()
+				if err != nil {
+					return nil, fmt.Errorf("m[\"t\"].AsLink(): %w", err)
+				}
+				subtreeCid, err := cid.Parse([]byte(subtree.Binary()))
+				if err != nil {
+					return nil, fmt.Errorf("failed to parse %q as CID: %w", val.String(), err)
+				}
+				if _, ok := blocks[subtreeCid]; ok && !visited[subtreeCid] {
+					results, err := findRecords(blocks, subtreeCid, key, visited, depth+1)
+					if err != nil {
+						return nil, err
+					}
+					for k, v := range results {
+						r[k] = v
+					}
+				}
+			}
+		}
+
+		left, ok := m["l"]
+		if ok && left.Kind() == datamodel.Kind_Link {
+			l, _ := left.AsLink()
+			if l != nil {
+				c, err := cid.Parse([]byte(l.Binary()))
+				if err != nil {
+					return nil, fmt.Errorf("failed to parse %q as CID: %w", l.String(), err)
+				}
+				if _, ok := blocks[c]; ok && !visited[c] {
+					results, err := findRecords(blocks, c, nil, visited, depth+1)
+					if err != nil {
+						return nil, err
+					}
+					for k, v := range results {
+						r[k] = v
+					}
+				}
+			}
+		}
+
+		return r, nil
+	}
+
+	return nil, fmt.Errorf("unrecognized block %q", root.String())
+}
+
+func parseMap(node datamodel.Node) (map[string]datamodel.Node, error) {
+	if node.Kind() != datamodel.Kind_Map {
+		return nil, fmt.Errorf("not a map")
+	}
+
+	m := map[string]datamodel.Node{}
+	iter := node.MapIterator()
+	for !iter.Done() {
+		k, v, err := iter.Next()
+		if err != nil {
+			return nil, fmt.Errorf("iterating over map fields: %w", err)
+		}
+		if k.Kind() != datamodel.Kind_String {
+			continue
+		}
+		ks, _ := k.AsString()
+		m[ks] = v
+	}
+	return m, nil
+}
+
+func GetRev(ctx context.Context, b io.Reader) (string, error) {
+	r, err := car.NewCarReader(b)
+	if err != nil {
+		return "", fmt.Errorf("failed to construct CAR reader: %w", err)
+	}
+
+	if len(r.Header.Roots) == 0 {
+		return "", fmt.Errorf("no roots specified in CAR header")
+	}
+
+	blocks := map[cid.Cid][]byte{}
+	for {
+		block, err := r.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return "", fmt.Errorf("reading next block: %w", err)
+		}
+		c, err := block.Cid().Prefix().Sum(block.RawData())
+		if err != nil {
+			return "", fmt.Errorf("failed to calculate CID from content")
+		}
+		if c.Equals(block.Cid()) {
+			blocks[block.Cid()] = block.RawData()
+		}
+	}
+
+	builder := basicnode.Prototype.Any.NewBuilder()
+	if err := (&dagcbor.DecodeOptions{AllowLinks: true}).Decode(builder, bytes.NewReader(blocks[r.Header.Roots[0]])); err != nil {
+		return "", fmt.Errorf("unmarshaling %q: %w", r.Header.Roots[0].String(), err)
+	}
+	node := builder.Build()
+
+	v, err := node.LookupByString("rev")
+	if err != nil {
+		return "", fmt.Errorf("looking up 'rev' field: %w", err)
+	}
+
+	s, err := v.AsString()
+	if err != nil {
+		return "", fmt.Errorf("rev.AsString(): %w", err)
+	}
+	return s, nil
+}
--- a/repo/repo.go
+++ b/repo/repo.go
@ -0,0 +1,103 @@
+package repo
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"time"
+
+	"gorm.io/gorm"
+
+	"github.com/uabluerail/indexer/models"
+	"github.com/uabluerail/indexer/pds"
+	"github.com/uabluerail/indexer/util/resolver"
+)
+
+type Repo struct {
+	gorm.Model
+	PDS                   models.ID `gorm:"index:rev_state_index,priority:2"`
+	DID                   string    `gorm:"uniqueIndex;column:did"`
+	LastIndexedRev        string    `gorm:"index:rev_state_index,expression:(last_indexed_rev < first_rev_since_reset),priority:1"`
+	FirstRevSinceReset    string
+	FirstCursorSinceReset int64
+	TombstonedAt          time.Time
+	LastIndexAttempt      time.Time
+	LastError             string
+}
+
+type Record struct {
+	gorm.Model
+	Repo       models.ID       `gorm:"index:idx_repo_record_key,unique,priority:1;not null"`
+	Collection string          `gorm:"index:idx_repo_record_key,unique,priority:2;not null"`
+	Rkey       string          `gorm:"index:idx_repo_record_key,unique,priority:3"`
+	Content    json.RawMessage `gorm:"type:JSONB"`
+	Deleted    bool
+}
+
+func AutoMigrate(db *gorm.DB) error {
+	return db.AutoMigrate(&Repo{}, &Record{})
+}
+
+func EnsureExists(ctx context.Context, db *gorm.DB, did string) (*Repo, error) {
+	r := Repo{}
+	if err := db.Model(&r).Where(&Repo{DID: did}).Take(&r).Error; err == nil {
+		// Already have a row, just return it.
+		return &r, nil
+	} else {
+		if !errors.Is(err, gorm.ErrRecordNotFound) {
+			return nil, fmt.Errorf("querying DB: %w", err)
+		}
+	}
+
+	// No row yet, so we need to create one (keeping in mind that it can be created
+	// concurrently by someone else).
+	// 1) resolve did (i.e., query PLC)
+	// 2) get PDS address from didDoc and ensure we have a record for it
+	// 3) in a transaction, check if we have a record for the repo
+	//     if we don't - just create a record
+	//     if we do - compare PDS IDs
+	//        if they don't match - also reset FirstRevSinceReset
+
+	doc, err := resolver.GetDocument(ctx, did)
+	if err != nil {
+		return nil, fmt.Errorf("fetching DID Document: %w", err)
+	}
+
+	pdsHost := ""
+	for _, srv := range doc.Service {
+		if srv.Type != "AtprotoPersonalDataServer" {
+			continue
+		}
+		pdsHost = srv.ServiceEndpoint
+	}
+	if pdsHost == "" {
+		return nil, fmt.Errorf("did not find any PDS in DID Document")
+	}
+	u, err := url.Parse(pdsHost)
+	if err != nil {
+		return nil, fmt.Errorf("PDS endpoint (%q) is an invalid URL: %w", pdsHost, err)
+	}
+	if u.Host == "" {
+		return nil, fmt.Errorf("PDS endpoint (%q) doesn't have a host part", pdsHost)
+	}
+	remote := pds.PDS{Host: u.String()}
+	if err := db.Model(&remote).Where(&pds.PDS{Host: remote.Host}).FirstOrCreate(&remote).Error; err != nil {
+		return nil, fmt.Errorf("failed to get PDS record from DB for %q: %w", remote.Host, err)
+	}
+	r = Repo{DID: did, PDS: models.ID(remote.ID)}
+	err = db.Transaction(func(tx *gorm.DB) error {
+		if err := tx.Model(&r).Where(&Repo{DID: r.DID}).FirstOrCreate(&r).Error; err != nil {
+			return fmt.Errorf("looking for repo: %w", err)
+		}
+		if r.PDS != models.ID(remote.ID) {
+			return tx.Model(&r).Select("FirstRevSinceReset").Updates(&Repo{FirstRevSinceReset: ""}).Error
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("upserting repo record: %w", err)
+	}
+	return &r, nil
+}
--- a/util/gormzerolog/logger.go
+++ b/util/gormzerolog/logger.go
@ -0,0 +1,97 @@
+package gormzerolog
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/rs/zerolog"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+type logAdapter struct {
+	logger *zerolog.Logger
+	config logger.Config
+}
+
+// New returns a logger that will use zerolog to log messages.
+// Both `config` and `logger` are optional. If `logger` is `nil`,
+// it will be retrieved from context.Context. Config options that
+// are configured in zerolog are not supported (currently that's
+// `Colorful` and `LogLevel`).
+func New(config *logger.Config, logger *zerolog.Logger) logger.Interface {
+	r := &logAdapter{logger: logger}
+	if config != nil {
+		r.config = *config
+	}
+	return r
+}
+
+func (l *logAdapter) ctx(ctx context.Context) *zerolog.Logger {
+	if l.logger != nil {
+		return l.logger
+	}
+	return zerolog.Ctx(ctx)
+}
+
+// Stuff below was copy-pasted from https://github.com/go-gorm/gorm/blob/v1.25.7/logger/logger.go
+// and edited.
+
+func (l *logAdapter) LogMode(level logger.LogLevel) logger.Interface {
+	return l
+}
+
+// Info print info
+func (l *logAdapter) Info(ctx context.Context, msg string, data ...interface{}) {
+	l.ctx(ctx).Info().CallerSkipFrame(3).Msgf(msg, data...)
+}
+
+// Warn print warn messages
+func (l *logAdapter) Warn(ctx context.Context, msg string, data ...interface{}) {
+	l.ctx(ctx).Warn().CallerSkipFrame(3).Msgf(msg, data...)
+}
+
+// Error print error messages
+func (l *logAdapter) Error(ctx context.Context, msg string, data ...interface{}) {
+	l.ctx(ctx).Error().CallerSkipFrame(3).Msgf(msg, data...)
+}
+
+// Trace print sql message
+//
+//nolint:cyclop
+func (l *logAdapter) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {
+	elapsed := time.Since(begin)
+	sql, rows := fc()
+	switch {
+	case err != nil && (!errors.Is(err, gorm.ErrRecordNotFound) || !l.config.IgnoreRecordNotFoundError):
+		log := l.ctx(ctx).Error().CallerSkipFrame(3).Err(err)
+		if rows >= 0 {
+			log.Int64("rows", rows)
+		}
+		log.Dur("dur", elapsed)
+		log.Msgf("%s: %s", sql, err)
+	case elapsed > l.config.SlowThreshold && l.config.SlowThreshold != 0:
+		log := l.ctx(ctx).Warn().CallerSkipFrame(3)
+		if rows >= 0 {
+			log.Int64("rows", rows)
+		}
+		log.Dur("dur", elapsed)
+		log.Msgf("SLOW SQL: %s", sql)
+	default:
+		log := l.ctx(ctx).Trace().CallerSkipFrame(3)
+		if rows >= 0 {
+			log.Int64("rows", rows)
+		}
+		log.Dur("dur", elapsed)
+		log.Msgf("%s", sql)
+	}
+}
+
+// ParamsFilter filter params
+func (l *logAdapter) ParamsFilter(ctx context.Context, sql string, params ...interface{}) (string, []interface{}) {
+	if l.config.ParameterizedQueries {
+		return sql, nil
+	}
+	return sql, params
+}
--- a/util/resolver/resolver.go
+++ b/util/resolver/resolver.go
@ -0,0 +1,52 @@
+package resolver
+
+import (
+	"context"
+	"errors"
+	"os"
+
+	"github.com/bluesky-social/indigo/api"
+	"github.com/bluesky-social/indigo/did"
+)
+
+var Resolver did.Resolver
+
+func init() {
+	resolver := did.NewMultiResolver()
+	plcAddr := os.Getenv("ATP_PLC_ADDR")
+	if plcAddr == "" {
+		plcAddr = "https://plc.directory"
+	}
+	resolver.AddHandler("plc", &fallbackResolver{
+		resolvers: []did.Resolver{
+			&api.PLCServer{Host: plcAddr},
+			&api.PLCServer{Host: "https://plc.directory"},
+		}})
+	resolver.AddHandler("web", &did.WebResolver{})
+
+	Resolver = resolver
+}
+
+func GetDocument(ctx context.Context, didstr string) (*did.Document, error) {
+	return Resolver.GetDocument(ctx, didstr)
+}
+
+type fallbackResolver struct {
+	resolvers []did.Resolver
+}
+
+func (r *fallbackResolver) GetDocument(ctx context.Context, didstr string) (*did.Document, error) {
+	errs := []error{}
+	for _, res := range r.resolvers {
+		if d, err := res.GetDocument(ctx, didstr); err == nil {
+			return d, nil
+		}
+	}
+	return nil, errors.Join(errs...)
+}
+
+func (r *fallbackResolver) FlushCacheFor(did string) {
+	for _, res := range r.resolvers {
+		res.FlushCacheFor(did)
+	}
+}