Add a script for breaking up records table into partitions

main
Max Ignatenko 2024-02-19 19:05:50 +00:00
parent 8b86f8f929
commit 6b400b1f43
3 changed files with 63 additions and 2 deletions

View File

@ -1,9 +1,9 @@
.PHONY: all build up update down start-db status logs .PHONY: all build up update down start-db status logs psql init-db
all: all:
go test -v ./... go test -v ./...
.env: example.env .env:
@cp example.env .env @cp example.env .env
@echo "Please edit .env to suit your environment before proceeding" @echo "Please edit .env to suit your environment before proceeding"
@exit 1 @exit 1
@ -30,3 +30,9 @@ logs:
psql: psql:
@docker compose exec -it postgres psql -U postgres -d bluesky @docker compose exec -it postgres psql -U postgres -d bluesky
init-db: init.sql
@docker compose up -d --build lister
@sleep 10
@docker compose stop lister
@cat init.sql | docker exec -i "$$(docker compose ps --format '{{.Names}}' postgres)" psql -U postgres -d bluesky

22
init.sql 100644
View File

@ -0,0 +1,22 @@
insert into pds (host) values ('https://agaric.us-west.host.bsky.network'),
('https://amanita.us-east.host.bsky.network'),
('https://blewit.us-west.host.bsky.network'),
('https://boletus.us-west.host.bsky.network'),
('https://bsky.social'),
('https://chaga.us-west.host.bsky.network'),
('https://conocybe.us-west.host.bsky.network'),
('https://enoki.us-east.host.bsky.network'),
('https://hydnum.us-west.host.bsky.network'),
('https://inkcap.us-east.host.bsky.network'),
('https://lepista.us-west.host.bsky.network'),
('https://lionsmane.us-east.host.bsky.network'),
('https://maitake.us-west.host.bsky.network'),
('https://morel.us-east.host.bsky.network'),
('https://oyster.us-east.host.bsky.network'),
('https://porcini.us-east.host.bsky.network'),
('https://puffball.us-east.host.bsky.network'),
('https://russula.us-west.host.bsky.network'),
('https://shiitake.us-east.host.bsky.network'),
('https://shimeji.us-east.host.bsky.network'),
('https://verpa.us-west.host.bsky.network')
on conflict do nothing;

33
partition.sql 100644
View File

@ -0,0 +1,33 @@
alter table records rename to records_like;
create table records
(like records_like including defaults)
partition by list (collection);
drop index idx_repo_record_key;
drop index idx_repo_rev;
alter table records_like drop constraint records_pkey;
create unique index records_pkey on records (id, collection);
create table records_default
partition of records default;
create table records_post
partition of records for values in ('app.bsky.feed.post');
create table records_follow
partition of records for values in ('app.bsky.graph.follow');
create table records_block
partition of records for values in ('app.bsky.graph.block');
create table records_repost
partition of records for values in ('app.bsky.feed.repost');
create table records_profile
partition of records for values in ('app.bsky.actor.profile');
with moved_rows as (
delete from records_like r
where collection <> 'app.bsky.feed.like'
returning r.*
)
insert into records select * from moved_rows;
alter table records attach partition records_like for values in ('app.bsky.feed.like');