Add temporal ladder export. Fix optout export typo.

csv_iexport.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+source .env
 
 set -e
 
@@ -23,13 +24,13 @@ docker compose exec -iT postgres psql -U postgres -d bluesky <<- EOF
 \echo Refreshing follows...
 refresh materialized view export_follows;
 \echo Refreshing like counts...
-refresh materialized view export_likes;
+refresh materialized view export_likes_ladder;
 \echo Refreshing reply counts...
-refresh materialized view export_replies;
+refresh materialized view export_replies_ladder;
 \echo Refreshing block list...
 refresh materialized view export_blocks;
 \echo Refreshing DID list...
-refresh materialized view export_dids;
+refresh materialized view export_dids_ladder;
 \echo Refreshing optout list...
 refresh materialized view export_optouts;
 EOF
@@ -66,7 +67,7 @@ likes_started=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
   -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$likes_started', '$to_timestamp', 'app.bsky.feed.like')"
 docker compose exec -it postgres psql -U postgres -d bluesky \
-  -c "copy (select * from export_likes) to stdout with csv header;" > ${CSV_DIR}/full/${date}/like_counts.csv
+  -c "copy (select * from export_likes_ladder) to stdout with csv header;" > ${CSV_DIR}/full/${date}/like_counts.csv
 echo "Finishing likes export..."
 likes_finished=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
@@ -77,7 +78,7 @@ posts_started=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
   -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$posts_started', '$to_timestamp', 'app.bsky.feed.post')"
 docker compose exec -it postgres psql -U postgres -d bluesky \
-  -c "copy (select * from export_replies) to stdout with csv header;" > ${CSV_DIR}/full/${date}/post_counts.csv
+  -c "copy (select * from export_replies_ladder) to stdout with csv header;" > ${CSV_DIR}/full/${date}/post_counts.csv
 echo "Finishing posts export..."
 posts_finished=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
@@ -88,7 +89,7 @@ dids_started=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
   -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$dids_started', '$to_timestamp', 'did')"
 docker compose exec -it postgres psql -U postgres -d bluesky \
-  -c "copy (select * from export_dids) to stdout with csv header;" > ${CSV_DIR}/full/${date}/dids.csv
+  -c "copy (select * from export_dids_ladder) to stdout with csv header;" > ${CSV_DIR}/full/${date}/dids.csv
 echo "Finishing dids export..."
 dids_finished=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
@@ -96,8 +97,7 @@ docker compose exec -it postgres psql -U postgres -d bluesky \
 
 echo "Starting optouts export..."
 docker compose exec -it postgres psql -U postgres -d bluesky \
-  -c "copy (select did from repos as r inner join records_block as rb on r.id=rb.repo where rb.content['subject']::text like '%did:
-plc:qevje4db3tazfbbialrlrkza%') to stdout with csv header;" > ${CSV_DIR}/full/${date}/optout.csv
+  -c "copy (select did from repos as r inner join records_block as rb on r.id=rb.repo where rb.content['subject']::text like '%did:plc:qevje4db3tazfbbialrlrkza%') to stdout with csv header;" > ${CSV_DIR}/full/${date}/optout.csv
 echo "Finishing optouts export..."
 
 

csv_iexport_month.sh

@@ -96,8 +96,7 @@ docker compose exec -it postgres psql -U postgres -d bluesky \
 
 echo "Starting optouts export..."
 docker compose exec -it postgres psql -U postgres -d bluesky \
-  -c "copy (select did from repos as r inner join records_block as rb on r.id=rb.repo where rb.content['subject']::text like '%did:
-plc:qevje4db3tazfbbialrlrkza%') to stdout with csv header;" > ${CSV_DIR}/monthly/${date}/optout.csv
+  -c "copy (select did from repos as r inner join records_block as rb on r.id=rb.repo where rb.content['subject']::text like '%did:plc:qevje4db3tazfbbialrlrkza%') to stdout with csv header;" > ${CSV_DIR}/monthly/${date}/optout.csv
 echo "Finishing optouts export..."
 
 

db-migration/migrations/20240515_export_ladder_views.sql

@@ -0,0 +1,44 @@
+CREATE OR REPLACE FUNCTION ladderq(date TIMESTAMP) RETURNS integer AS $$
+SELECT
+CASE
+  WHEN date > CURRENT_DATE - INTERVAL '30' DAY THEN 10
+  WHEN date > CURRENT_DATE - INTERVAL '90' DAY THEN 5
+  ELSE 1
+END;
+$$ LANGUAGE sql STRICT IMMUTABLE;
+
+create materialized view export_likes_ladder
+as select repos.did as ":START_ID",
+    split_part(jsonb_extract_path_text(content, 'subject',  'uri'), '/', 3) as ":END_ID",
+    sum(ladderq(records.created_at::TIMESTAMP)) as "count:long"
+  from records join repos on records.repo = repos.id
+  where records.collection = 'app.bsky.feed.like'
+    and repos.did <> split_part(jsonb_extract_path_text(content, 'subject',  'uri'), '/', 3)
+  group by repos.did, split_part(jsonb_extract_path_text(content, 'subject',  'uri'), '/', 3)
+with no data;
+create index export_like_subject_ladder on export_likes_ladder (":END_ID");
+
+create materialized view export_replies_ladder
+as select repos.did as ":START_ID",
+    split_part(jsonb_extract_path_text(content, 'reply', 'parent',  'uri'), '/', 3) as ":END_ID",
+    sum(ladderq(records.created_at::TIMESTAMP)) as "count:long"
+  from records join repos on records.repo = repos.id
+  where records.collection = 'app.bsky.feed.post'
+    and repos.did <> split_part(jsonb_extract_path_text(content, 'reply', 'parent',  'uri'), '/', 3)
+  group by repos.did, split_part(jsonb_extract_path_text(content, 'reply', 'parent',  'uri'), '/', 3)
+with no data;
+create index export_reply_subject_ladder on export_replies_ladder (":END_ID");
+
+create materialized view export_dids_ladder
+as select distinct did as "did:ID" from (
+    select did from repos
+      union
+    select distinct ":END_ID" as did from export_follows
+      union
+    select distinct ":END_ID" as did from export_likes_ladder
+      union
+    select distinct ":END_ID" as did from export_replies_ladder
+      union
+    select distinct ":END_ID" as did from export_blocks
+  )
+with no data;