astra
/
plc-mirror
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Ladder fixed.

main
mathan 2024-05-16 22:17:17 -07:00
parent 64b7533315
commit a9076c44c0
2 changed files with 14 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
csv_iexport.sh
View File

@ -24,13 +24,13 @@ docker compose exec -iT postgres psql -U postgres -d bluesky <<- EOF
\echo Refreshing follows... \echo Refreshing follows...
refresh materialized view export_follows; refresh materialized view export_follows;
\echo Refreshing like counts... \echo Refreshing like counts...
refresh materialized view export_likes; refresh materialized view export_likes_ladder;
\echo Refreshing reply counts... \echo Refreshing reply counts...
refresh materialized view export_replies; refresh materialized view export_replies_ladder;
\echo Refreshing block list... \echo Refreshing block list...
refresh materialized view export_blocks; refresh materialized view export_blocks;
\echo Refreshing DID list... \echo Refreshing DID list...
refresh materialized view export_dids; refresh materialized view export_dids_ladder;
\echo Refreshing optout list... \echo Refreshing optout list...
refresh materialized view export_optouts; refresh materialized view export_optouts;
EOF EOF
@ -67,7 +67,7 @@ likes_started=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "insert into incremental_export_log (started, to_tsmp, collection) values ('$likes_started', '$to_timestamp', 'app.bsky.feed.like')" -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$likes_started', '$to_timestamp', 'app.bsky.feed.like')"
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "copy (select * from export_likes) to stdout with csv header;" > ${CSV_DIR}/full/${date}/like_counts.csv -c "copy (select * from export_likes_ladder) to stdout with csv header;" > ${CSV_DIR}/full/${date}/like_counts.csv
echo "Finishing likes export..." echo "Finishing likes export..."
likes_finished=$(date -Iseconds --utc) likes_finished=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
@ -78,7 +78,7 @@ posts_started=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "insert into incremental_export_log (started, to_tsmp, collection) values ('$posts_started', '$to_timestamp', 'app.bsky.feed.post')" -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$posts_started', '$to_timestamp', 'app.bsky.feed.post')"
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "copy (select * from export_replies) to stdout with csv header;" > ${CSV_DIR}/full/${date}/post_counts.csv -c "copy (select * from export_replies_ladder) to stdout with csv header;" > ${CSV_DIR}/full/${date}/post_counts.csv
echo "Finishing posts export..." echo "Finishing posts export..."
posts_finished=$(date -Iseconds --utc) posts_finished=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
@ -89,7 +89,7 @@ dids_started=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "insert into incremental_export_log (started, to_tsmp, collection) values ('$dids_started', '$to_timestamp', 'did')" -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$dids_started', '$to_timestamp', 'did')"
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "copy (select * from export_dids) to stdout with csv header;" > ${CSV_DIR}/full/${date}/dids.csv -c "copy (select * from export_dids_ladder) to stdout with csv header;" > ${CSV_DIR}/full/${date}/dids.csv
echo "Finishing dids export..." echo "Finishing dids export..."
dids_finished=$(date -Iseconds --utc) dids_finished=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \

21
db-migration/migrations/do_not_use_20240516_export_ladder_views.sql → db-migration/migrations/20240517_export_ladder_views.sql
View File

@ -1,13 +1,3 @@
CREATE OR REPLACE FUNCTION ladderq(date TIMESTAMP) RETURNS integer AS $$
SELECT
CASE
WHEN date > CURRENT_DATE - INTERVAL '30' DAY THEN 10
WHEN date > CURRENT_DATE - INTERVAL '60' DAY THEN 5
WHEN date > CURRENT_DATE - INTERVAL '90' DAY THEN 3
ELSE 1
END;
$$ LANGUAGE sql STRICT IMMUTABLE;
drop materialized view export_dids_ladder; drop materialized view export_dids_ladder;
drop materialized view export_replies_ladder; drop materialized view export_replies_ladder;
drop materialized view export_likes_ladder; drop materialized view export_likes_ladder;
@ -15,7 +5,10 @@ drop materialized view export_likes_ladder;
create materialized view export_likes_ladder create materialized view export_likes_ladder
as select repos.did as ":START_ID", as select repos.did as ":START_ID",
split_part(jsonb_extract_path_text(content, 'subject', 'uri'), '/', 3) as ":END_ID", split_part(jsonb_extract_path_text(content, 'subject', 'uri'), '/', 3) as ":END_ID",
sum(ladderq(records.created_at::TIMESTAMP)) as "count:long" count(*) FILTER (WHERE records.created_at > CURRENT_DATE - INTERVAL '30' DAY) * 10 +
count(*) FILTER (WHERE records.created_at > CURRENT_DATE - INTERVAL '60' DAY) * 5 +
count(*) FILTER (WHERE records.created_at > CURRENT_DATE - INTERVAL '90' DAY) * 3 +
count(*) FILTER (WHERE records.created_at > CURRENT_DATE - INTERVAL '360' DAY) * 1 as "count:long"
from records join repos on records.repo = repos.id from records join repos on records.repo = repos.id
where records.collection = 'app.bsky.feed.like' where records.collection = 'app.bsky.feed.like'
and repos.did <> split_part(jsonb_extract_path_text(content, 'subject', 'uri'), '/', 3) and repos.did <> split_part(jsonb_extract_path_text(content, 'subject', 'uri'), '/', 3)
@ -26,7 +19,10 @@ create index export_like_subject_ladder on export_likes_ladder (":END_ID");
create materialized view export_replies_ladder create materialized view export_replies_ladder
as select repos.did as ":START_ID", as select repos.did as ":START_ID",
split_part(jsonb_extract_path_text(content, 'reply', 'parent', 'uri'), '/', 3) as ":END_ID", split_part(jsonb_extract_path_text(content, 'reply', 'parent', 'uri'), '/', 3) as ":END_ID",
sum(ladderq(records.created_at::TIMESTAMP)) as "count:long" count(*) FILTER (WHERE records.created_at > CURRENT_DATE - INTERVAL '30' DAY) * 10 +
count(*) FILTER (WHERE records.created_at > CURRENT_DATE - INTERVAL '60' DAY) * 5 +
count(*) FILTER (WHERE records.created_at > CURRENT_DATE - INTERVAL '90' DAY) * 3 +
count(*) FILTER (WHERE records.created_at > CURRENT_DATE - INTERVAL '360' DAY) * 1 as "count:long"
from records join repos on records.repo = repos.id from records join repos on records.repo = repos.id
where records.collection = 'app.bsky.feed.post' where records.collection = 'app.bsky.feed.post'
and repos.did <> split_part(jsonb_extract_path_text(content, 'reply', 'parent', 'uri'), '/', 3) and repos.did <> split_part(jsonb_extract_path_text(content, 'reply', 'parent', 'uri'), '/', 3)
@ -48,5 +44,4 @@ as select distinct did as "did:ID" from (
) )
with no data; with no data;
create index idx_records_repo on records (repo);
create index idx_records_created_at on records (created_at); create index idx_records_created_at on records (created_at);