astra
/
plc-mirror
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added optouts to export.

main
mathan 2024-05-06 17:04:57 -07:00
parent 1201ee3868
commit 53db0fc5e7
2 changed files with 14 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
csv_iexport.sh
View File

@ -22,6 +22,8 @@ refresh materialized view export_replies;
refresh materialized view export_blocks; refresh materialized view export_blocks;
\echo Refreshing DID list... \echo Refreshing DID list...
refresh materialized view export_dids; refresh materialized view export_dids;
\echo Refreshing optout list...
refresh materialized view export_optouts;
EOF EOF
# ------------------------------ Dump views into .csv ---------------------------------- # ------------------------------ Dump views into .csv ----------------------------------
@ -48,7 +50,7 @@ docker compose exec -it postgres psql -U postgres -d bluesky \
echo "Finishing blocks export..." echo "Finishing blocks export..."
block_finished=$(date -Iseconds --utc) block_finished=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "update incremental_export_log set finished='$block_finished' where started='$block_started' and to_tsmp='$to_timestamp' and collection = 'app.bsky.graph.follow'" -c "update incremental_export_log set finished='$block_finished' where started='$block_started' and to_tsmp='$to_timestamp' and collection = 'app.bsky.graph.block'"
echo "Starting likes export..." echo "Starting likes export..."
@ -84,6 +86,12 @@ dids_finished=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "update incremental_export_log set finished='$dids_finished' where started='$dids_started' and to_tsmp='$to_timestamp' and collection = 'did'" -c "update incremental_export_log set finished='$dids_finished' where started='$dids_started' and to_tsmp='$to_timestamp' and collection = 'did'"
echo "Starting optouts export..."
docker compose exec -it postgres psql -U postgres -d bluesky \
-c "copy (select did from repos as r inner join records_block as rb on r.id=rb.repo where rb.content['subject']::text like '%did:
plc:qevje4db3tazfbbialrlrkza%') to stdout with csv header;" > ${CSV_DIR}/optout.csv
echo "Finishing optouts export..."
# ------------------------------ DO NOT Free up space used by materialized views for incremental refresh ---------------------------------- # ------------------------------ DO NOT Free up space used by materialized views for incremental refresh ----------------------------------

5
db-migration/migrations/20240506_export_optouts.sql 100644
View File

@ -0,0 +1,5 @@
drop materialized view export_optouts;
create materialized view export_optouts
as select did as "did:ID" from repos as r inner join records_block as rb on r.id=rb.repo where rb.content['subject']::text like '%did:plc:qevje4db3tazfbbialrlrkza%'
with no data;