astra
/
plc-mirror
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update iexport.

main
mathan 2024-03-27 16:18:18 -07:00
parent ffa2faa420
commit 3aca467516
3 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -66,7 +66,7 @@ csv-export:
csv-iexport: csv-iexport:
@docker compose up -d postgres @docker compose up -d postgres
@sleep 10 @sleep 10
@nohup ./csv_iexport.sh > csv_iexport.out & @sudo nohup ./csv_iexport.sh > csv_iexport.out &
kill-csv-export: kill-csv-export:
@kill -9 `pgrep csv_export.sh` @kill -9 `pgrep csv_export.sh`

16
csv_iexport.sh
View File

@ -5,8 +5,8 @@ set -e
# ------------------------------ Write data timestamp ---------------------------------- # ------------------------------ Write data timestamp ----------------------------------
to_timestamp=$(date -Iseconds --utc) to_timestamp=$(date -Iseconds --utc)
echo "export_start" > timestamp.csv echo "export_start" > ${CSV_DIR}/timestamp.csv
echo "${to_timestamp}" >> timestamp.csv echo "${to_timestamp}" >> ${CSV_DIR}/timestamp.csv
# ------------------------------ Refresh views ---------------------------------- # ------------------------------ Refresh views ----------------------------------
@ -33,7 +33,7 @@ folows_started=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "insert into incremental_export_log (started, to_tsmp, collection) values ('$folows_started', '$to_timestamp', 'app.bsky.graph.follow')" -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$folows_started', '$to_timestamp', 'app.bsky.graph.follow')"
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "copy (select * from export_follows) to stdout with csv header;" > follows.csv -c "copy (select * from export_follows) to stdout with csv header;" > ${CSV_DIR}/follows.csv
echo "Finishing follows export..." echo "Finishing follows export..."
folows_finished=$(date -Iseconds --utc) folows_finished=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
@ -44,7 +44,7 @@ block_started=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "insert into incremental_export_log (started, to_tsmp, collection) values ('$block_started', '$to_timestamp', 'app.bsky.graph.block')" -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$block_started', '$to_timestamp', 'app.bsky.graph.block')"
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "copy (select * from export_blocks) to stdout with csv header;" > blocks.csv -c "copy (select * from export_blocks) to stdout with csv header;" > ${CSV_DIR}/blocks.csv
echo "Finishing blocks export..." echo "Finishing blocks export..."
block_finished=$(date -Iseconds --utc) block_finished=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
@ -56,7 +56,7 @@ likes_started=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "insert into incremental_export_log (started, to_tsmp, collection) values ('$likes_started', '$to_timestamp', 'app.bsky.feed.like')" -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$likes_started', '$to_timestamp', 'app.bsky.feed.like')"
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "copy (select * from export_likes) to stdout with csv header;" > like_counts.csv -c "copy (select * from export_likes) to stdout with csv header;" > ${CSV_DIR}/like_counts.csv
echo "Finishing likes export..." echo "Finishing likes export..."
likes_finished=$(date -Iseconds --utc) likes_finished=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
@ -67,7 +67,7 @@ posts_started=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "insert into incremental_export_log (started, to_tsmp, collection) values ('$posts_started', '$to_timestamp', 'app.bsky.feed.post')" -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$posts_started', '$to_timestamp', 'app.bsky.feed.post')"
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "copy (select * from export_replies) to stdout with csv header;" > post_counts.csv -c "copy (select * from export_replies) to stdout with csv header;" > ${CSV_DIR}/post_counts.csv
echo "Finishing posts export..." echo "Finishing posts export..."
posts_finished=$(date -Iseconds --utc) posts_finished=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
@ -78,7 +78,7 @@ dids_started=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "insert into incremental_export_log (started, to_tsmp, collection) values ('$dids_started', '$to_timestamp', 'did')" -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$dids_started', '$to_timestamp', 'did')"
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "copy (select * from export_dids) to stdout with csv header;" > dids.csv -c "copy (select * from export_dids) to stdout with csv header;" > ${CSV_DIR}/dids.csv
echo "Finishing dids export..." echo "Finishing dids export..."
dids_finished=$(date -Iseconds --utc) dids_finished=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
@ -94,7 +94,7 @@ handles_started=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \
-c "insert into incremental_export_log (started, to_tsmp, collection) values ('$handles_started', '$to_timestamp', 'handle')" -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$handles_started', '$to_timestamp', 'handle')"
docker exec -t plc-postgres-1 psql -U postgres -d plc \ docker exec -t plc-postgres-1 psql -U postgres -d plc \
-c 'copy (select handle, did as "did:ID" from actors) to stdout with (format csv , header, force_quote ("handle"));' | sed -E -e 's/([^\\])\\",/\1\\\\",/g' > handles.csv -c 'copy (select handle, did as "did:ID" from actors) to stdout with (format csv , header, force_quote ("handle"));' | sed -E -e 's/([^\\])\\",/\1\\\\",/g' > ${CSV_DIR}/handles.csv
echo "Finishing dids export..." echo "Finishing dids export..."
handles_finished=$(date -Iseconds --utc) handles_finished=$(date -Iseconds --utc)
docker compose exec -it postgres psql -U postgres -d bluesky \ docker compose exec -it postgres psql -U postgres -d bluesky \

1
example.env
View File

@ -1,4 +1,5 @@
POSTGRES_PASSWORD='some password' POSTGRES_PASSWORD='some password'
DATA_DIR=/path/to/storage/dir DATA_DIR=/path/to/storage/dir
CSV_DIR=/path/to/csv/export/dir
# If you have a PLC mirror (and you do want to have it), put it's address here # If you have a PLC mirror (and you do want to have it), put it's address here
#PLC_ADDRESS=http://host.docker.internal:8097 #PLC_ADDRESS=http://host.docker.internal:8097