Update iexport.

Makefile

@@ -66,7 +66,7 @@ csv-export:
 csv-iexport:
 	@docker compose up -d postgres
 	@sleep 10
-	@nohup ./csv_iexport.sh > csv_iexport.out &
+	@sudo nohup ./csv_iexport.sh > csv_iexport.out &
 
 kill-csv-export:
 	@kill -9 `pgrep csv_export.sh`

csv_iexport.sh

@@ -5,8 +5,8 @@ set -e
 # ------------------------------ Write data timestamp ----------------------------------
 
 to_timestamp=$(date -Iseconds --utc)
-echo "export_start" > timestamp.csv
-echo "${to_timestamp}" >> timestamp.csv
+echo "export_start" > ${CSV_DIR}/timestamp.csv
+echo "${to_timestamp}" >> ${CSV_DIR}/timestamp.csv
 
 # ------------------------------ Refresh views ----------------------------------
 
@@ -33,7 +33,7 @@ folows_started=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
   -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$folows_started', '$to_timestamp', 'app.bsky.graph.follow')"
 docker compose exec -it postgres psql -U postgres -d bluesky \
-  -c "copy (select * from export_follows) to stdout with csv header;" > follows.csv
+  -c "copy (select * from export_follows) to stdout with csv header;" > ${CSV_DIR}/follows.csv
 echo "Finishing follows export..."
 folows_finished=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
@@ -44,7 +44,7 @@ block_started=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
   -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$block_started', '$to_timestamp', 'app.bsky.graph.block')"
 docker compose exec -it postgres psql -U postgres -d bluesky \
-  -c "copy (select * from export_blocks) to stdout with csv header;" > blocks.csv
+  -c "copy (select * from export_blocks) to stdout with csv header;" > ${CSV_DIR}/blocks.csv
 echo "Finishing blocks export..."
 block_finished=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
@@ -56,7 +56,7 @@ likes_started=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
   -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$likes_started', '$to_timestamp', 'app.bsky.feed.like')"
 docker compose exec -it postgres psql -U postgres -d bluesky \
-  -c "copy (select * from export_likes) to stdout with csv header;" > like_counts.csv
+  -c "copy (select * from export_likes) to stdout with csv header;" > ${CSV_DIR}/like_counts.csv
 echo "Finishing likes export..."
 likes_finished=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
@@ -67,7 +67,7 @@ posts_started=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
   -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$posts_started', '$to_timestamp', 'app.bsky.feed.post')"
 docker compose exec -it postgres psql -U postgres -d bluesky \
-  -c "copy (select * from export_replies) to stdout with csv header;" > post_counts.csv
+  -c "copy (select * from export_replies) to stdout with csv header;" > ${CSV_DIR}/post_counts.csv
 echo "Finishing posts export..."
 posts_finished=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
@@ -78,7 +78,7 @@ dids_started=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
   -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$dids_started', '$to_timestamp', 'did')"
 docker compose exec -it postgres psql -U postgres -d bluesky \
-  -c "copy (select * from export_dids) to stdout with csv header;" > dids.csv
+  -c "copy (select * from export_dids) to stdout with csv header;" > ${CSV_DIR}/dids.csv
 echo "Finishing dids export..."
 dids_finished=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
@@ -94,7 +94,7 @@ handles_started=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \
   -c "insert into incremental_export_log (started, to_tsmp, collection) values ('$handles_started', '$to_timestamp', 'handle')"
 docker exec -t plc-postgres-1 psql -U postgres -d plc \
-  -c 'copy (select handle, did as "did:ID" from actors) to stdout with (format csv , header, force_quote ("handle"));' | sed -E -e 's/([^\\])\\",/\1\\\\",/g' > handles.csv
+  -c 'copy (select handle, did as "did:ID" from actors) to stdout with (format csv , header, force_quote ("handle"));' | sed -E -e 's/([^\\])\\",/\1\\\\",/g' > ${CSV_DIR}/handles.csv
 echo "Finishing dids export..."
 handles_finished=$(date -Iseconds --utc)
 docker compose exec -it postgres psql -U postgres -d bluesky \

example.env

@@ -1,4 +1,5 @@
 POSTGRES_PASSWORD='some password'
 DATA_DIR=/path/to/storage/dir
+CSV_DIR=/path/to/csv/export/dir
 # If you have a PLC mirror (and you do want to have it), put it's address here
 #PLC_ADDRESS=http://host.docker.internal:8097