astra
/
plc-mirror
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Export the number of bad firehose records for each PDS 

Fixes #1
main
Max Ignatenko 2024-03-29 12:02:05 +00:00
parent 7c09c37a51
commit 89d3a264c9
1 changed files with 22 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
metrics/prometheus/exporters/query-exporter/config.yaml
View File

@ -12,6 +12,10 @@ metrics:
repos_failed: repos_failed:
type: gauge type: gauge
description: Repositories that we failed to index description: Repositories that we failed to index
consumer_bad_records:
type: gauge
description: Records received from firehose that we failed to process
labels: [pds, error]
# posts_lang: # posts_lang:
# type: summary # type: summary
# description: Posts by language # description: Posts by language
@ -47,3 +51,21 @@ queries:
# databases: [db1] # databases: [db1]
# metrics: [posts_lang] # metrics: [posts_lang]
# sql: select count(*) as uk from records where collection in ('app.bsky.feed.post') and content::text like '%"langs": ["uk"]%'; # sql: select count(*) as uk from records where collection in ('app.bsky.feed.post') and content::text like '%"langs": ["uk"]%';
bad_records:
interval: 30
databases: [db1]
metrics: [consumer_bad_records]
sql: |
select count(*) as consumer_bad_records, host as pds, error
from (
select id, created_at, pds, cursor, content,
regexp_replace(regexp_replace(regexp_replace(error,
'did:[\:a-z0-9]+', 'did:xxx', 'g'),
'json\.RawMessage\{[^}]+\}', 'json.RawMessage{...}', 'g'),
'[0-9]{1,3}(\.[0-9]{1,3}){3}\:[0-9]+', '<IP>\:<port>', 'g') as error
from bad_records
)
join
pds
on pds=pds.id
group by error, host;