72 lines
2.3 KiB
YAML
72 lines
2.3 KiB
YAML
databases:
|
|
db1:
|
|
dsn: env:POSTGRES_URL
|
|
|
|
metrics:
|
|
repos_fully_indexed:
|
|
type: gauge
|
|
description: Repositories fully indexed
|
|
repos_seen:
|
|
type: gauge
|
|
description: Repositories seen
|
|
repos_failed:
|
|
type: gauge
|
|
description: Repositories that we failed to index
|
|
consumer_bad_records:
|
|
type: gauge
|
|
description: Records received from firehose that we failed to process
|
|
labels: [pds, error]
|
|
# posts_lang:
|
|
# type: summary
|
|
# description: Posts by language
|
|
# labels: [uk, lt, et, lv, pl, ga, fi, sv,
|
|
# en, jp, de, fr, pt, es, nl, ko, tr, zh, ru]
|
|
|
|
queries:
|
|
query1:
|
|
interval: 30
|
|
databases: [db1]
|
|
metrics: [repos_fully_indexed]
|
|
sql: >
|
|
select count(*) as repos_fully_indexed
|
|
from repos left join pds on repos.pds = pds.id
|
|
where failed_attempts < 3
|
|
and last_indexed_rev <> ''
|
|
and (last_indexed_rev >= first_rev_since_reset
|
|
or first_rev_since_reset is null or first_rev_since_reset = '')
|
|
and (repos.first_cursor_since_reset >= pds.first_cursor_since_reset
|
|
or repos.first_cursor_since_reset is null or repos.first_cursor_since_reset = 0);
|
|
query2:
|
|
interval: 30
|
|
databases: [db1]
|
|
metrics: [repos_seen]
|
|
sql: select count(*) as repos_seen from repos;
|
|
query3:
|
|
interval: 30
|
|
databases: [db1]
|
|
metrics: [repos_failed]
|
|
sql: select count(*) as repos_failed from repos where failed_attempts >= 3;
|
|
# query4:
|
|
# interval: 300
|
|
# databases: [db1]
|
|
# metrics: [posts_lang]
|
|
# sql: select count(*) as uk from records where collection in ('app.bsky.feed.post') and content::text like '%"langs": ["uk"]%';
|
|
bad_records:
|
|
interval: 30
|
|
databases: [db1]
|
|
metrics: [consumer_bad_records]
|
|
sql: |
|
|
select count(*) as consumer_bad_records, host as pds, error
|
|
from (
|
|
select id, created_at, pds, cursor, content,
|
|
regexp_replace(regexp_replace(regexp_replace(error,
|
|
'did:[\:a-z0-9]+', 'did:xxx', 'g'),
|
|
'json\.RawMessage\{[^}]+\}', 'json.RawMessage{...}', 'g'),
|
|
'[0-9]{1,3}(\.[0-9]{1,3}){3}\:[0-9]+', '<IP>\:<port>', 'g') as error
|
|
from bad_records
|
|
)
|
|
join
|
|
pds
|
|
on pds=pds.id
|
|
group by error, host;
|