Export the number of bad firehose records for each PDS

Fixes #1

metrics/prometheus/exporters/query-exporter/config.yaml

@@ -12,6 +12,10 @@ metrics:
   repos_failed:
     type: gauge
     description: Repositories that we failed to index
+  consumer_bad_records:
+    type: gauge
+    description: Records received from firehose that we failed to process
+    labels: [pds, error]
   # posts_lang:
   #   type: summary
   #   description: Posts by language
@@ -47,3 +51,21 @@ queries:
   #   databases: [db1]
   #   metrics: [posts_lang]
   #   sql: select count(*) as uk from records where collection in ('app.bsky.feed.post') and content::text like '%"langs": ["uk"]%';
+  bad_records:
+    interval: 30
+    databases: [db1]
+    metrics: [consumer_bad_records]
+    sql: |
+      select count(*) as consumer_bad_records, host as pds, error
+        from (
+          select id, created_at, pds, cursor, content,
+              regexp_replace(regexp_replace(regexp_replace(error,
+                'did:[\:a-z0-9]+', 'did:xxx', 'g'),
+                'json\.RawMessage\{[^}]+\}', 'json.RawMessage{...}', 'g'),
+                '[0-9]{1,3}(\.[0-9]{1,3}){3}\:[0-9]+', '<IP>\:<port>', 'g') as error
+          from bad_records
+          )
+        join
+          pds
+        on pds=pds.id
+        group by error, host;