Added metric for posts by lang (olivčykom).

cmd/record-indexer/metrics.go

@@ -35,6 +35,11 @@ var recordsInserted = promauto.NewCounter(prometheus.CounterOpts{
 	Help: "Number of records inserted into DB",
 })
 
+// var postsByLanguageIndexed = promauto.NewCounterVec(prometheus.CounterOpts{
+// 	Name: "indexer_posts_by_language_inserted_count",
+// 	Help: "Number of posts inserted into DB by language",
+// }, []string{"lang"})
+
 var workerPoolSize = promauto.NewGauge(prometheus.GaugeOpts{
 	Name: "indexer_workers_count",
 	Help: "Current number of workers running",

cmd/record-indexer/workerpool.go

@@ -221,6 +221,10 @@ retry:
 			continue
 		}
 		v = regexp.MustCompile(`[^\\](\\\\)*(\\u0000)`).ReplaceAll(v, []byte(`$1<0x00>`))
+		// lang, err := repo.GetLang(ctx, v)
+		// if err == nil {
+		// 	postsByLanguageIndexed.WithLabelValues(u.String(), lang).Inc()
+		// }
 		recs = append(recs, repo.Record{
 			Repo:       models.ID(work.Repo.ID),
 			Collection: parts[0],

db-migration/20240217_partition.sql

@@ -23,6 +23,8 @@ partition of records for values in ('app.bsky.feed.repost');
 create table records_profile
 partition of records for values in ('app.bsky.actor.profile');
 
+
+-- SLOW, can run overnight, make sure to run in tmux or eternal terminal
 with moved_rows as (
         delete from records_like r
         where collection <> 'app.bsky.feed.like'

metrics/grafana/dashboards/indexer-dash.json

@@ -1850,7 +1850,7 @@
         ]
     },
     "time": {
-        "from": "now-1h",
+        "from": "now-6h",
         "to": "now"
     },
     "timepicker": {},

repo/mst.go

@@ -294,3 +294,25 @@ func GetRev(ctx context.Context, b io.Reader) (string, error) {
 	}
 	return s, nil
 }
+
+// func GetLang(ctx context.Context, value json.RawMessage) (string, error) {
+// 	var content map[string]interface{}
+// 	var lang string
+// 	err := json.Unmarshal([]byte(value), &content)
+
+// 	if err != nil {
+// 		return "", fmt.Errorf("failed to extract lang from content")
+// 	}
+
+// 	if content["$type"] != "app.bsky.feed.post" ||
+// 		content["langs"] == nil ||
+// 		content["langs"].([]string) == nil ||
+// 		len(content["langs"].([]string)) == 0 {
+// 		return "", errors.New("not a post")
+// 	}
+
+// 	//todo: do something a bit less dumb than that
+// 	lang = content["langs"].([]string)[0]
+
+// 	return lang, nil
+// }