feat(loadtest): sync add real 500-session capacity harness
This commit is contained in:
iamxvbaba
parent
ac0566f779
commit
141f2f20c4
39 changed files with 4157 additions and 42 deletions
618
internal/observability/metrics/registry.go
Normal file
618
internal/observability/metrics/registry.go
Normal file
|
|
@ -0,0 +1,618 @@
|
|||
// Package metrics provides a dependency-free Prometheus text exporter for the
|
||||
// bounded runtime signals emitted by telesrv. It deliberately accepts only a
|
||||
// small fixed label shape and caps dynamic series so observability cannot become
|
||||
// an attacker-controlled memory cache.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// One additional fixed series reports observations rejected by this cap, so
|
||||
// the complete exporter remains bounded to 8192 resident series.
|
||||
defaultMaxSeries = int64(8191)
|
||||
maxLabelBytes = 96
|
||||
maxProviderSeries = 1024
|
||||
)
|
||||
|
||||
var durationBuckets = [...]time.Duration{
|
||||
time.Millisecond,
|
||||
5 * time.Millisecond,
|
||||
10 * time.Millisecond,
|
||||
25 * time.Millisecond,
|
||||
50 * time.Millisecond,
|
||||
100 * time.Millisecond,
|
||||
250 * time.Millisecond,
|
||||
500 * time.Millisecond,
|
||||
time.Second,
|
||||
2 * time.Second,
|
||||
5 * time.Second,
|
||||
10 * time.Second,
|
||||
30 * time.Second,
|
||||
}
|
||||
|
||||
// Label is a bounded Prometheus label attached to a provider sample.
|
||||
type Label struct {
|
||||
Name string
|
||||
Value string
|
||||
}
|
||||
|
||||
// GaugeSample is an identity-free point-in-time value supplied at scrape time.
|
||||
type GaugeSample struct {
|
||||
Name string
|
||||
Labels []Label
|
||||
Value float64
|
||||
}
|
||||
|
||||
// GaugeProvider is evaluated only during a scrape. Providers must be bounded
|
||||
// and must not perform unbounded database scans.
|
||||
type GaugeProvider func() []GaugeSample
|
||||
|
||||
type seriesKey struct {
|
||||
name string
|
||||
k1 string
|
||||
v1 string
|
||||
k2 string
|
||||
v2 string
|
||||
k3 string
|
||||
v3 string
|
||||
}
|
||||
|
||||
func newSeriesKey(name string, labels ...Label) seriesKey {
|
||||
key := seriesKey{name: sanitizeMetricName(name)}
|
||||
if len(labels) > 0 {
|
||||
key.k1, key.v1 = sanitizeLabelName(labels[0].Name), sanitizeLabelValue(labels[0].Value)
|
||||
}
|
||||
if len(labels) > 1 {
|
||||
key.k2, key.v2 = sanitizeLabelName(labels[1].Name), sanitizeLabelValue(labels[1].Value)
|
||||
}
|
||||
if len(labels) > 2 {
|
||||
key.k3, key.v3 = sanitizeLabelName(labels[2].Name), sanitizeLabelValue(labels[2].Value)
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
func (k seriesKey) overflow() seriesKey {
|
||||
if k.k1 != "" {
|
||||
k.v1 = "overflow"
|
||||
}
|
||||
if k.k2 != "" {
|
||||
k.v2 = "overflow"
|
||||
}
|
||||
if k.k3 != "" {
|
||||
k.v3 = "overflow"
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
type counterSeries struct {
|
||||
key seriesKey
|
||||
value atomic.Uint64
|
||||
}
|
||||
|
||||
type gaugeSeries struct {
|
||||
key seriesKey
|
||||
value atomic.Int64
|
||||
}
|
||||
|
||||
type histogramSeries struct {
|
||||
key seriesKey
|
||||
buckets [len(durationBuckets)]atomic.Uint64
|
||||
count atomic.Uint64
|
||||
sumNS atomic.Int64
|
||||
}
|
||||
|
||||
func (h *histogramSeries) observe(d time.Duration) {
|
||||
if d < 0 {
|
||||
d = 0
|
||||
}
|
||||
h.count.Add(1)
|
||||
h.sumNS.Add(int64(d))
|
||||
for i, bound := range durationBuckets {
|
||||
if d <= bound {
|
||||
h.buckets[i].Add(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Registry implements the mtprotoedge and rpc metric hooks and serves the
|
||||
// Prometheus text exposition format.
|
||||
type Registry struct {
|
||||
maxSeries int64
|
||||
series atomic.Int64
|
||||
seriesMu sync.Mutex
|
||||
dropped atomic.Uint64
|
||||
counters sync.Map // seriesKey -> *counterSeries
|
||||
gauges sync.Map // seriesKey -> *gaugeSeries
|
||||
hist sync.Map // seriesKey -> *histogramSeries
|
||||
|
||||
providersMu sync.RWMutex
|
||||
providers []GaugeProvider
|
||||
}
|
||||
|
||||
// New returns an empty bounded registry.
|
||||
func New() *Registry {
|
||||
return &Registry{maxSeries: defaultMaxSeries}
|
||||
}
|
||||
|
||||
// AddGaugeProvider registers a bounded point-in-time provider.
|
||||
func (r *Registry) AddGaugeProvider(provider GaugeProvider) {
|
||||
if r == nil || provider == nil {
|
||||
return
|
||||
}
|
||||
r.providersMu.Lock()
|
||||
r.providers = append(r.providers, provider)
|
||||
r.providersMu.Unlock()
|
||||
}
|
||||
|
||||
func (r *Registry) counter(key seriesKey) *counterSeries {
|
||||
if existing, ok := r.counters.Load(key); ok {
|
||||
return existing.(*counterSeries)
|
||||
}
|
||||
r.seriesMu.Lock()
|
||||
defer r.seriesMu.Unlock()
|
||||
if existing, ok := r.counters.Load(key); ok {
|
||||
return existing.(*counterSeries)
|
||||
}
|
||||
if r.series.Load() >= r.maxSeries {
|
||||
r.dropped.Add(1)
|
||||
return &counterSeries{}
|
||||
}
|
||||
created := &counterSeries{key: key}
|
||||
r.counters.Store(key, created)
|
||||
r.series.Add(1)
|
||||
return created
|
||||
}
|
||||
|
||||
func (r *Registry) gauge(key seriesKey) *gaugeSeries {
|
||||
if existing, ok := r.gauges.Load(key); ok {
|
||||
return existing.(*gaugeSeries)
|
||||
}
|
||||
r.seriesMu.Lock()
|
||||
defer r.seriesMu.Unlock()
|
||||
if existing, ok := r.gauges.Load(key); ok {
|
||||
return existing.(*gaugeSeries)
|
||||
}
|
||||
if r.series.Load() >= r.maxSeries {
|
||||
r.dropped.Add(1)
|
||||
return &gaugeSeries{}
|
||||
}
|
||||
created := &gaugeSeries{key: key}
|
||||
r.gauges.Store(key, created)
|
||||
r.series.Add(1)
|
||||
return created
|
||||
}
|
||||
|
||||
func (r *Registry) histogram(key seriesKey) *histogramSeries {
|
||||
if existing, ok := r.hist.Load(key); ok {
|
||||
return existing.(*histogramSeries)
|
||||
}
|
||||
r.seriesMu.Lock()
|
||||
defer r.seriesMu.Unlock()
|
||||
if existing, ok := r.hist.Load(key); ok {
|
||||
return existing.(*histogramSeries)
|
||||
}
|
||||
if r.series.Load() >= r.maxSeries {
|
||||
r.dropped.Add(1)
|
||||
return &histogramSeries{}
|
||||
}
|
||||
created := &histogramSeries{key: key}
|
||||
r.hist.Store(key, created)
|
||||
r.series.Add(1)
|
||||
return created
|
||||
}
|
||||
|
||||
func (r *Registry) inc(name string, labels ...Label) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.counter(newSeriesKey(name, labels...)).value.Add(1)
|
||||
}
|
||||
|
||||
func (r *Registry) add(name string, value uint64, labels ...Label) {
|
||||
if r == nil || value == 0 {
|
||||
return
|
||||
}
|
||||
r.counter(newSeriesKey(name, labels...)).value.Add(value)
|
||||
}
|
||||
|
||||
func (r *Registry) addGauge(name string, delta int64, labels ...Label) {
|
||||
if r == nil || delta == 0 {
|
||||
return
|
||||
}
|
||||
r.gauge(newSeriesKey(name, labels...)).value.Add(delta)
|
||||
}
|
||||
|
||||
func (r *Registry) observe(name string, d time.Duration, labels ...Label) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.histogram(newSeriesKey(name, labels...)).observe(d)
|
||||
}
|
||||
|
||||
// ConnOpened implements mtprotoedge.Metrics.
|
||||
func (r *Registry) ConnOpened() {
|
||||
r.inc("telesrv_mtproto_connections_opened_total")
|
||||
r.addGauge("telesrv_mtproto_connections_active", 1)
|
||||
}
|
||||
|
||||
// ConnClosed implements mtprotoedge.Metrics.
|
||||
func (r *Registry) ConnClosed() {
|
||||
r.inc("telesrv_mtproto_connections_closed_total")
|
||||
r.addGauge("telesrv_mtproto_connections_active", -1)
|
||||
}
|
||||
|
||||
// HandshakeDone implements mtprotoedge.Metrics.
|
||||
func (r *Registry) HandshakeDone(d time.Duration) {
|
||||
r.inc("telesrv_mtproto_handshakes_total")
|
||||
r.observe("telesrv_mtproto_handshake_duration_seconds", d)
|
||||
}
|
||||
|
||||
// RPCHandled implements mtprotoedge.Metrics.
|
||||
func (r *Registry) RPCHandled(method string, d time.Duration, err error) {
|
||||
labels := []Label{{Name: "method", Value: method}, {Name: "outcome", Value: errorOutcome(err)}}
|
||||
r.inc("telesrv_mtproto_rpc_handled_total", labels...)
|
||||
r.observe("telesrv_mtproto_rpc_duration_seconds", d, labels...)
|
||||
}
|
||||
|
||||
// InboundRPCQueued implements mtprotoedge.Metrics.
|
||||
func (r *Registry) InboundRPCQueued(method string, length, capacity int) {
|
||||
r.inc("telesrv_mtproto_inbound_rpc_queued_total", Label{Name: "method", Value: method})
|
||||
r.add("telesrv_mtproto_inbound_rpc_queue_depth_observed_total", uint64(max(length, 0)), Label{Name: "method", Value: method})
|
||||
if capacity > 0 && length >= capacity {
|
||||
r.inc("telesrv_mtproto_inbound_rpc_queue_full_total", Label{Name: "method", Value: method})
|
||||
}
|
||||
}
|
||||
|
||||
// InboundRPCStarted implements mtprotoedge.Metrics.
|
||||
func (r *Registry) InboundRPCStarted(method string, queueWait time.Duration) {
|
||||
r.observe("telesrv_mtproto_inbound_rpc_queue_wait_seconds", queueWait, Label{Name: "method", Value: method})
|
||||
}
|
||||
|
||||
// InboundRPCDropped implements mtprotoedge.Metrics.
|
||||
func (r *Registry) InboundRPCDropped(method, reason string) {
|
||||
r.inc("telesrv_mtproto_inbound_rpc_dropped_total", Label{Name: "method", Value: method}, Label{Name: "reason", Value: reason})
|
||||
}
|
||||
|
||||
// OutboundSend implements mtprotoedge.Metrics.
|
||||
func (r *Registry) OutboundSend(typeID uint32, queueWait time.Duration, bytes int, err error) {
|
||||
labels := []Label{{Name: "type_id", Value: fmt.Sprintf("%08x", typeID)}, {Name: "outcome", Value: errorOutcome(err)}}
|
||||
r.inc("telesrv_mtproto_outbound_send_total", labels...)
|
||||
r.add("telesrv_mtproto_outbound_send_bytes_total", uint64(max(bytes, 0)), labels...)
|
||||
r.observe("telesrv_mtproto_outbound_queue_wait_seconds", queueWait, labels...)
|
||||
}
|
||||
|
||||
// OutboundResend implements mtprotoedge.Metrics.
|
||||
func (r *Registry) OutboundResend(count int, err error) {
|
||||
labels := []Label{{Name: "outcome", Value: errorOutcome(err)}}
|
||||
r.inc("telesrv_mtproto_outbound_resend_requests_total", labels...)
|
||||
r.add("telesrv_mtproto_outbound_resent_frames_total", uint64(max(count, 0)), labels...)
|
||||
}
|
||||
|
||||
// OutboundDropped implements mtprotoedge.Metrics.
|
||||
func (r *Registry) OutboundDropped(reason string) {
|
||||
r.inc("telesrv_mtproto_outbound_dropped_total", Label{Name: "reason", Value: reason})
|
||||
}
|
||||
|
||||
// OutboundQueueWait implements mtprotoedge.Metrics.
|
||||
func (r *Registry) OutboundQueueWait(length, capacity int) {
|
||||
r.inc("telesrv_mtproto_outbound_queue_wait_total")
|
||||
if capacity > 0 && length >= capacity {
|
||||
r.inc("telesrv_mtproto_outbound_queue_full_total")
|
||||
}
|
||||
}
|
||||
|
||||
// RPCResultPrepared implements mtprotoedge.RPCResultMetrics.
|
||||
func (r *Registry) RPCResultPrepared(method, priority string, innerBytes, wireBytes int, compressed bool) {
|
||||
encoding := "plain"
|
||||
if compressed {
|
||||
encoding = "gzip"
|
||||
}
|
||||
labels := []Label{{Name: "method", Value: method}, {Name: "priority", Value: priority}, {Name: "encoding", Value: encoding}}
|
||||
r.inc("telesrv_mtproto_rpc_result_prepared_total", labels...)
|
||||
r.add("telesrv_mtproto_rpc_result_inner_bytes_total", uint64(max(innerBytes, 0)), labels...)
|
||||
r.add("telesrv_mtproto_rpc_result_wire_bytes_total", uint64(max(wireBytes, 0)), labels...)
|
||||
}
|
||||
|
||||
// RPCResultDelivered implements mtprotoedge.RPCResultMetrics.
|
||||
func (r *Registry) RPCResultDelivered(method string, egressLatency time.Duration, wireBytes int, err error) {
|
||||
labels := []Label{{Name: "method", Value: method}, {Name: "outcome", Value: errorOutcome(err)}}
|
||||
r.inc("telesrv_mtproto_rpc_result_delivered_total", labels...)
|
||||
r.add("telesrv_mtproto_rpc_result_delivered_bytes_total", uint64(max(wireBytes, 0)), labels...)
|
||||
r.observe("telesrv_mtproto_rpc_result_egress_seconds", egressLatency, labels...)
|
||||
}
|
||||
|
||||
// LogicalOutboxAcknowledged implements mtprotoedge.LogicalOutboxMetrics.
|
||||
func (r *Registry) LogicalOutboxAcknowledged(bytes int, retainedFor time.Duration, rpcResult bool) {
|
||||
kind := "service_or_update"
|
||||
if rpcResult {
|
||||
kind = "rpc_result"
|
||||
}
|
||||
labels := []Label{{Name: "kind", Value: kind}}
|
||||
r.inc("telesrv_mtproto_logical_outbox_acked_frames_total", labels...)
|
||||
r.add("telesrv_mtproto_logical_outbox_acked_bytes_total", uint64(max(bytes, 0)), labels...)
|
||||
r.observe("telesrv_mtproto_logical_outbox_retained_seconds", retainedFor, labels...)
|
||||
}
|
||||
|
||||
// ConnectionIntake implements mtprotoedge.ConnectionIntakeMetrics.
|
||||
func (r *Registry) ConnectionIntake(stage, outcome string, d time.Duration) {
|
||||
labels := []Label{{Name: "stage", Value: stage}, {Name: "outcome", Value: outcome}}
|
||||
r.inc("telesrv_mtproto_connection_intake_total", labels...)
|
||||
r.observe("telesrv_mtproto_connection_intake_seconds", d, labels...)
|
||||
}
|
||||
|
||||
// MessageSend implements rpc.Metrics.
|
||||
func (r *Registry) MessageSend(d time.Duration, duplicate bool, err error) {
|
||||
dup := "false"
|
||||
if duplicate {
|
||||
dup = "true"
|
||||
}
|
||||
labels := []Label{{Name: "outcome", Value: errorOutcome(err)}, {Name: "duplicate", Value: dup}}
|
||||
r.inc("telesrv_rpc_message_send_total", labels...)
|
||||
r.observe("telesrv_rpc_message_send_duration_seconds", d, labels...)
|
||||
}
|
||||
|
||||
// MessageRateLimited implements rpc.Metrics.
|
||||
func (r *Registry) MessageRateLimited(retryAfterSeconds int) {
|
||||
r.inc("telesrv_rpc_message_rate_limited_total")
|
||||
r.add("telesrv_rpc_message_rate_limit_wait_seconds_total", uint64(max(retryAfterSeconds, 0)))
|
||||
}
|
||||
|
||||
// OutboxClaimed implements rpc.Metrics.
|
||||
func (r *Registry) OutboxClaimed(count int) {
|
||||
r.add("telesrv_rpc_outbox_claimed_total", uint64(max(count, 0)))
|
||||
}
|
||||
|
||||
// OutboxDelivered implements rpc.Metrics.
|
||||
func (r *Registry) OutboxDelivered(d time.Duration) {
|
||||
r.inc("telesrv_rpc_outbox_delivered_total")
|
||||
r.observe("telesrv_rpc_outbox_delivery_seconds", d)
|
||||
}
|
||||
|
||||
// OutboxFailed implements rpc.Metrics.
|
||||
func (r *Registry) OutboxFailed(err error) {
|
||||
r.inc("telesrv_rpc_outbox_failed_total", Label{Name: "outcome", Value: errorOutcome(err)})
|
||||
}
|
||||
|
||||
// ServeHTTP writes Prometheus text format.
|
||||
func (r *Registry) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
var counters []*counterSeries
|
||||
r.counters.Range(func(_, value any) bool {
|
||||
counters = append(counters, value.(*counterSeries))
|
||||
return true
|
||||
})
|
||||
var gauges []*gaugeSeries
|
||||
r.gauges.Range(func(_, value any) bool {
|
||||
gauges = append(gauges, value.(*gaugeSeries))
|
||||
return true
|
||||
})
|
||||
var histograms []*histogramSeries
|
||||
r.hist.Range(func(_, value any) bool {
|
||||
histograms = append(histograms, value.(*histogramSeries))
|
||||
return true
|
||||
})
|
||||
|
||||
providerSamples := r.providerSamples()
|
||||
sort.Slice(counters, func(i, j int) bool { return lessSeries(counters[i].key, counters[j].key) })
|
||||
sort.Slice(gauges, func(i, j int) bool { return lessSeries(gauges[i].key, gauges[j].key) })
|
||||
sort.Slice(histograms, func(i, j int) bool { return lessSeries(histograms[i].key, histograms[j].key) })
|
||||
sort.Slice(providerSamples, func(i, j int) bool {
|
||||
if providerSamples[i].Name != providerSamples[j].Name {
|
||||
return providerSamples[i].Name < providerSamples[j].Name
|
||||
}
|
||||
return labelsString(providerSamples[i].Labels) < labelsString(providerSamples[j].Labels)
|
||||
})
|
||||
|
||||
var out strings.Builder
|
||||
fmt.Fprintln(&out, "# TYPE telesrv_metrics_dropped_observations_total counter")
|
||||
fmt.Fprintf(&out, "telesrv_metrics_dropped_observations_total %d\n", r.dropped.Load())
|
||||
writeCounterSeries(&out, counters)
|
||||
writeGaugeSeries(&out, gauges, providerSamples)
|
||||
writeHistogramSeries(&out, histograms)
|
||||
_, _ = w.Write([]byte(out.String()))
|
||||
}
|
||||
|
||||
func (r *Registry) providerSamples() (samples []GaugeSample) {
|
||||
r.providersMu.RLock()
|
||||
providers := append([]GaugeProvider(nil), r.providers...)
|
||||
r.providersMu.RUnlock()
|
||||
for _, provider := range providers {
|
||||
func() {
|
||||
defer func() { _ = recover() }()
|
||||
for _, sample := range provider() {
|
||||
if len(samples) >= maxProviderSeries {
|
||||
r.dropped.Add(1)
|
||||
return
|
||||
}
|
||||
sample.Name = sanitizeMetricName(sample.Name)
|
||||
if len(sample.Labels) > 3 {
|
||||
sample.Labels = sample.Labels[:3]
|
||||
}
|
||||
for i := range sample.Labels {
|
||||
sample.Labels[i].Name = sanitizeLabelName(sample.Labels[i].Name)
|
||||
sample.Labels[i].Value = sanitizeLabelValue(sample.Labels[i].Value)
|
||||
}
|
||||
samples = append(samples, sample)
|
||||
}
|
||||
}()
|
||||
}
|
||||
return samples
|
||||
}
|
||||
|
||||
func writeCounterSeries(out *strings.Builder, series []*counterSeries) {
|
||||
last := ""
|
||||
for _, item := range series {
|
||||
if item.key.name != last {
|
||||
fmt.Fprintf(out, "# TYPE %s counter\n", item.key.name)
|
||||
last = item.key.name
|
||||
}
|
||||
writeSample(out, item.key.name, keyLabels(item.key), float64(item.value.Load()))
|
||||
}
|
||||
}
|
||||
|
||||
func writeGaugeSeries(out *strings.Builder, series []*gaugeSeries, provider []GaugeSample) {
|
||||
type sample struct {
|
||||
name string
|
||||
labels []Label
|
||||
value float64
|
||||
}
|
||||
all := make([]sample, 0, len(series)+len(provider))
|
||||
for _, item := range series {
|
||||
all = append(all, sample{name: item.key.name, labels: keyLabels(item.key), value: float64(item.value.Load())})
|
||||
}
|
||||
for _, item := range provider {
|
||||
all = append(all, sample{name: item.Name, labels: item.Labels, value: item.Value})
|
||||
}
|
||||
sort.Slice(all, func(i, j int) bool {
|
||||
if all[i].name != all[j].name {
|
||||
return all[i].name < all[j].name
|
||||
}
|
||||
return labelsString(all[i].labels) < labelsString(all[j].labels)
|
||||
})
|
||||
last := ""
|
||||
for _, item := range all {
|
||||
if item.name != last {
|
||||
fmt.Fprintf(out, "# TYPE %s gauge\n", item.name)
|
||||
last = item.name
|
||||
}
|
||||
writeSample(out, item.name, item.labels, item.value)
|
||||
}
|
||||
}
|
||||
|
||||
func writeHistogramSeries(out *strings.Builder, series []*histogramSeries) {
|
||||
last := ""
|
||||
for _, item := range series {
|
||||
if item.key.name != last {
|
||||
fmt.Fprintf(out, "# TYPE %s histogram\n", item.key.name)
|
||||
last = item.key.name
|
||||
}
|
||||
labels := keyLabels(item.key)
|
||||
for i, bound := range durationBuckets {
|
||||
bucketLabels := append(append([]Label(nil), labels...), Label{Name: "le", Value: strconv.FormatFloat(bound.Seconds(), 'g', -1, 64)})
|
||||
writeSample(out, item.key.name+"_bucket", bucketLabels, float64(item.buckets[i].Load()))
|
||||
}
|
||||
writeSample(out, item.key.name+"_bucket", append(append([]Label(nil), labels...), Label{Name: "le", Value: "+Inf"}), float64(item.count.Load()))
|
||||
writeSample(out, item.key.name+"_sum", labels, time.Duration(item.sumNS.Load()).Seconds())
|
||||
writeSample(out, item.key.name+"_count", labels, float64(item.count.Load()))
|
||||
}
|
||||
}
|
||||
|
||||
func writeSample(out *strings.Builder, name string, labels []Label, value float64) {
|
||||
out.WriteString(name)
|
||||
if len(labels) > 0 {
|
||||
out.WriteByte('{')
|
||||
for i, label := range labels {
|
||||
if i > 0 {
|
||||
out.WriteByte(',')
|
||||
}
|
||||
out.WriteString(label.Name)
|
||||
out.WriteString("=\"")
|
||||
out.WriteString(escapeLabel(label.Value))
|
||||
out.WriteByte('"')
|
||||
}
|
||||
out.WriteByte('}')
|
||||
}
|
||||
out.WriteByte(' ')
|
||||
out.WriteString(strconv.FormatFloat(value, 'g', -1, 64))
|
||||
out.WriteByte('\n')
|
||||
}
|
||||
|
||||
func keyLabels(key seriesKey) []Label {
|
||||
labels := make([]Label, 0, 3)
|
||||
if key.k1 != "" {
|
||||
labels = append(labels, Label{Name: key.k1, Value: key.v1})
|
||||
}
|
||||
if key.k2 != "" {
|
||||
labels = append(labels, Label{Name: key.k2, Value: key.v2})
|
||||
}
|
||||
if key.k3 != "" {
|
||||
labels = append(labels, Label{Name: key.k3, Value: key.v3})
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
func lessSeries(a, b seriesKey) bool {
|
||||
if a.name != b.name {
|
||||
return a.name < b.name
|
||||
}
|
||||
return a.k1+a.v1+a.k2+a.v2+a.k3+a.v3 < b.k1+b.v1+b.k2+b.v2+b.k3+b.v3
|
||||
}
|
||||
|
||||
func labelsString(labels []Label) string {
|
||||
var b strings.Builder
|
||||
for _, label := range labels {
|
||||
b.WriteString(label.Name)
|
||||
b.WriteByte('=')
|
||||
b.WriteString(label.Value)
|
||||
b.WriteByte(',')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func errorOutcome(err error) string {
|
||||
if err == nil {
|
||||
return "ok"
|
||||
}
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return "canceled"
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return "timeout"
|
||||
}
|
||||
message := strings.ToUpper(err.Error())
|
||||
switch {
|
||||
case strings.Contains(message, "FLOOD_WAIT"):
|
||||
return "flood_wait"
|
||||
case strings.Contains(message, "WORKER_BUSY") || strings.Contains(message, "BUDGET") || strings.Contains(message, "CAPACITY"):
|
||||
return "edge_overload"
|
||||
default:
|
||||
return "error"
|
||||
}
|
||||
}
|
||||
|
||||
func sanitizeMetricName(value string) string {
|
||||
if value == "" {
|
||||
return "telesrv_invalid_metric"
|
||||
}
|
||||
var b strings.Builder
|
||||
for i, r := range value {
|
||||
valid := r == '_' || r == ':' || r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || i > 0 && r >= '0' && r <= '9'
|
||||
if valid {
|
||||
b.WriteRune(r)
|
||||
} else {
|
||||
b.WriteByte('_')
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func sanitizeLabelName(value string) string {
|
||||
return strings.ReplaceAll(sanitizeMetricName(value), ":", "_")
|
||||
}
|
||||
|
||||
func sanitizeLabelValue(value string) string {
|
||||
if value == "" {
|
||||
return "unknown"
|
||||
}
|
||||
if len(value) > maxLabelBytes {
|
||||
return value[:maxLabelBytes]
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func escapeLabel(value string) string {
|
||||
value = strings.ReplaceAll(value, "\\", "\\\\")
|
||||
value = strings.ReplaceAll(value, "\n", "\\n")
|
||||
return strings.ReplaceAll(value, "\"", "\\\"")
|
||||
}
|
||||
83
internal/observability/metrics/registry_test.go
Normal file
83
internal/observability/metrics/registry_test.go
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
package metrics
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"telesrv/internal/mtprotoedge"
|
||||
"telesrv/internal/rpc"
|
||||
)
|
||||
|
||||
var (
|
||||
_ mtprotoedge.Metrics = (*Registry)(nil)
|
||||
_ mtprotoedge.RPCResultMetrics = (*Registry)(nil)
|
||||
_ mtprotoedge.LogicalOutboxMetrics = (*Registry)(nil)
|
||||
_ mtprotoedge.ConnectionIntakeMetrics = (*Registry)(nil)
|
||||
_ rpc.Metrics = (*Registry)(nil)
|
||||
)
|
||||
|
||||
func TestRegistryExportsBoundedAggregateMetrics(t *testing.T) {
|
||||
registry := New()
|
||||
registry.maxSeries = 2
|
||||
registry.RPCHandled("help.getConfig", 5*time.Millisecond, nil)
|
||||
registry.RPCHandled("users.getUsers", time.Second, errors.New("secret auth_key_id=deadbeef session=123"))
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
registry.ServeHTTP(recorder, httptest.NewRequest("GET", "/metrics", nil))
|
||||
body := recorder.Body.String()
|
||||
if recorder.Code != 200 {
|
||||
t.Fatalf("status = %d, want 200", recorder.Code)
|
||||
}
|
||||
if !strings.Contains(body, "telesrv_mtproto_rpc_handled_total") || !strings.Contains(body, "telesrv_mtproto_rpc_duration_seconds_bucket") {
|
||||
t.Fatalf("expected RPC counter and histogram, got:\n%s", body)
|
||||
}
|
||||
if strings.Contains(body, "deadbeef") || strings.Contains(body, "session=123") {
|
||||
t.Fatalf("raw error identity leaked into metrics:\n%s", body)
|
||||
}
|
||||
if got := registry.series.Load(); got != registry.maxSeries {
|
||||
t.Fatalf("resident dynamic series = %d, want cap %d", got, registry.maxSeries)
|
||||
}
|
||||
if got := registry.dropped.Load(); got == 0 {
|
||||
t.Fatal("series overflow was not reported")
|
||||
}
|
||||
if !strings.Contains(body, "telesrv_metrics_dropped_observations_total 2") {
|
||||
t.Fatalf("overflow counter missing from:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrySanitizesAndBoundsProviderSamples(t *testing.T) {
|
||||
registry := New()
|
||||
registry.AddGaugeProvider(func() []GaugeSample {
|
||||
return []GaugeSample{{
|
||||
Name: "9 invalid metric",
|
||||
Labels: []Label{{Name: "bad:label", Value: "quoted\"\nvalue"}},
|
||||
Value: 3,
|
||||
}}
|
||||
})
|
||||
recorder := httptest.NewRecorder()
|
||||
registry.ServeHTTP(recorder, httptest.NewRequest("GET", "/metrics", nil))
|
||||
body := recorder.Body.String()
|
||||
if !strings.Contains(body, `__invalid_metric{bad_label="quoted\"\nvalue"} 3`) {
|
||||
t.Fatalf("provider sample was not safely sanitized:\n%s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrorOutcomeHasFixedCardinality(t *testing.T) {
|
||||
tests := []struct {
|
||||
err error
|
||||
want string
|
||||
}{
|
||||
{nil, "ok"},
|
||||
{errors.New("FLOOD_WAIT_1 for phone 123"), "flood_wait"},
|
||||
{errors.New("global capacity exceeded for auth key"), "edge_overload"},
|
||||
{errors.New("arbitrary user-controlled failure"), "error"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
if got := errorOutcome(test.err); got != test.want {
|
||||
t.Errorf("errorOutcome(%v) = %q, want %q", test.err, got, test.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue