From fe731d43cd0488e5bf8051d66f70b98b729738fd Mon Sep 17 00:00:00 2001 From: binwiederhier Date: Tue, 14 Mar 2023 10:19:15 -0400 Subject: [PATCH] More metrics --- go.mod | 2 +- go.sum | 13 +++------- server/file_cache.go | 2 ++ server/server.go | 25 +++++++++++++++--- server/server_metrics.go | 52 +++++++++++++++++++++++++++++-------- server/server_middleware.go | 1 + server/smtp_server.go | 2 ++ server/util.go | 6 ++--- 8 files changed, 75 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 5bea5aca..3f5d5af8 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require github.com/pkg/errors v0.9.1 // indirect require ( firebase.google.com/go/v4 v4.10.0 + github.com/prometheus/client_golang v1.14.0 github.com/stripe/stripe-go/v74 v74.10.0 ) @@ -51,7 +52,6 @@ require ( github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect diff --git a/go.sum b/go.sum index 4eb15f7b..0dc5f1c3 100644 --- a/go.sum +++ b/go.sum @@ -151,6 +151,7 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= @@ -189,8 +190,10 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= @@ -256,8 +259,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stripe/stripe-go/v74 v74.9.0 h1:yQ3O8jmtoAjKARzjLGmwYj2ZxqYbdtWVjFeovNGDtjg= -github.com/stripe/stripe-go/v74 v74.9.0/go.mod h1:5PoXNp30AJ3tGq57ZcFuaMylzNi8KpwlrYAFmO1fHZw= github.com/stripe/stripe-go/v74 v74.10.0 h1:Edd5uO1/41wyd163ZTTA8b+8t/wVgdnJQk3Ry1lbLIs= github.com/stripe/stripe-go/v74 v74.10.0/go.mod h1:f9L6LvaXa35ja7eyvP6GQswoaIPaBRvGAimAO+udbBw= github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU= @@ -409,8 +410,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -424,8 +423,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -536,8 +533,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230227214838-9b19f0bdc514 h1:rtNKfB++wz5mtDY2t5C8TXlU5y52ojSu7tZo0z7u8eQ= -google.golang.org/genproto v0.0.0-20230227214838-9b19f0bdc514/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 h1:QQF+HdiI4iocoxUjjpLgvTYDHKm99C/VtTBFnfiCJos= google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -570,9 +565,9 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/server/file_cache.go b/server/file_cache.go index 35cb0f4b..e820519c 100644 --- a/server/file_cache.go +++ b/server/file_cache.go @@ -67,6 +67,7 @@ func (c *fileCache) Write(id string, in io.Reader, limiters ...util.Limiter) (in } c.mu.Lock() c.totalSizeCurrent += size + metrics.attachmentsTotalSize.Set(float64(c.totalSizeCurrent)) c.mu.Unlock() return size, nil } @@ -89,6 +90,7 @@ func (c *fileCache) Remove(ids ...string) error { c.mu.Lock() c.totalSizeCurrent = size c.mu.Unlock() + metrics.attachmentsTotalSize.Set(float64(size)) return nil } diff --git a/server/server.go b/server/server.go index 8454869c..f2a478d2 100644 --- a/server/server.go +++ b/server/server.go @@ -596,8 +596,14 @@ func (s *Server) handleMatrixDiscovery(w http.ResponseWriter) error { } func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*message, error) { - t := fromContext[*topic](r, contextTopic) - vrate := fromContext[*visitor](r, contextRateVisitor) + t, err := fromContext[*topic](r, contextTopic) + if err != nil { + return nil, err + } + vrate, err := fromContext[*visitor](r, contextRateVisitor) + if err != nil { + return nil, err + } body, err := util.Peek(r.Body, s.config.MessageLimit) if err != nil { return nil, err @@ -676,6 +682,9 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*message, e s.mu.Lock() s.messages++ s.mu.Unlock() + if unifiedpush { + metrics.unifiedPushPublishedSuccess.Inc() + } return m, nil } @@ -693,9 +702,16 @@ func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v * _, err := s.handlePublishInternal(r, v) if err != nil { metrics.messagesPublishedFailure.Inc() + metrics.matrixPublishedFailure.Inc() if e, ok := err.(*errHTTP); ok && e.HTTPCode == errHTTPInsufficientStorageUnifiedPush.HTTPCode { - topic := fromContext[*topic](r, contextTopic) - pushKey := fromContext[string](r, contextMatrixPushKey) + topic, err := fromContext[*topic](r, contextTopic) + if err != nil { + return err + } + pushKey, err := fromContext[string](r, contextMatrixPushKey) + if err != nil { + return err + } if time.Since(topic.LastAccess()) > matrixRejectPushKeyForUnifiedPushTopicWithoutRateVisitorAfter { return writeMatrixResponse(w, pushKey) } @@ -703,6 +719,7 @@ func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v * return err } metrics.messagesPublishedSuccess.Inc() + metrics.matrixPublishedSuccess.Inc() return writeMatrixSuccess(w) } diff --git a/server/server_metrics.go b/server/server_metrics.go index 7aa9b2db..36c78c94 100644 --- a/server/server_metrics.go +++ b/server/server_metrics.go @@ -9,17 +9,23 @@ var ( ) type serverMetrics struct { - messagesPublishedSuccess prometheus.Counter - messagesPublishedFailure prometheus.Counter - messagesCached prometheus.Gauge - firebasePublishedSuccess prometheus.Counter - firebasePublishedFailure prometheus.Counter - emailsPublishedSuccess prometheus.Counter - emailsPublishedFailure prometheus.Counter - visitors prometheus.Gauge - subscribers prometheus.Gauge - topics prometheus.Gauge - httpRequests *prometheus.CounterVec + messagesPublishedSuccess prometheus.Counter + messagesPublishedFailure prometheus.Counter + messagesCached prometheus.Gauge + firebasePublishedSuccess prometheus.Counter + firebasePublishedFailure prometheus.Counter + emailsPublishedSuccess prometheus.Counter + emailsPublishedFailure prometheus.Counter + emailsReceivedSuccess prometheus.Counter + emailsReceivedFailure prometheus.Counter + unifiedPushPublishedSuccess prometheus.Counter + matrixPublishedSuccess prometheus.Counter + matrixPublishedFailure prometheus.Counter + attachmentsTotalSize prometheus.Gauge + visitors prometheus.Gauge + subscribers prometheus.Gauge + topics prometheus.Gauge + httpRequests *prometheus.CounterVec } func newMetrics() *serverMetrics { @@ -45,6 +51,24 @@ func newMetrics() *serverMetrics { emailsPublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{ Name: "ntfy_emails_sent_failure", }), + emailsReceivedSuccess: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "ntfy_emails_received_success", + }), + emailsReceivedFailure: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "ntfy_emails_received_failure", + }), + unifiedPushPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "ntfy_unifiedpush_published_success", + }), + matrixPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "ntfy_matrix_published_success", + }), + matrixPublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "ntfy_matrix_published_failure", + }), + attachmentsTotalSize: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "ntfy_attachments_total_size", + }), visitors: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "ntfy_visitors_total", }), @@ -66,6 +90,12 @@ func newMetrics() *serverMetrics { m.firebasePublishedFailure, m.emailsPublishedSuccess, m.emailsPublishedFailure, + m.emailsReceivedSuccess, + m.emailsReceivedFailure, + m.unifiedPushPublishedSuccess, + m.matrixPublishedSuccess, + m.matrixPublishedFailure, + m.attachmentsTotalSize, m.visitors, m.subscribers, m.topics, diff --git a/server/server_middleware.go b/server/server_middleware.go index 5c83cf70..3e12ba07 100644 --- a/server/server_middleware.go +++ b/server/server_middleware.go @@ -12,6 +12,7 @@ const ( contextRateVisitor contextKey = iota + 2586 contextTopic contextMatrixPushKey + contextUnifiedPush ) func (s *Server) limitRequests(next handleFunc) handleFunc { diff --git a/server/smtp_server.go b/server/smtp_server.go index f6844521..4d3cdf22 100644 --- a/server/smtp_server.go +++ b/server/smtp_server.go @@ -165,6 +165,7 @@ func (s *smtpSession) Data(r io.Reader) error { s.backend.mu.Lock() s.backend.success++ s.backend.mu.Unlock() + metrics.emailsReceivedSuccess.Inc() return nil }) } @@ -217,6 +218,7 @@ func (s *smtpSession) withFailCount(fn func() error) error { // We do not want to spam the log with WARN messages. logem(s.conn).Err(err).Debug("Incoming mail error") s.backend.failure++ + metrics.emailsReceivedFailure.Inc() } return err } diff --git a/server/util.go b/server/util.go index c719118b..390e7fb1 100644 --- a/server/util.go +++ b/server/util.go @@ -107,10 +107,10 @@ func withContext(r *http.Request, ctx map[contextKey]any) *http.Request { return r.WithContext(c) } -func fromContext[T any](r *http.Request, key contextKey) T { +func fromContext[T any](r *http.Request, key contextKey) (T, error) { t, ok := r.Context().Value(key).(T) if !ok { - panic(fmt.Sprintf("cannot find key %v in request context", key)) + return t, fmt.Errorf("cannot find key %v in request context", key) } - return t + return t, nil }