Allow /metrics on default port; reduce memory if not enabled
parent
bb3fe4f830
commit
358b344916
|
@ -40,7 +40,6 @@ var flagsServe = append(
|
||||||
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-http", Aliases: []string{"listen_http", "l"}, EnvVars: []string{"NTFY_LISTEN_HTTP"}, Value: server.DefaultListenHTTP, Usage: "ip:port used as HTTP listen address"}),
|
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-http", Aliases: []string{"listen_http", "l"}, EnvVars: []string{"NTFY_LISTEN_HTTP"}, Value: server.DefaultListenHTTP, Usage: "ip:port used as HTTP listen address"}),
|
||||||
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-https", Aliases: []string{"listen_https", "L"}, EnvVars: []string{"NTFY_LISTEN_HTTPS"}, Usage: "ip:port used as HTTPS listen address"}),
|
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-https", Aliases: []string{"listen_https", "L"}, EnvVars: []string{"NTFY_LISTEN_HTTPS"}, Usage: "ip:port used as HTTPS listen address"}),
|
||||||
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-unix", Aliases: []string{"listen_unix", "U"}, EnvVars: []string{"NTFY_LISTEN_UNIX"}, Usage: "listen on unix socket path"}),
|
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-unix", Aliases: []string{"listen_unix", "U"}, EnvVars: []string{"NTFY_LISTEN_UNIX"}, Usage: "listen on unix socket path"}),
|
||||||
altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-metrics-http", Aliases: []string{"listen_metrics_http"}, EnvVars: []string{"NTFY_LISTEN_METRICS_HTTP"}, Usage: "ip:port used to expose the metrics endpoint"}),
|
|
||||||
altsrc.NewIntFlag(&cli.IntFlag{Name: "listen-unix-mode", Aliases: []string{"listen_unix_mode"}, EnvVars: []string{"NTFY_LISTEN_UNIX_MODE"}, DefaultText: "system default", Usage: "file permissions of unix socket, e.g. 0700"}),
|
altsrc.NewIntFlag(&cli.IntFlag{Name: "listen-unix-mode", Aliases: []string{"listen_unix_mode"}, EnvVars: []string{"NTFY_LISTEN_UNIX_MODE"}, DefaultText: "system default", Usage: "file permissions of unix socket, e.g. 0700"}),
|
||||||
altsrc.NewStringFlag(&cli.StringFlag{Name: "key-file", Aliases: []string{"key_file", "K"}, EnvVars: []string{"NTFY_KEY_FILE"}, Usage: "private key file, if listen-https is set"}),
|
altsrc.NewStringFlag(&cli.StringFlag{Name: "key-file", Aliases: []string{"key_file", "K"}, EnvVars: []string{"NTFY_KEY_FILE"}, Usage: "private key file, if listen-https is set"}),
|
||||||
altsrc.NewStringFlag(&cli.StringFlag{Name: "cert-file", Aliases: []string{"cert_file", "E"}, EnvVars: []string{"NTFY_CERT_FILE"}, Usage: "certificate file, if listen-https is set"}),
|
altsrc.NewStringFlag(&cli.StringFlag{Name: "cert-file", Aliases: []string{"cert_file", "E"}, EnvVars: []string{"NTFY_CERT_FILE"}, Usage: "certificate file, if listen-https is set"}),
|
||||||
|
@ -87,6 +86,8 @@ var flagsServe = append(
|
||||||
altsrc.NewStringFlag(&cli.StringFlag{Name: "stripe-secret-key", Aliases: []string{"stripe_secret_key"}, EnvVars: []string{"NTFY_STRIPE_SECRET_KEY"}, Value: "", Usage: "key used for the Stripe API communication, this enables payments"}),
|
altsrc.NewStringFlag(&cli.StringFlag{Name: "stripe-secret-key", Aliases: []string{"stripe_secret_key"}, EnvVars: []string{"NTFY_STRIPE_SECRET_KEY"}, Value: "", Usage: "key used for the Stripe API communication, this enables payments"}),
|
||||||
altsrc.NewStringFlag(&cli.StringFlag{Name: "stripe-webhook-key", Aliases: []string{"stripe_webhook_key"}, EnvVars: []string{"NTFY_STRIPE_WEBHOOK_KEY"}, Value: "", Usage: "key required to validate the authenticity of incoming webhooks from Stripe"}),
|
altsrc.NewStringFlag(&cli.StringFlag{Name: "stripe-webhook-key", Aliases: []string{"stripe_webhook_key"}, EnvVars: []string{"NTFY_STRIPE_WEBHOOK_KEY"}, Value: "", Usage: "key required to validate the authenticity of incoming webhooks from Stripe"}),
|
||||||
altsrc.NewStringFlag(&cli.StringFlag{Name: "billing-contact", Aliases: []string{"billing_contact"}, EnvVars: []string{"NTFY_BILLING_CONTACT"}, Value: "", Usage: "e-mail or website to display in upgrade dialog (only if payments are enabled)"}),
|
altsrc.NewStringFlag(&cli.StringFlag{Name: "billing-contact", Aliases: []string{"billing_contact"}, EnvVars: []string{"NTFY_BILLING_CONTACT"}, Value: "", Usage: "e-mail or website to display in upgrade dialog (only if payments are enabled)"}),
|
||||||
|
altsrc.NewBoolFlag(&cli.BoolFlag{Name: "enable-metrics", Aliases: []string{"enable_metrics"}, EnvVars: []string{"NTFY_ENABLE_METRICS"}, Value: false, Usage: "if set, Prometheus metrics are exposed via the /metrics endpoint"}),
|
||||||
|
altsrc.NewStringFlag(&cli.StringFlag{Name: "metrics-listen-http", Aliases: []string{"metrics_listen_http"}, EnvVars: []string{"NTFY_METRICS_LISTEN_HTTP"}, Usage: "ip:port used to expose the metrics endpoint (implicitly enables metrics)"}),
|
||||||
)
|
)
|
||||||
|
|
||||||
var cmdServe = &cli.Command{
|
var cmdServe = &cli.Command{
|
||||||
|
@ -119,7 +120,6 @@ func execServe(c *cli.Context) error {
|
||||||
listenHTTPS := c.String("listen-https")
|
listenHTTPS := c.String("listen-https")
|
||||||
listenUnix := c.String("listen-unix")
|
listenUnix := c.String("listen-unix")
|
||||||
listenUnixMode := c.Int("listen-unix-mode")
|
listenUnixMode := c.Int("listen-unix-mode")
|
||||||
listenMetricsHTTP := c.String("listen-metrics-http")
|
|
||||||
keyFile := c.String("key-file")
|
keyFile := c.String("key-file")
|
||||||
certFile := c.String("cert-file")
|
certFile := c.String("cert-file")
|
||||||
firebaseKeyFile := c.String("firebase-key-file")
|
firebaseKeyFile := c.String("firebase-key-file")
|
||||||
|
@ -165,6 +165,8 @@ func execServe(c *cli.Context) error {
|
||||||
stripeSecretKey := c.String("stripe-secret-key")
|
stripeSecretKey := c.String("stripe-secret-key")
|
||||||
stripeWebhookKey := c.String("stripe-webhook-key")
|
stripeWebhookKey := c.String("stripe-webhook-key")
|
||||||
billingContact := c.String("billing-contact")
|
billingContact := c.String("billing-contact")
|
||||||
|
metricsListenHTTP := c.String("metrics-listen-http")
|
||||||
|
enableMetrics := c.Bool("enable-metrics") || metricsListenHTTP != ""
|
||||||
|
|
||||||
// Check values
|
// Check values
|
||||||
if firebaseKeyFile != "" && !util.FileExists(firebaseKeyFile) {
|
if firebaseKeyFile != "" && !util.FileExists(firebaseKeyFile) {
|
||||||
|
@ -271,7 +273,6 @@ func execServe(c *cli.Context) error {
|
||||||
conf.ListenHTTPS = listenHTTPS
|
conf.ListenHTTPS = listenHTTPS
|
||||||
conf.ListenUnix = listenUnix
|
conf.ListenUnix = listenUnix
|
||||||
conf.ListenUnixMode = fs.FileMode(listenUnixMode)
|
conf.ListenUnixMode = fs.FileMode(listenUnixMode)
|
||||||
conf.ListenMetricsHTTP = listenMetricsHTTP
|
|
||||||
conf.KeyFile = keyFile
|
conf.KeyFile = keyFile
|
||||||
conf.CertFile = certFile
|
conf.CertFile = certFile
|
||||||
conf.FirebaseKeyFile = firebaseKeyFile
|
conf.FirebaseKeyFile = firebaseKeyFile
|
||||||
|
@ -318,6 +319,8 @@ func execServe(c *cli.Context) error {
|
||||||
conf.EnableSignup = enableSignup
|
conf.EnableSignup = enableSignup
|
||||||
conf.EnableLogin = enableLogin
|
conf.EnableLogin = enableLogin
|
||||||
conf.EnableReservations = enableReservations
|
conf.EnableReservations = enableReservations
|
||||||
|
conf.EnableMetrics = enableMetrics
|
||||||
|
conf.MetricsListenHTTP = metricsListenHTTP
|
||||||
conf.Version = c.App.Version
|
conf.Version = c.App.Version
|
||||||
|
|
||||||
// Set up hot-reloading of config
|
// Set up hot-reloading of config
|
||||||
|
|
|
@ -1103,9 +1103,23 @@ See [Installation for Docker](install.md#docker) for an example of how this coul
|
||||||
If configured, ntfy can expose a `/metrics` endpoint for [Prometheus](https://prometheus.io/), which can then be used to
|
If configured, ntfy can expose a `/metrics` endpoint for [Prometheus](https://prometheus.io/), which can then be used to
|
||||||
create dashboards and alerts (e.g. via [Grafana](https://grafana.com/)).
|
create dashboards and alerts (e.g. via [Grafana](https://grafana.com/)).
|
||||||
|
|
||||||
To configure the metrics endpoint, set the `listen-metrics-http` option to a listen address
|
To configure the metrics endpoint, either set `enable-metrics` and/or set the `listen-metrics-http` option to a dedicated
|
||||||
|
listen address. Metrics may be considered sensitive information, so before you enable them, be sure you know what you are
|
||||||
|
doing, and/or secure access to the endpoint in your reverse proxy.
|
||||||
|
|
||||||
XXXXXXXXXXXXXXXXXXX
|
- `enable-metrics` enables the /metrics endpoint for the default ntfy server (i.e. HTTP, HTTPS and/or Unix socket)
|
||||||
|
- `metrics-listen-http` exposes the metrics endpoint via a dedicated [IP]:port. If set, this option implicitly
|
||||||
|
enables metrics as well, e.g. "10.0.1.1:9090" or ":9090"
|
||||||
|
|
||||||
|
=== Using default port
|
||||||
|
```yaml
|
||||||
|
enable-metrics: true
|
||||||
|
```
|
||||||
|
|
||||||
|
=== Using dedicated IP/port
|
||||||
|
```yaml
|
||||||
|
metrics-listen-http: "10.0.1.1:9090"
|
||||||
|
```
|
||||||
|
|
||||||
## Logging & debugging
|
## Logging & debugging
|
||||||
By default, ntfy logs to the console (stderr), with an `info` log level, and in a human-readable text format.
|
By default, ntfy logs to the console (stderr), with an `info` log level, and in a human-readable text format.
|
||||||
|
|
|
@ -61,7 +61,7 @@ var (
|
||||||
|
|
||||||
// DefaultDisallowedTopics defines the topics that are forbidden, because they are used elsewhere. This array can be
|
// DefaultDisallowedTopics defines the topics that are forbidden, because they are used elsewhere. This array can be
|
||||||
// extended using the server.yml config. If updated, also update in Android and web app.
|
// extended using the server.yml config. If updated, also update in Android and web app.
|
||||||
DefaultDisallowedTopics = []string{"docs", "static", "file", "app", "account", "settings", "signup", "login", "v1"}
|
DefaultDisallowedTopics = []string{"docs", "static", "file", "app", "metrics", "account", "settings", "signup", "login", "v1"}
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config is the main config struct for the application. Use New to instantiate a default config struct.
|
// Config is the main config struct for the application. Use New to instantiate a default config struct.
|
||||||
|
@ -72,7 +72,6 @@ type Config struct {
|
||||||
ListenHTTPS string
|
ListenHTTPS string
|
||||||
ListenUnix string
|
ListenUnix string
|
||||||
ListenUnixMode fs.FileMode
|
ListenUnixMode fs.FileMode
|
||||||
ListenMetricsHTTP string
|
|
||||||
KeyFile string
|
KeyFile string
|
||||||
CertFile string
|
CertFile string
|
||||||
FirebaseKeyFile string
|
FirebaseKeyFile string
|
||||||
|
@ -106,6 +105,8 @@ type Config struct {
|
||||||
SMTPServerListen string
|
SMTPServerListen string
|
||||||
SMTPServerDomain string
|
SMTPServerDomain string
|
||||||
SMTPServerAddrPrefix string
|
SMTPServerAddrPrefix string
|
||||||
|
MetricsEnable bool
|
||||||
|
MetricsListenHTTP string
|
||||||
MessageLimit int
|
MessageLimit int
|
||||||
MinDelay time.Duration
|
MinDelay time.Duration
|
||||||
MaxDelay time.Duration
|
MaxDelay time.Duration
|
||||||
|
@ -135,6 +136,7 @@ type Config struct {
|
||||||
EnableSignup bool // Enable creation of accounts via API and UI
|
EnableSignup bool // Enable creation of accounts via API and UI
|
||||||
EnableLogin bool
|
EnableLogin bool
|
||||||
EnableReservations bool // Allow users with role "user" to own/reserve topics
|
EnableReservations bool // Allow users with role "user" to own/reserve topics
|
||||||
|
EnableMetrics bool
|
||||||
AccessControlAllowOrigin string // CORS header field to restrict access from web clients
|
AccessControlAllowOrigin string // CORS header field to restrict access from web clients
|
||||||
Version string // injected by App
|
Version string // injected by App
|
||||||
}
|
}
|
||||||
|
|
|
@ -67,7 +67,7 @@ func (c *fileCache) Write(id string, in io.Reader, limiters ...util.Limiter) (in
|
||||||
}
|
}
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
c.totalSizeCurrent += size
|
c.totalSizeCurrent += size
|
||||||
metrics.attachmentsTotalSize.Set(float64(c.totalSizeCurrent))
|
mset(metricAttachmentsTotalSize, c.totalSizeCurrent)
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
return size, nil
|
return size, nil
|
||||||
}
|
}
|
||||||
|
@ -90,7 +90,7 @@ func (c *fileCache) Remove(ids ...string) error {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
c.totalSizeCurrent = size
|
c.totalSizeCurrent = size
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
metrics.attachmentsTotalSize.Set(float64(size))
|
mset(metricAttachmentsTotalSize, size)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -52,6 +52,7 @@ type Server struct {
|
||||||
fileCache *fileCache // File system based cache that stores attachments
|
fileCache *fileCache // File system based cache that stores attachments
|
||||||
stripe stripeAPI // Stripe API, can be replaced with a mock
|
stripe stripeAPI // Stripe API, can be replaced with a mock
|
||||||
priceCache *util.LookupCache[map[string]int64] // Stripe price ID -> price as cents (USD implied!)
|
priceCache *util.LookupCache[map[string]int64] // Stripe price ID -> price as cents (USD implied!)
|
||||||
|
metricsHandler http.Handler // Handles /metrics if enable-metrics set, and listen-metrics-http not set
|
||||||
closeChan chan bool
|
closeChan chan bool
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
}
|
}
|
||||||
|
@ -74,6 +75,7 @@ var (
|
||||||
webConfigPath = "/config.js"
|
webConfigPath = "/config.js"
|
||||||
accountPath = "/account"
|
accountPath = "/account"
|
||||||
matrixPushPath = "/_matrix/push/v1/notify"
|
matrixPushPath = "/_matrix/push/v1/notify"
|
||||||
|
metricsPath = "/metrics"
|
||||||
apiHealthPath = "/v1/health"
|
apiHealthPath = "/v1/health"
|
||||||
apiTiers = "/v1/tiers"
|
apiTiers = "/v1/tiers"
|
||||||
apiAccountPath = "/v1/account"
|
apiAccountPath = "/v1/account"
|
||||||
|
@ -212,6 +214,9 @@ func (s *Server) Run() error {
|
||||||
if s.config.SMTPServerListen != "" {
|
if s.config.SMTPServerListen != "" {
|
||||||
listenStr += fmt.Sprintf(" %s[smtp]", s.config.SMTPServerListen)
|
listenStr += fmt.Sprintf(" %s[smtp]", s.config.SMTPServerListen)
|
||||||
}
|
}
|
||||||
|
if s.config.MetricsListenHTTP != "" {
|
||||||
|
listenStr += fmt.Sprintf(" %s[http/metrics]", s.config.MetricsListenHTTP)
|
||||||
|
}
|
||||||
log.Tag(tagStartup).Info("Listening on%s, ntfy %s, log level is %s", listenStr, s.config.Version, log.CurrentLevel().String())
|
log.Tag(tagStartup).Info("Listening on%s, ntfy %s, log level is %s", listenStr, s.config.Version, log.CurrentLevel().String())
|
||||||
if log.IsFile() {
|
if log.IsFile() {
|
||||||
fmt.Fprintf(os.Stderr, "Listening on%s, ntfy %s\n", listenStr, s.config.Version)
|
fmt.Fprintf(os.Stderr, "Listening on%s, ntfy %s\n", listenStr, s.config.Version)
|
||||||
|
@ -258,11 +263,15 @@ func (s *Server) Run() error {
|
||||||
errChan <- httpServer.Serve(s.unixListener)
|
errChan <- httpServer.Serve(s.unixListener)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
if s.config.ListenMetricsHTTP != "" {
|
if s.config.MetricsListenHTTP != "" {
|
||||||
s.httpMetricsServer = &http.Server{Addr: s.config.ListenMetricsHTTP, Handler: promhttp.Handler()}
|
initMetrics()
|
||||||
|
s.httpMetricsServer = &http.Server{Addr: s.config.MetricsListenHTTP, Handler: promhttp.Handler()}
|
||||||
go func() {
|
go func() {
|
||||||
errChan <- s.httpMetricsServer.ListenAndServe()
|
errChan <- s.httpMetricsServer.ListenAndServe()
|
||||||
}()
|
}()
|
||||||
|
} else if s.config.EnableMetrics {
|
||||||
|
initMetrics()
|
||||||
|
s.metricsHandler = promhttp.Handler()
|
||||||
}
|
}
|
||||||
if s.config.SMTPServerListen != "" {
|
if s.config.SMTPServerListen != "" {
|
||||||
go func() {
|
go func() {
|
||||||
|
@ -324,7 +333,9 @@ func (s *Server) handle(w http.ResponseWriter, r *http.Request) {
|
||||||
s.handleError(w, r, v, err)
|
s.handleError(w, r, v, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
metrics.httpRequests.WithLabelValues("200", "20000", r.Method).Inc()
|
if metricHTTPRequests != nil {
|
||||||
|
metricHTTPRequests.WithLabelValues("200", "20000", r.Method).Inc()
|
||||||
|
}
|
||||||
}).
|
}).
|
||||||
Debug("HTTP request finished")
|
Debug("HTTP request finished")
|
||||||
}
|
}
|
||||||
|
@ -334,7 +345,9 @@ func (s *Server) handleError(w http.ResponseWriter, r *http.Request, v *visitor,
|
||||||
if !ok {
|
if !ok {
|
||||||
httpErr = errHTTPInternalError
|
httpErr = errHTTPInternalError
|
||||||
}
|
}
|
||||||
metrics.httpRequests.WithLabelValues(fmt.Sprintf("%d", httpErr.HTTPCode), fmt.Sprintf("%d", httpErr.Code), r.Method).Inc()
|
if metricHTTPRequests != nil {
|
||||||
|
metricHTTPRequests.WithLabelValues(fmt.Sprintf("%d", httpErr.HTTPCode), fmt.Sprintf("%d", httpErr.Code), r.Method).Inc()
|
||||||
|
}
|
||||||
isRateLimiting := util.Contains(rateLimitingErrorCodes, httpErr.HTTPCode)
|
isRateLimiting := util.Contains(rateLimitingErrorCodes, httpErr.HTTPCode)
|
||||||
isNormalError := strings.Contains(err.Error(), "i/o timeout") || util.Contains(normalErrorCodes, httpErr.HTTPCode)
|
isNormalError := strings.Contains(err.Error(), "i/o timeout") || util.Contains(normalErrorCodes, httpErr.HTTPCode)
|
||||||
ev := logvr(v, r).Err(err)
|
ev := logvr(v, r).Err(err)
|
||||||
|
@ -415,6 +428,8 @@ func (s *Server) handleInternal(w http.ResponseWriter, r *http.Request, v *visit
|
||||||
return s.ensurePaymentsEnabled(s.handleBillingTiersGet)(w, r, v)
|
return s.ensurePaymentsEnabled(s.handleBillingTiersGet)(w, r, v)
|
||||||
} else if r.Method == http.MethodGet && r.URL.Path == matrixPushPath {
|
} else if r.Method == http.MethodGet && r.URL.Path == matrixPushPath {
|
||||||
return s.handleMatrixDiscovery(w)
|
return s.handleMatrixDiscovery(w)
|
||||||
|
} else if r.Method == http.MethodGet && r.URL.Path == metricsPath && s.metricsHandler != nil {
|
||||||
|
return s.handleMetrics(w, r, v)
|
||||||
} else if r.Method == http.MethodGet && staticRegex.MatchString(r.URL.Path) {
|
} else if r.Method == http.MethodGet && staticRegex.MatchString(r.URL.Path) {
|
||||||
return s.ensureWebEnabled(s.handleStatic)(w, r, v)
|
return s.ensureWebEnabled(s.handleStatic)(w, r, v)
|
||||||
} else if r.Method == http.MethodGet && docsRegex.MatchString(r.URL.Path) {
|
} else if r.Method == http.MethodGet && docsRegex.MatchString(r.URL.Path) {
|
||||||
|
@ -507,6 +522,13 @@ func (s *Server) handleWebConfig(w http.ResponseWriter, _ *http.Request, _ *visi
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handleMetrics returns Prometheus metrics. This endpoint is only called if enable-metrics is set,
|
||||||
|
// and listen-metrics-http is not set.
|
||||||
|
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request, _ *visitor) error {
|
||||||
|
s.metricsHandler.ServeHTTP(w, r)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Server) handleStatic(w http.ResponseWriter, r *http.Request, _ *visitor) error {
|
func (s *Server) handleStatic(w http.ResponseWriter, r *http.Request, _ *visitor) error {
|
||||||
r.URL.Path = webSiteDir + r.URL.Path
|
r.URL.Path = webSiteDir + r.URL.Path
|
||||||
util.Gzip(http.FileServer(http.FS(webFsCached))).ServeHTTP(w, r)
|
util.Gzip(http.FileServer(http.FS(webFsCached))).ServeHTTP(w, r)
|
||||||
|
@ -683,7 +705,7 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*message, e
|
||||||
s.messages++
|
s.messages++
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
if unifiedpush {
|
if unifiedpush {
|
||||||
metrics.unifiedPushPublishedSuccess.Inc()
|
minc(metricUnifiedPushPublishedSuccess)
|
||||||
}
|
}
|
||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
|
@ -691,18 +713,18 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*message, e
|
||||||
func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visitor) error {
|
func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visitor) error {
|
||||||
m, err := s.handlePublishInternal(r, v)
|
m, err := s.handlePublishInternal(r, v)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
metrics.messagesPublishedFailure.Inc()
|
minc(metricMessagesPublishedFailure)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
metrics.messagesPublishedSuccess.Inc()
|
minc(metricMessagesPublishedSuccess)
|
||||||
return s.writeJSON(w, m)
|
return s.writeJSON(w, m)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v *visitor) error {
|
func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v *visitor) error {
|
||||||
_, err := s.handlePublishInternal(r, v)
|
_, err := s.handlePublishInternal(r, v)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
metrics.messagesPublishedFailure.Inc()
|
minc(metricMessagesPublishedFailure)
|
||||||
metrics.matrixPublishedFailure.Inc()
|
minc(metricMatrixPublishedFailure)
|
||||||
if e, ok := err.(*errHTTP); ok && e.HTTPCode == errHTTPInsufficientStorageUnifiedPush.HTTPCode {
|
if e, ok := err.(*errHTTP); ok && e.HTTPCode == errHTTPInsufficientStorageUnifiedPush.HTTPCode {
|
||||||
topic, err := fromContext[*topic](r, contextTopic)
|
topic, err := fromContext[*topic](r, contextTopic)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -718,15 +740,15 @@ func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v *
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
metrics.messagesPublishedSuccess.Inc()
|
minc(metricMessagesPublishedSuccess)
|
||||||
metrics.matrixPublishedSuccess.Inc()
|
minc(metricMatrixPublishedSuccess)
|
||||||
return writeMatrixSuccess(w)
|
return writeMatrixSuccess(w)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) sendToFirebase(v *visitor, m *message) {
|
func (s *Server) sendToFirebase(v *visitor, m *message) {
|
||||||
logvm(v, m).Tag(tagFirebase).Debug("Publishing to Firebase")
|
logvm(v, m).Tag(tagFirebase).Debug("Publishing to Firebase")
|
||||||
if err := s.firebaseClient.Send(v, m); err != nil {
|
if err := s.firebaseClient.Send(v, m); err != nil {
|
||||||
metrics.firebasePublishedFailure.Inc()
|
minc(metricFirebasePublishedFailure)
|
||||||
if err == errFirebaseTemporarilyBanned {
|
if err == errFirebaseTemporarilyBanned {
|
||||||
logvm(v, m).Tag(tagFirebase).Err(err).Debug("Unable to publish to Firebase: %v", err.Error())
|
logvm(v, m).Tag(tagFirebase).Err(err).Debug("Unable to publish to Firebase: %v", err.Error())
|
||||||
} else {
|
} else {
|
||||||
|
@ -734,17 +756,17 @@ func (s *Server) sendToFirebase(v *visitor, m *message) {
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
metrics.firebasePublishedSuccess.Inc()
|
minc(metricFirebasePublishedSuccess)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) sendEmail(v *visitor, m *message, email string) {
|
func (s *Server) sendEmail(v *visitor, m *message, email string) {
|
||||||
logvm(v, m).Tag(tagEmail).Field("email", email).Debug("Sending email to %s", email)
|
logvm(v, m).Tag(tagEmail).Field("email", email).Debug("Sending email to %s", email)
|
||||||
if err := s.smtpSender.Send(v, m, email); err != nil {
|
if err := s.smtpSender.Send(v, m, email); err != nil {
|
||||||
logvm(v, m).Tag(tagEmail).Field("email", email).Err(err).Warn("Unable to send email to %s: %v", email, err.Error())
|
logvm(v, m).Tag(tagEmail).Field("email", email).Err(err).Warn("Unable to send email to %s: %v", email, err.Error())
|
||||||
metrics.emailsPublishedFailure.Inc()
|
minc(metricEmailsPublishedFailure)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
metrics.emailsPublishedSuccess.Inc()
|
minc(metricEmailsPublishedSuccess)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) forwardPollRequest(v *visitor, m *message) {
|
func (s *Server) forwardPollRequest(v *visitor, m *message) {
|
||||||
|
|
|
@ -263,6 +263,19 @@
|
||||||
# stripe-webhook-key:
|
# stripe-webhook-key:
|
||||||
# billing-contact:
|
# billing-contact:
|
||||||
|
|
||||||
|
# Metrics
|
||||||
|
#
|
||||||
|
# ntfy can expose Prometheus-style metrics via a /metrics endpoint, or on a dedicated listen IP/port.
|
||||||
|
# Metrics may be considered sensitive information, so before you enable them, be sure you know what you are
|
||||||
|
# doing, and/or secure access to the endpoint in your reverse proxy.
|
||||||
|
#
|
||||||
|
# - enable-metrics enables the /metrics endpoint for the default ntfy server (i.e. HTTP, HTTPS and/or Unix socket)
|
||||||
|
# - metrics-listen-http exposes the metrics endpoint via a dedicated [IP]:port. If set, this option implicitly
|
||||||
|
# enables metrics as well, e.g. "10.0.1.1:9090" or ":9090"
|
||||||
|
#
|
||||||
|
# enable-metrics: false
|
||||||
|
# metrics-listen-http:
|
||||||
|
|
||||||
# Logging options
|
# Logging options
|
||||||
#
|
#
|
||||||
# By default, ntfy logs to the console (stderr), with an "info" log level, and in a human-readable text format.
|
# By default, ntfy logs to the console (stderr), with an "info" log level, and in a human-readable text format.
|
||||||
|
|
|
@ -83,12 +83,10 @@ func (s *Server) execManager() {
|
||||||
"emails_sent_failure": sentMailFailure,
|
"emails_sent_failure": sentMailFailure,
|
||||||
}).
|
}).
|
||||||
Info("Server stats")
|
Info("Server stats")
|
||||||
if s.httpMetricsServer != nil {
|
mset(metricMessagesCached, messagesCached)
|
||||||
metrics.messagesCached.Set(float64(messagesCached))
|
mset(metricVisitors, visitorsCount)
|
||||||
metrics.visitors.Set(float64(visitorsCount))
|
mset(metricSubscribers, subscribers)
|
||||||
metrics.subscribers.Set(float64(subscribers))
|
mset(metricTopics, topicsCount)
|
||||||
metrics.topics.Set(float64(topicsCount))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) pruneVisitors() {
|
func (s *Server) pruneVisitors() {
|
||||||
|
|
|
@ -5,101 +5,108 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
metrics = newMetrics()
|
metricMessagesPublishedSuccess prometheus.Counter
|
||||||
|
metricMessagesPublishedFailure prometheus.Counter
|
||||||
|
metricMessagesCached prometheus.Gauge
|
||||||
|
metricFirebasePublishedSuccess prometheus.Counter
|
||||||
|
metricFirebasePublishedFailure prometheus.Counter
|
||||||
|
metricEmailsPublishedSuccess prometheus.Counter
|
||||||
|
metricEmailsPublishedFailure prometheus.Counter
|
||||||
|
metricEmailsReceivedSuccess prometheus.Counter
|
||||||
|
metricEmailsReceivedFailure prometheus.Counter
|
||||||
|
metricUnifiedPushPublishedSuccess prometheus.Counter
|
||||||
|
metricMatrixPublishedSuccess prometheus.Counter
|
||||||
|
metricMatrixPublishedFailure prometheus.Counter
|
||||||
|
metricAttachmentsTotalSize prometheus.Gauge
|
||||||
|
metricVisitors prometheus.Gauge
|
||||||
|
metricSubscribers prometheus.Gauge
|
||||||
|
metricTopics prometheus.Gauge
|
||||||
|
metricHTTPRequests *prometheus.CounterVec
|
||||||
)
|
)
|
||||||
|
|
||||||
type serverMetrics struct {
|
func initMetrics() {
|
||||||
messagesPublishedSuccess prometheus.Counter
|
metricMessagesPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
messagesPublishedFailure prometheus.Counter
|
Name: "ntfy_messages_published_success",
|
||||||
messagesCached prometheus.Gauge
|
})
|
||||||
firebasePublishedSuccess prometheus.Counter
|
metricMessagesPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
firebasePublishedFailure prometheus.Counter
|
Name: "ntfy_messages_published_failure",
|
||||||
emailsPublishedSuccess prometheus.Counter
|
})
|
||||||
emailsPublishedFailure prometheus.Counter
|
metricMessagesCached = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
emailsReceivedSuccess prometheus.Counter
|
Name: "ntfy_messages_cached_total",
|
||||||
emailsReceivedFailure prometheus.Counter
|
})
|
||||||
unifiedPushPublishedSuccess prometheus.Counter
|
metricFirebasePublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
matrixPublishedSuccess prometheus.Counter
|
Name: "ntfy_firebase_published_success",
|
||||||
matrixPublishedFailure prometheus.Counter
|
})
|
||||||
attachmentsTotalSize prometheus.Gauge
|
metricFirebasePublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
visitors prometheus.Gauge
|
Name: "ntfy_firebase_published_failure",
|
||||||
subscribers prometheus.Gauge
|
})
|
||||||
topics prometheus.Gauge
|
metricEmailsPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
httpRequests *prometheus.CounterVec
|
Name: "ntfy_emails_sent_success",
|
||||||
|
})
|
||||||
|
metricEmailsPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "ntfy_emails_sent_failure",
|
||||||
|
})
|
||||||
|
metricEmailsReceivedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "ntfy_emails_received_success",
|
||||||
|
})
|
||||||
|
metricEmailsReceivedFailure = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "ntfy_emails_received_failure",
|
||||||
|
})
|
||||||
|
metricUnifiedPushPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "ntfy_unifiedpush_published_success",
|
||||||
|
})
|
||||||
|
metricMatrixPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "ntfy_matrix_published_success",
|
||||||
|
})
|
||||||
|
metricMatrixPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
|
Name: "ntfy_matrix_published_failure",
|
||||||
|
})
|
||||||
|
metricAttachmentsTotalSize = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
|
Name: "ntfy_attachments_total_size",
|
||||||
|
})
|
||||||
|
metricVisitors = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
|
Name: "ntfy_visitors_total",
|
||||||
|
})
|
||||||
|
metricSubscribers = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
|
Name: "ntfy_subscribers_total",
|
||||||
|
})
|
||||||
|
metricTopics = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
|
Name: "ntfy_topics_total",
|
||||||
|
})
|
||||||
|
metricHTTPRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||||
|
Name: "ntfy_http_requests_total",
|
||||||
|
}, []string{"http_code", "ntfy_code", "http_method"})
|
||||||
|
prometheus.MustRegister(
|
||||||
|
metricMessagesPublishedSuccess,
|
||||||
|
metricMessagesPublishedFailure,
|
||||||
|
metricMessagesCached,
|
||||||
|
metricFirebasePublishedSuccess,
|
||||||
|
metricFirebasePublishedFailure,
|
||||||
|
metricEmailsPublishedSuccess,
|
||||||
|
metricEmailsPublishedFailure,
|
||||||
|
metricEmailsReceivedSuccess,
|
||||||
|
metricEmailsReceivedFailure,
|
||||||
|
metricUnifiedPushPublishedSuccess,
|
||||||
|
metricMatrixPublishedSuccess,
|
||||||
|
metricMatrixPublishedFailure,
|
||||||
|
metricAttachmentsTotalSize,
|
||||||
|
metricVisitors,
|
||||||
|
metricSubscribers,
|
||||||
|
metricTopics,
|
||||||
|
metricHTTPRequests,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newMetrics() *serverMetrics {
|
// minc increments a prometheus.Counter if it is non-nil
|
||||||
m := &serverMetrics{
|
func minc(counter prometheus.Counter) {
|
||||||
messagesPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{
|
if counter != nil {
|
||||||
Name: "ntfy_messages_published_success",
|
counter.Inc()
|
||||||
}),
|
}
|
||||||
messagesPublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{
|
}
|
||||||
Name: "ntfy_messages_published_failure",
|
|
||||||
}),
|
// mset sets a prometheus.Gauge if it is non-nil
|
||||||
messagesCached: prometheus.NewGauge(prometheus.GaugeOpts{
|
func mset[T int | int64 | float64](gauge prometheus.Gauge, value T) {
|
||||||
Name: "ntfy_messages_cached_total",
|
if gauge != nil {
|
||||||
}),
|
gauge.Set(float64(value))
|
||||||
firebasePublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{
|
|
||||||
Name: "ntfy_firebase_published_success",
|
|
||||||
}),
|
|
||||||
firebasePublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{
|
|
||||||
Name: "ntfy_firebase_published_failure",
|
|
||||||
}),
|
|
||||||
emailsPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{
|
|
||||||
Name: "ntfy_emails_sent_success",
|
|
||||||
}),
|
|
||||||
emailsPublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{
|
|
||||||
Name: "ntfy_emails_sent_failure",
|
|
||||||
}),
|
|
||||||
emailsReceivedSuccess: prometheus.NewCounter(prometheus.CounterOpts{
|
|
||||||
Name: "ntfy_emails_received_success",
|
|
||||||
}),
|
|
||||||
emailsReceivedFailure: prometheus.NewCounter(prometheus.CounterOpts{
|
|
||||||
Name: "ntfy_emails_received_failure",
|
|
||||||
}),
|
|
||||||
unifiedPushPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{
|
|
||||||
Name: "ntfy_unifiedpush_published_success",
|
|
||||||
}),
|
|
||||||
matrixPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{
|
|
||||||
Name: "ntfy_matrix_published_success",
|
|
||||||
}),
|
|
||||||
matrixPublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{
|
|
||||||
Name: "ntfy_matrix_published_failure",
|
|
||||||
}),
|
|
||||||
attachmentsTotalSize: prometheus.NewGauge(prometheus.GaugeOpts{
|
|
||||||
Name: "ntfy_attachments_total_size",
|
|
||||||
}),
|
|
||||||
visitors: prometheus.NewGauge(prometheus.GaugeOpts{
|
|
||||||
Name: "ntfy_visitors_total",
|
|
||||||
}),
|
|
||||||
subscribers: prometheus.NewGauge(prometheus.GaugeOpts{
|
|
||||||
Name: "ntfy_subscribers_total",
|
|
||||||
}),
|
|
||||||
topics: prometheus.NewGauge(prometheus.GaugeOpts{
|
|
||||||
Name: "ntfy_topics_total",
|
|
||||||
}),
|
|
||||||
httpRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
||||||
Name: "ntfy_http_requests_total",
|
|
||||||
}, []string{"http_code", "ntfy_code", "http_method"}),
|
|
||||||
}
|
}
|
||||||
prometheus.MustRegister(
|
|
||||||
m.messagesPublishedSuccess,
|
|
||||||
m.messagesPublishedFailure,
|
|
||||||
m.messagesCached,
|
|
||||||
m.firebasePublishedSuccess,
|
|
||||||
m.firebasePublishedFailure,
|
|
||||||
m.emailsPublishedSuccess,
|
|
||||||
m.emailsPublishedFailure,
|
|
||||||
m.emailsReceivedSuccess,
|
|
||||||
m.emailsReceivedFailure,
|
|
||||||
m.unifiedPushPublishedSuccess,
|
|
||||||
m.matrixPublishedSuccess,
|
|
||||||
m.matrixPublishedFailure,
|
|
||||||
m.attachmentsTotalSize,
|
|
||||||
m.visitors,
|
|
||||||
m.subscribers,
|
|
||||||
m.topics,
|
|
||||||
m.httpRequests,
|
|
||||||
)
|
|
||||||
return m
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -165,7 +165,7 @@ func (s *smtpSession) Data(r io.Reader) error {
|
||||||
s.backend.mu.Lock()
|
s.backend.mu.Lock()
|
||||||
s.backend.success++
|
s.backend.success++
|
||||||
s.backend.mu.Unlock()
|
s.backend.mu.Unlock()
|
||||||
metrics.emailsReceivedSuccess.Inc()
|
minc(metricEmailsReceivedSuccess)
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -218,7 +218,7 @@ func (s *smtpSession) withFailCount(fn func() error) error {
|
||||||
// We do not want to spam the log with WARN messages.
|
// We do not want to spam the log with WARN messages.
|
||||||
logem(s.conn).Err(err).Debug("Incoming mail error")
|
logem(s.conn).Err(err).Debug("Incoming mail error")
|
||||||
s.backend.failure++
|
s.backend.failure++
|
||||||
metrics.emailsReceivedFailure.Inc()
|
minc(metricEmailsReceivedFailure)
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue