Allow /metrics on default port; reduce memory if not enabled
This commit is contained in:
		
							parent
							
								
									bb3fe4f830
								
							
						
					
					
						commit
						358b344916
					
				
					 9 changed files with 184 additions and 125 deletions
				
			
		|  | @ -40,7 +40,6 @@ var flagsServe = append( | ||||||
| 	altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-http", Aliases: []string{"listen_http", "l"}, EnvVars: []string{"NTFY_LISTEN_HTTP"}, Value: server.DefaultListenHTTP, Usage: "ip:port used as HTTP listen address"}), | 	altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-http", Aliases: []string{"listen_http", "l"}, EnvVars: []string{"NTFY_LISTEN_HTTP"}, Value: server.DefaultListenHTTP, Usage: "ip:port used as HTTP listen address"}), | ||||||
| 	altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-https", Aliases: []string{"listen_https", "L"}, EnvVars: []string{"NTFY_LISTEN_HTTPS"}, Usage: "ip:port used as HTTPS listen address"}), | 	altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-https", Aliases: []string{"listen_https", "L"}, EnvVars: []string{"NTFY_LISTEN_HTTPS"}, Usage: "ip:port used as HTTPS listen address"}), | ||||||
| 	altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-unix", Aliases: []string{"listen_unix", "U"}, EnvVars: []string{"NTFY_LISTEN_UNIX"}, Usage: "listen on unix socket path"}), | 	altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-unix", Aliases: []string{"listen_unix", "U"}, EnvVars: []string{"NTFY_LISTEN_UNIX"}, Usage: "listen on unix socket path"}), | ||||||
| 	altsrc.NewStringFlag(&cli.StringFlag{Name: "listen-metrics-http", Aliases: []string{"listen_metrics_http"}, EnvVars: []string{"NTFY_LISTEN_METRICS_HTTP"}, Usage: "ip:port used to expose the metrics endpoint"}), |  | ||||||
| 	altsrc.NewIntFlag(&cli.IntFlag{Name: "listen-unix-mode", Aliases: []string{"listen_unix_mode"}, EnvVars: []string{"NTFY_LISTEN_UNIX_MODE"}, DefaultText: "system default", Usage: "file permissions of unix socket, e.g. 0700"}), | 	altsrc.NewIntFlag(&cli.IntFlag{Name: "listen-unix-mode", Aliases: []string{"listen_unix_mode"}, EnvVars: []string{"NTFY_LISTEN_UNIX_MODE"}, DefaultText: "system default", Usage: "file permissions of unix socket, e.g. 0700"}), | ||||||
| 	altsrc.NewStringFlag(&cli.StringFlag{Name: "key-file", Aliases: []string{"key_file", "K"}, EnvVars: []string{"NTFY_KEY_FILE"}, Usage: "private key file, if listen-https is set"}), | 	altsrc.NewStringFlag(&cli.StringFlag{Name: "key-file", Aliases: []string{"key_file", "K"}, EnvVars: []string{"NTFY_KEY_FILE"}, Usage: "private key file, if listen-https is set"}), | ||||||
| 	altsrc.NewStringFlag(&cli.StringFlag{Name: "cert-file", Aliases: []string{"cert_file", "E"}, EnvVars: []string{"NTFY_CERT_FILE"}, Usage: "certificate file, if listen-https is set"}), | 	altsrc.NewStringFlag(&cli.StringFlag{Name: "cert-file", Aliases: []string{"cert_file", "E"}, EnvVars: []string{"NTFY_CERT_FILE"}, Usage: "certificate file, if listen-https is set"}), | ||||||
|  | @ -87,6 +86,8 @@ var flagsServe = append( | ||||||
| 	altsrc.NewStringFlag(&cli.StringFlag{Name: "stripe-secret-key", Aliases: []string{"stripe_secret_key"}, EnvVars: []string{"NTFY_STRIPE_SECRET_KEY"}, Value: "", Usage: "key used for the Stripe API communication, this enables payments"}), | 	altsrc.NewStringFlag(&cli.StringFlag{Name: "stripe-secret-key", Aliases: []string{"stripe_secret_key"}, EnvVars: []string{"NTFY_STRIPE_SECRET_KEY"}, Value: "", Usage: "key used for the Stripe API communication, this enables payments"}), | ||||||
| 	altsrc.NewStringFlag(&cli.StringFlag{Name: "stripe-webhook-key", Aliases: []string{"stripe_webhook_key"}, EnvVars: []string{"NTFY_STRIPE_WEBHOOK_KEY"}, Value: "", Usage: "key required to validate the authenticity of incoming webhooks from Stripe"}), | 	altsrc.NewStringFlag(&cli.StringFlag{Name: "stripe-webhook-key", Aliases: []string{"stripe_webhook_key"}, EnvVars: []string{"NTFY_STRIPE_WEBHOOK_KEY"}, Value: "", Usage: "key required to validate the authenticity of incoming webhooks from Stripe"}), | ||||||
| 	altsrc.NewStringFlag(&cli.StringFlag{Name: "billing-contact", Aliases: []string{"billing_contact"}, EnvVars: []string{"NTFY_BILLING_CONTACT"}, Value: "", Usage: "e-mail or website to display in upgrade dialog (only if payments are enabled)"}), | 	altsrc.NewStringFlag(&cli.StringFlag{Name: "billing-contact", Aliases: []string{"billing_contact"}, EnvVars: []string{"NTFY_BILLING_CONTACT"}, Value: "", Usage: "e-mail or website to display in upgrade dialog (only if payments are enabled)"}), | ||||||
|  | 	altsrc.NewBoolFlag(&cli.BoolFlag{Name: "enable-metrics", Aliases: []string{"enable_metrics"}, EnvVars: []string{"NTFY_ENABLE_METRICS"}, Value: false, Usage: "if set, Prometheus metrics are exposed via the /metrics endpoint"}), | ||||||
|  | 	altsrc.NewStringFlag(&cli.StringFlag{Name: "metrics-listen-http", Aliases: []string{"metrics_listen_http"}, EnvVars: []string{"NTFY_METRICS_LISTEN_HTTP"}, Usage: "ip:port used to expose the metrics endpoint (implicitly enables metrics)"}), | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
| var cmdServe = &cli.Command{ | var cmdServe = &cli.Command{ | ||||||
|  | @ -119,7 +120,6 @@ func execServe(c *cli.Context) error { | ||||||
| 	listenHTTPS := c.String("listen-https") | 	listenHTTPS := c.String("listen-https") | ||||||
| 	listenUnix := c.String("listen-unix") | 	listenUnix := c.String("listen-unix") | ||||||
| 	listenUnixMode := c.Int("listen-unix-mode") | 	listenUnixMode := c.Int("listen-unix-mode") | ||||||
| 	listenMetricsHTTP := c.String("listen-metrics-http") |  | ||||||
| 	keyFile := c.String("key-file") | 	keyFile := c.String("key-file") | ||||||
| 	certFile := c.String("cert-file") | 	certFile := c.String("cert-file") | ||||||
| 	firebaseKeyFile := c.String("firebase-key-file") | 	firebaseKeyFile := c.String("firebase-key-file") | ||||||
|  | @ -165,6 +165,8 @@ func execServe(c *cli.Context) error { | ||||||
| 	stripeSecretKey := c.String("stripe-secret-key") | 	stripeSecretKey := c.String("stripe-secret-key") | ||||||
| 	stripeWebhookKey := c.String("stripe-webhook-key") | 	stripeWebhookKey := c.String("stripe-webhook-key") | ||||||
| 	billingContact := c.String("billing-contact") | 	billingContact := c.String("billing-contact") | ||||||
|  | 	metricsListenHTTP := c.String("metrics-listen-http") | ||||||
|  | 	enableMetrics := c.Bool("enable-metrics") || metricsListenHTTP != "" | ||||||
| 
 | 
 | ||||||
| 	// Check values | 	// Check values | ||||||
| 	if firebaseKeyFile != "" && !util.FileExists(firebaseKeyFile) { | 	if firebaseKeyFile != "" && !util.FileExists(firebaseKeyFile) { | ||||||
|  | @ -271,7 +273,6 @@ func execServe(c *cli.Context) error { | ||||||
| 	conf.ListenHTTPS = listenHTTPS | 	conf.ListenHTTPS = listenHTTPS | ||||||
| 	conf.ListenUnix = listenUnix | 	conf.ListenUnix = listenUnix | ||||||
| 	conf.ListenUnixMode = fs.FileMode(listenUnixMode) | 	conf.ListenUnixMode = fs.FileMode(listenUnixMode) | ||||||
| 	conf.ListenMetricsHTTP = listenMetricsHTTP |  | ||||||
| 	conf.KeyFile = keyFile | 	conf.KeyFile = keyFile | ||||||
| 	conf.CertFile = certFile | 	conf.CertFile = certFile | ||||||
| 	conf.FirebaseKeyFile = firebaseKeyFile | 	conf.FirebaseKeyFile = firebaseKeyFile | ||||||
|  | @ -318,6 +319,8 @@ func execServe(c *cli.Context) error { | ||||||
| 	conf.EnableSignup = enableSignup | 	conf.EnableSignup = enableSignup | ||||||
| 	conf.EnableLogin = enableLogin | 	conf.EnableLogin = enableLogin | ||||||
| 	conf.EnableReservations = enableReservations | 	conf.EnableReservations = enableReservations | ||||||
|  | 	conf.EnableMetrics = enableMetrics | ||||||
|  | 	conf.MetricsListenHTTP = metricsListenHTTP | ||||||
| 	conf.Version = c.App.Version | 	conf.Version = c.App.Version | ||||||
| 
 | 
 | ||||||
| 	// Set up hot-reloading of config | 	// Set up hot-reloading of config | ||||||
|  |  | ||||||
|  | @ -1103,9 +1103,23 @@ See [Installation for Docker](install.md#docker) for an example of how this coul | ||||||
| If configured, ntfy can expose a `/metrics` endpoint for [Prometheus](https://prometheus.io/), which can then be used to | If configured, ntfy can expose a `/metrics` endpoint for [Prometheus](https://prometheus.io/), which can then be used to | ||||||
| create dashboards and alerts (e.g. via [Grafana](https://grafana.com/)). | create dashboards and alerts (e.g. via [Grafana](https://grafana.com/)). | ||||||
| 
 | 
 | ||||||
| To configure the metrics endpoint, set the `listen-metrics-http` option to a listen address | To configure the metrics endpoint, either set `enable-metrics` and/or set the `listen-metrics-http` option to a dedicated | ||||||
|  | listen address. Metrics may be considered sensitive information, so before you enable them, be sure you know what you are | ||||||
|  | doing, and/or secure access to the endpoint in your reverse proxy. | ||||||
| 
 | 
 | ||||||
| XXXXXXXXXXXXXXXXXXX | - `enable-metrics` enables the /metrics endpoint for the default ntfy server (i.e. HTTP, HTTPS and/or Unix socket) | ||||||
|  | - `metrics-listen-http` exposes the metrics endpoint via a dedicated [IP]:port. If set, this option implicitly | ||||||
|  |   enables metrics as well, e.g. "10.0.1.1:9090" or ":9090" | ||||||
|  | 
 | ||||||
|  | === Using default port | ||||||
|  |     ```yaml | ||||||
|  |     enable-metrics: true | ||||||
|  |     ``` | ||||||
|  | 
 | ||||||
|  | === Using dedicated IP/port | ||||||
|  |     ```yaml | ||||||
|  |     metrics-listen-http: "10.0.1.1:9090" | ||||||
|  |     ``` | ||||||
| 
 | 
 | ||||||
| ## Logging & debugging | ## Logging & debugging | ||||||
| By default, ntfy logs to the console (stderr), with an `info` log level, and in a human-readable text format. | By default, ntfy logs to the console (stderr), with an `info` log level, and in a human-readable text format. | ||||||
|  |  | ||||||
|  | @ -61,7 +61,7 @@ var ( | ||||||
| 
 | 
 | ||||||
| 	// DefaultDisallowedTopics defines the topics that are forbidden, because they are used elsewhere. This array can be | 	// DefaultDisallowedTopics defines the topics that are forbidden, because they are used elsewhere. This array can be | ||||||
| 	// extended using the server.yml config. If updated, also update in Android and web app. | 	// extended using the server.yml config. If updated, also update in Android and web app. | ||||||
| 	DefaultDisallowedTopics = []string{"docs", "static", "file", "app", "account", "settings", "signup", "login", "v1"} | 	DefaultDisallowedTopics = []string{"docs", "static", "file", "app", "metrics", "account", "settings", "signup", "login", "v1"} | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
| // Config is the main config struct for the application. Use New to instantiate a default config struct. | // Config is the main config struct for the application. Use New to instantiate a default config struct. | ||||||
|  | @ -72,7 +72,6 @@ type Config struct { | ||||||
| 	ListenHTTPS                          string | 	ListenHTTPS                          string | ||||||
| 	ListenUnix                           string | 	ListenUnix                           string | ||||||
| 	ListenUnixMode                       fs.FileMode | 	ListenUnixMode                       fs.FileMode | ||||||
| 	ListenMetricsHTTP                    string |  | ||||||
| 	KeyFile                              string | 	KeyFile                              string | ||||||
| 	CertFile                             string | 	CertFile                             string | ||||||
| 	FirebaseKeyFile                      string | 	FirebaseKeyFile                      string | ||||||
|  | @ -106,6 +105,8 @@ type Config struct { | ||||||
| 	SMTPServerListen                     string | 	SMTPServerListen                     string | ||||||
| 	SMTPServerDomain                     string | 	SMTPServerDomain                     string | ||||||
| 	SMTPServerAddrPrefix                 string | 	SMTPServerAddrPrefix                 string | ||||||
|  | 	MetricsEnable                        bool | ||||||
|  | 	MetricsListenHTTP                    string | ||||||
| 	MessageLimit                         int | 	MessageLimit                         int | ||||||
| 	MinDelay                             time.Duration | 	MinDelay                             time.Duration | ||||||
| 	MaxDelay                             time.Duration | 	MaxDelay                             time.Duration | ||||||
|  | @ -135,6 +136,7 @@ type Config struct { | ||||||
| 	EnableSignup                         bool // Enable creation of accounts via API and UI | 	EnableSignup                         bool // Enable creation of accounts via API and UI | ||||||
| 	EnableLogin                          bool | 	EnableLogin                          bool | ||||||
| 	EnableReservations                   bool // Allow users with role "user" to own/reserve topics | 	EnableReservations                   bool // Allow users with role "user" to own/reserve topics | ||||||
|  | 	EnableMetrics                        bool | ||||||
| 	AccessControlAllowOrigin             string // CORS header field to restrict access from web clients | 	AccessControlAllowOrigin             string // CORS header field to restrict access from web clients | ||||||
| 	Version                              string // injected by App | 	Version                              string // injected by App | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -67,7 +67,7 @@ func (c *fileCache) Write(id string, in io.Reader, limiters ...util.Limiter) (in | ||||||
| 	} | 	} | ||||||
| 	c.mu.Lock() | 	c.mu.Lock() | ||||||
| 	c.totalSizeCurrent += size | 	c.totalSizeCurrent += size | ||||||
| 	metrics.attachmentsTotalSize.Set(float64(c.totalSizeCurrent)) | 	mset(metricAttachmentsTotalSize, c.totalSizeCurrent) | ||||||
| 	c.mu.Unlock() | 	c.mu.Unlock() | ||||||
| 	return size, nil | 	return size, nil | ||||||
| } | } | ||||||
|  | @ -90,7 +90,7 @@ func (c *fileCache) Remove(ids ...string) error { | ||||||
| 	c.mu.Lock() | 	c.mu.Lock() | ||||||
| 	c.totalSizeCurrent = size | 	c.totalSizeCurrent = size | ||||||
| 	c.mu.Unlock() | 	c.mu.Unlock() | ||||||
| 	metrics.attachmentsTotalSize.Set(float64(size)) | 	mset(metricAttachmentsTotalSize, size) | ||||||
| 	return nil | 	return nil | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -52,6 +52,7 @@ type Server struct { | ||||||
| 	fileCache         *fileCache                          // File system based cache that stores attachments | 	fileCache         *fileCache                          // File system based cache that stores attachments | ||||||
| 	stripe            stripeAPI                           // Stripe API, can be replaced with a mock | 	stripe            stripeAPI                           // Stripe API, can be replaced with a mock | ||||||
| 	priceCache        *util.LookupCache[map[string]int64] // Stripe price ID -> price as cents (USD implied!) | 	priceCache        *util.LookupCache[map[string]int64] // Stripe price ID -> price as cents (USD implied!) | ||||||
|  | 	metricsHandler    http.Handler                        // Handles /metrics if enable-metrics set, and listen-metrics-http not set | ||||||
| 	closeChan         chan bool | 	closeChan         chan bool | ||||||
| 	mu                sync.Mutex | 	mu                sync.Mutex | ||||||
| } | } | ||||||
|  | @ -74,6 +75,7 @@ var ( | ||||||
| 	webConfigPath                                        = "/config.js" | 	webConfigPath                                        = "/config.js" | ||||||
| 	accountPath                                          = "/account" | 	accountPath                                          = "/account" | ||||||
| 	matrixPushPath                                       = "/_matrix/push/v1/notify" | 	matrixPushPath                                       = "/_matrix/push/v1/notify" | ||||||
|  | 	metricsPath                                          = "/metrics" | ||||||
| 	apiHealthPath                                        = "/v1/health" | 	apiHealthPath                                        = "/v1/health" | ||||||
| 	apiTiers                                             = "/v1/tiers" | 	apiTiers                                             = "/v1/tiers" | ||||||
| 	apiAccountPath                                       = "/v1/account" | 	apiAccountPath                                       = "/v1/account" | ||||||
|  | @ -212,6 +214,9 @@ func (s *Server) Run() error { | ||||||
| 	if s.config.SMTPServerListen != "" { | 	if s.config.SMTPServerListen != "" { | ||||||
| 		listenStr += fmt.Sprintf(" %s[smtp]", s.config.SMTPServerListen) | 		listenStr += fmt.Sprintf(" %s[smtp]", s.config.SMTPServerListen) | ||||||
| 	} | 	} | ||||||
|  | 	if s.config.MetricsListenHTTP != "" { | ||||||
|  | 		listenStr += fmt.Sprintf(" %s[http/metrics]", s.config.MetricsListenHTTP) | ||||||
|  | 	} | ||||||
| 	log.Tag(tagStartup).Info("Listening on%s, ntfy %s, log level is %s", listenStr, s.config.Version, log.CurrentLevel().String()) | 	log.Tag(tagStartup).Info("Listening on%s, ntfy %s, log level is %s", listenStr, s.config.Version, log.CurrentLevel().String()) | ||||||
| 	if log.IsFile() { | 	if log.IsFile() { | ||||||
| 		fmt.Fprintf(os.Stderr, "Listening on%s, ntfy %s\n", listenStr, s.config.Version) | 		fmt.Fprintf(os.Stderr, "Listening on%s, ntfy %s\n", listenStr, s.config.Version) | ||||||
|  | @ -258,11 +263,15 @@ func (s *Server) Run() error { | ||||||
| 			errChan <- httpServer.Serve(s.unixListener) | 			errChan <- httpServer.Serve(s.unixListener) | ||||||
| 		}() | 		}() | ||||||
| 	} | 	} | ||||||
| 	if s.config.ListenMetricsHTTP != "" { | 	if s.config.MetricsListenHTTP != "" { | ||||||
| 		s.httpMetricsServer = &http.Server{Addr: s.config.ListenMetricsHTTP, Handler: promhttp.Handler()} | 		initMetrics() | ||||||
|  | 		s.httpMetricsServer = &http.Server{Addr: s.config.MetricsListenHTTP, Handler: promhttp.Handler()} | ||||||
| 		go func() { | 		go func() { | ||||||
| 			errChan <- s.httpMetricsServer.ListenAndServe() | 			errChan <- s.httpMetricsServer.ListenAndServe() | ||||||
| 		}() | 		}() | ||||||
|  | 	} else if s.config.EnableMetrics { | ||||||
|  | 		initMetrics() | ||||||
|  | 		s.metricsHandler = promhttp.Handler() | ||||||
| 	} | 	} | ||||||
| 	if s.config.SMTPServerListen != "" { | 	if s.config.SMTPServerListen != "" { | ||||||
| 		go func() { | 		go func() { | ||||||
|  | @ -324,7 +333,9 @@ func (s *Server) handle(w http.ResponseWriter, r *http.Request) { | ||||||
| 				s.handleError(w, r, v, err) | 				s.handleError(w, r, v, err) | ||||||
| 				return | 				return | ||||||
| 			} | 			} | ||||||
| 			metrics.httpRequests.WithLabelValues("200", "20000", r.Method).Inc() | 			if metricHTTPRequests != nil { | ||||||
|  | 				metricHTTPRequests.WithLabelValues("200", "20000", r.Method).Inc() | ||||||
|  | 			} | ||||||
| 		}). | 		}). | ||||||
| 		Debug("HTTP request finished") | 		Debug("HTTP request finished") | ||||||
| } | } | ||||||
|  | @ -334,7 +345,9 @@ func (s *Server) handleError(w http.ResponseWriter, r *http.Request, v *visitor, | ||||||
| 	if !ok { | 	if !ok { | ||||||
| 		httpErr = errHTTPInternalError | 		httpErr = errHTTPInternalError | ||||||
| 	} | 	} | ||||||
| 	metrics.httpRequests.WithLabelValues(fmt.Sprintf("%d", httpErr.HTTPCode), fmt.Sprintf("%d", httpErr.Code), r.Method).Inc() | 	if metricHTTPRequests != nil { | ||||||
|  | 		metricHTTPRequests.WithLabelValues(fmt.Sprintf("%d", httpErr.HTTPCode), fmt.Sprintf("%d", httpErr.Code), r.Method).Inc() | ||||||
|  | 	} | ||||||
| 	isRateLimiting := util.Contains(rateLimitingErrorCodes, httpErr.HTTPCode) | 	isRateLimiting := util.Contains(rateLimitingErrorCodes, httpErr.HTTPCode) | ||||||
| 	isNormalError := strings.Contains(err.Error(), "i/o timeout") || util.Contains(normalErrorCodes, httpErr.HTTPCode) | 	isNormalError := strings.Contains(err.Error(), "i/o timeout") || util.Contains(normalErrorCodes, httpErr.HTTPCode) | ||||||
| 	ev := logvr(v, r).Err(err) | 	ev := logvr(v, r).Err(err) | ||||||
|  | @ -415,6 +428,8 @@ func (s *Server) handleInternal(w http.ResponseWriter, r *http.Request, v *visit | ||||||
| 		return s.ensurePaymentsEnabled(s.handleBillingTiersGet)(w, r, v) | 		return s.ensurePaymentsEnabled(s.handleBillingTiersGet)(w, r, v) | ||||||
| 	} else if r.Method == http.MethodGet && r.URL.Path == matrixPushPath { | 	} else if r.Method == http.MethodGet && r.URL.Path == matrixPushPath { | ||||||
| 		return s.handleMatrixDiscovery(w) | 		return s.handleMatrixDiscovery(w) | ||||||
|  | 	} else if r.Method == http.MethodGet && r.URL.Path == metricsPath && s.metricsHandler != nil { | ||||||
|  | 		return s.handleMetrics(w, r, v) | ||||||
| 	} else if r.Method == http.MethodGet && staticRegex.MatchString(r.URL.Path) { | 	} else if r.Method == http.MethodGet && staticRegex.MatchString(r.URL.Path) { | ||||||
| 		return s.ensureWebEnabled(s.handleStatic)(w, r, v) | 		return s.ensureWebEnabled(s.handleStatic)(w, r, v) | ||||||
| 	} else if r.Method == http.MethodGet && docsRegex.MatchString(r.URL.Path) { | 	} else if r.Method == http.MethodGet && docsRegex.MatchString(r.URL.Path) { | ||||||
|  | @ -507,6 +522,13 @@ func (s *Server) handleWebConfig(w http.ResponseWriter, _ *http.Request, _ *visi | ||||||
| 	return err | 	return err | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  | // handleMetrics returns Prometheus metrics. This endpoint is only called if enable-metrics is set, | ||||||
|  | // and listen-metrics-http is not set. | ||||||
|  | func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request, _ *visitor) error { | ||||||
|  | 	s.metricsHandler.ServeHTTP(w, r) | ||||||
|  | 	return nil | ||||||
|  | } | ||||||
|  | 
 | ||||||
| func (s *Server) handleStatic(w http.ResponseWriter, r *http.Request, _ *visitor) error { | func (s *Server) handleStatic(w http.ResponseWriter, r *http.Request, _ *visitor) error { | ||||||
| 	r.URL.Path = webSiteDir + r.URL.Path | 	r.URL.Path = webSiteDir + r.URL.Path | ||||||
| 	util.Gzip(http.FileServer(http.FS(webFsCached))).ServeHTTP(w, r) | 	util.Gzip(http.FileServer(http.FS(webFsCached))).ServeHTTP(w, r) | ||||||
|  | @ -683,7 +705,7 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*message, e | ||||||
| 	s.messages++ | 	s.messages++ | ||||||
| 	s.mu.Unlock() | 	s.mu.Unlock() | ||||||
| 	if unifiedpush { | 	if unifiedpush { | ||||||
| 		metrics.unifiedPushPublishedSuccess.Inc() | 		minc(metricUnifiedPushPublishedSuccess) | ||||||
| 	} | 	} | ||||||
| 	return m, nil | 	return m, nil | ||||||
| } | } | ||||||
|  | @ -691,18 +713,18 @@ func (s *Server) handlePublishInternal(r *http.Request, v *visitor) (*message, e | ||||||
| func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visitor) error { | func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visitor) error { | ||||||
| 	m, err := s.handlePublishInternal(r, v) | 	m, err := s.handlePublishInternal(r, v) | ||||||
| 	if err != nil { | 	if err != nil { | ||||||
| 		metrics.messagesPublishedFailure.Inc() | 		minc(metricMessagesPublishedFailure) | ||||||
| 		return err | 		return err | ||||||
| 	} | 	} | ||||||
| 	metrics.messagesPublishedSuccess.Inc() | 	minc(metricMessagesPublishedSuccess) | ||||||
| 	return s.writeJSON(w, m) | 	return s.writeJSON(w, m) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v *visitor) error { | func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v *visitor) error { | ||||||
| 	_, err := s.handlePublishInternal(r, v) | 	_, err := s.handlePublishInternal(r, v) | ||||||
| 	if err != nil { | 	if err != nil { | ||||||
| 		metrics.messagesPublishedFailure.Inc() | 		minc(metricMessagesPublishedFailure) | ||||||
| 		metrics.matrixPublishedFailure.Inc() | 		minc(metricMatrixPublishedFailure) | ||||||
| 		if e, ok := err.(*errHTTP); ok && e.HTTPCode == errHTTPInsufficientStorageUnifiedPush.HTTPCode { | 		if e, ok := err.(*errHTTP); ok && e.HTTPCode == errHTTPInsufficientStorageUnifiedPush.HTTPCode { | ||||||
| 			topic, err := fromContext[*topic](r, contextTopic) | 			topic, err := fromContext[*topic](r, contextTopic) | ||||||
| 			if err != nil { | 			if err != nil { | ||||||
|  | @ -718,15 +740,15 @@ func (s *Server) handlePublishMatrix(w http.ResponseWriter, r *http.Request, v * | ||||||
| 		} | 		} | ||||||
| 		return err | 		return err | ||||||
| 	} | 	} | ||||||
| 	metrics.messagesPublishedSuccess.Inc() | 	minc(metricMessagesPublishedSuccess) | ||||||
| 	metrics.matrixPublishedSuccess.Inc() | 	minc(metricMatrixPublishedSuccess) | ||||||
| 	return writeMatrixSuccess(w) | 	return writeMatrixSuccess(w) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| func (s *Server) sendToFirebase(v *visitor, m *message) { | func (s *Server) sendToFirebase(v *visitor, m *message) { | ||||||
| 	logvm(v, m).Tag(tagFirebase).Debug("Publishing to Firebase") | 	logvm(v, m).Tag(tagFirebase).Debug("Publishing to Firebase") | ||||||
| 	if err := s.firebaseClient.Send(v, m); err != nil { | 	if err := s.firebaseClient.Send(v, m); err != nil { | ||||||
| 		metrics.firebasePublishedFailure.Inc() | 		minc(metricFirebasePublishedFailure) | ||||||
| 		if err == errFirebaseTemporarilyBanned { | 		if err == errFirebaseTemporarilyBanned { | ||||||
| 			logvm(v, m).Tag(tagFirebase).Err(err).Debug("Unable to publish to Firebase: %v", err.Error()) | 			logvm(v, m).Tag(tagFirebase).Err(err).Debug("Unable to publish to Firebase: %v", err.Error()) | ||||||
| 		} else { | 		} else { | ||||||
|  | @ -734,17 +756,17 @@ func (s *Server) sendToFirebase(v *visitor, m *message) { | ||||||
| 		} | 		} | ||||||
| 		return | 		return | ||||||
| 	} | 	} | ||||||
| 	metrics.firebasePublishedSuccess.Inc() | 	minc(metricFirebasePublishedSuccess) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| func (s *Server) sendEmail(v *visitor, m *message, email string) { | func (s *Server) sendEmail(v *visitor, m *message, email string) { | ||||||
| 	logvm(v, m).Tag(tagEmail).Field("email", email).Debug("Sending email to %s", email) | 	logvm(v, m).Tag(tagEmail).Field("email", email).Debug("Sending email to %s", email) | ||||||
| 	if err := s.smtpSender.Send(v, m, email); err != nil { | 	if err := s.smtpSender.Send(v, m, email); err != nil { | ||||||
| 		logvm(v, m).Tag(tagEmail).Field("email", email).Err(err).Warn("Unable to send email to %s: %v", email, err.Error()) | 		logvm(v, m).Tag(tagEmail).Field("email", email).Err(err).Warn("Unable to send email to %s: %v", email, err.Error()) | ||||||
| 		metrics.emailsPublishedFailure.Inc() | 		minc(metricEmailsPublishedFailure) | ||||||
| 		return | 		return | ||||||
| 	} | 	} | ||||||
| 	metrics.emailsPublishedSuccess.Inc() | 	minc(metricEmailsPublishedSuccess) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| func (s *Server) forwardPollRequest(v *visitor, m *message) { | func (s *Server) forwardPollRequest(v *visitor, m *message) { | ||||||
|  |  | ||||||
|  | @ -263,6 +263,19 @@ | ||||||
| # stripe-webhook-key: | # stripe-webhook-key: | ||||||
| # billing-contact: | # billing-contact: | ||||||
| 
 | 
 | ||||||
|  | # Metrics | ||||||
|  | # | ||||||
|  | # ntfy can expose Prometheus-style metrics via a /metrics endpoint, or on a dedicated listen IP/port. | ||||||
|  | # Metrics may be considered sensitive information, so before you enable them, be sure you know what you are | ||||||
|  | # doing, and/or secure access to the endpoint in your reverse proxy. | ||||||
|  | # | ||||||
|  | # - enable-metrics enables the /metrics endpoint for the default ntfy server (i.e. HTTP, HTTPS and/or Unix socket) | ||||||
|  | # - metrics-listen-http exposes the metrics endpoint via a dedicated [IP]:port. If set, this option implicitly | ||||||
|  | #   enables metrics as well, e.g. "10.0.1.1:9090" or ":9090" | ||||||
|  | # | ||||||
|  | # enable-metrics: false | ||||||
|  | # metrics-listen-http: | ||||||
|  | 
 | ||||||
| # Logging options | # Logging options | ||||||
| # | # | ||||||
| # By default, ntfy logs to the console (stderr), with an "info" log level, and in a human-readable text format. | # By default, ntfy logs to the console (stderr), with an "info" log level, and in a human-readable text format. | ||||||
|  |  | ||||||
|  | @ -83,12 +83,10 @@ func (s *Server) execManager() { | ||||||
| 			"emails_sent_failure":     sentMailFailure, | 			"emails_sent_failure":     sentMailFailure, | ||||||
| 		}). | 		}). | ||||||
| 		Info("Server stats") | 		Info("Server stats") | ||||||
| 	if s.httpMetricsServer != nil { | 	mset(metricMessagesCached, messagesCached) | ||||||
| 		metrics.messagesCached.Set(float64(messagesCached)) | 	mset(metricVisitors, visitorsCount) | ||||||
| 		metrics.visitors.Set(float64(visitorsCount)) | 	mset(metricSubscribers, subscribers) | ||||||
| 		metrics.subscribers.Set(float64(subscribers)) | 	mset(metricTopics, topicsCount) | ||||||
| 		metrics.topics.Set(float64(topicsCount)) |  | ||||||
| 	} |  | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| func (s *Server) pruneVisitors() { | func (s *Server) pruneVisitors() { | ||||||
|  |  | ||||||
|  | @ -5,101 +5,108 @@ import ( | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
| var ( | var ( | ||||||
| 	metrics = newMetrics() | 	metricMessagesPublishedSuccess    prometheus.Counter | ||||||
|  | 	metricMessagesPublishedFailure    prometheus.Counter | ||||||
|  | 	metricMessagesCached              prometheus.Gauge | ||||||
|  | 	metricFirebasePublishedSuccess    prometheus.Counter | ||||||
|  | 	metricFirebasePublishedFailure    prometheus.Counter | ||||||
|  | 	metricEmailsPublishedSuccess      prometheus.Counter | ||||||
|  | 	metricEmailsPublishedFailure      prometheus.Counter | ||||||
|  | 	metricEmailsReceivedSuccess       prometheus.Counter | ||||||
|  | 	metricEmailsReceivedFailure       prometheus.Counter | ||||||
|  | 	metricUnifiedPushPublishedSuccess prometheus.Counter | ||||||
|  | 	metricMatrixPublishedSuccess      prometheus.Counter | ||||||
|  | 	metricMatrixPublishedFailure      prometheus.Counter | ||||||
|  | 	metricAttachmentsTotalSize        prometheus.Gauge | ||||||
|  | 	metricVisitors                    prometheus.Gauge | ||||||
|  | 	metricSubscribers                 prometheus.Gauge | ||||||
|  | 	metricTopics                      prometheus.Gauge | ||||||
|  | 	metricHTTPRequests                *prometheus.CounterVec | ||||||
| ) | ) | ||||||
| 
 | 
 | ||||||
| type serverMetrics struct { | func initMetrics() { | ||||||
| 	messagesPublishedSuccess    prometheus.Counter | 	metricMessagesPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{ | ||||||
| 	messagesPublishedFailure    prometheus.Counter |  | ||||||
| 	messagesCached              prometheus.Gauge |  | ||||||
| 	firebasePublishedSuccess    prometheus.Counter |  | ||||||
| 	firebasePublishedFailure    prometheus.Counter |  | ||||||
| 	emailsPublishedSuccess      prometheus.Counter |  | ||||||
| 	emailsPublishedFailure      prometheus.Counter |  | ||||||
| 	emailsReceivedSuccess       prometheus.Counter |  | ||||||
| 	emailsReceivedFailure       prometheus.Counter |  | ||||||
| 	unifiedPushPublishedSuccess prometheus.Counter |  | ||||||
| 	matrixPublishedSuccess      prometheus.Counter |  | ||||||
| 	matrixPublishedFailure      prometheus.Counter |  | ||||||
| 	attachmentsTotalSize        prometheus.Gauge |  | ||||||
| 	visitors                    prometheus.Gauge |  | ||||||
| 	subscribers                 prometheus.Gauge |  | ||||||
| 	topics                      prometheus.Gauge |  | ||||||
| 	httpRequests                *prometheus.CounterVec |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| func newMetrics() *serverMetrics { |  | ||||||
| 	m := &serverMetrics{ |  | ||||||
| 		messagesPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{ |  | ||||||
| 		Name: "ntfy_messages_published_success", | 		Name: "ntfy_messages_published_success", | ||||||
| 		}), | 	}) | ||||||
| 		messagesPublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{ | 	metricMessagesPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{ | ||||||
| 		Name: "ntfy_messages_published_failure", | 		Name: "ntfy_messages_published_failure", | ||||||
| 		}), | 	}) | ||||||
| 		messagesCached: prometheus.NewGauge(prometheus.GaugeOpts{ | 	metricMessagesCached = prometheus.NewGauge(prometheus.GaugeOpts{ | ||||||
| 		Name: "ntfy_messages_cached_total", | 		Name: "ntfy_messages_cached_total", | ||||||
| 		}), | 	}) | ||||||
| 		firebasePublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{ | 	metricFirebasePublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{ | ||||||
| 		Name: "ntfy_firebase_published_success", | 		Name: "ntfy_firebase_published_success", | ||||||
| 		}), | 	}) | ||||||
| 		firebasePublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{ | 	metricFirebasePublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{ | ||||||
| 		Name: "ntfy_firebase_published_failure", | 		Name: "ntfy_firebase_published_failure", | ||||||
| 		}), | 	}) | ||||||
| 		emailsPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{ | 	metricEmailsPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{ | ||||||
| 		Name: "ntfy_emails_sent_success", | 		Name: "ntfy_emails_sent_success", | ||||||
| 		}), | 	}) | ||||||
| 		emailsPublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{ | 	metricEmailsPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{ | ||||||
| 		Name: "ntfy_emails_sent_failure", | 		Name: "ntfy_emails_sent_failure", | ||||||
| 		}), | 	}) | ||||||
| 		emailsReceivedSuccess: prometheus.NewCounter(prometheus.CounterOpts{ | 	metricEmailsReceivedSuccess = prometheus.NewCounter(prometheus.CounterOpts{ | ||||||
| 		Name: "ntfy_emails_received_success", | 		Name: "ntfy_emails_received_success", | ||||||
| 		}), | 	}) | ||||||
| 		emailsReceivedFailure: prometheus.NewCounter(prometheus.CounterOpts{ | 	metricEmailsReceivedFailure = prometheus.NewCounter(prometheus.CounterOpts{ | ||||||
| 		Name: "ntfy_emails_received_failure", | 		Name: "ntfy_emails_received_failure", | ||||||
| 		}), | 	}) | ||||||
| 		unifiedPushPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{ | 	metricUnifiedPushPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{ | ||||||
| 		Name: "ntfy_unifiedpush_published_success", | 		Name: "ntfy_unifiedpush_published_success", | ||||||
| 		}), | 	}) | ||||||
| 		matrixPublishedSuccess: prometheus.NewCounter(prometheus.CounterOpts{ | 	metricMatrixPublishedSuccess = prometheus.NewCounter(prometheus.CounterOpts{ | ||||||
| 		Name: "ntfy_matrix_published_success", | 		Name: "ntfy_matrix_published_success", | ||||||
| 		}), | 	}) | ||||||
| 		matrixPublishedFailure: prometheus.NewCounter(prometheus.CounterOpts{ | 	metricMatrixPublishedFailure = prometheus.NewCounter(prometheus.CounterOpts{ | ||||||
| 		Name: "ntfy_matrix_published_failure", | 		Name: "ntfy_matrix_published_failure", | ||||||
| 		}), | 	}) | ||||||
| 		attachmentsTotalSize: prometheus.NewGauge(prometheus.GaugeOpts{ | 	metricAttachmentsTotalSize = prometheus.NewGauge(prometheus.GaugeOpts{ | ||||||
| 		Name: "ntfy_attachments_total_size", | 		Name: "ntfy_attachments_total_size", | ||||||
| 		}), | 	}) | ||||||
| 		visitors: prometheus.NewGauge(prometheus.GaugeOpts{ | 	metricVisitors = prometheus.NewGauge(prometheus.GaugeOpts{ | ||||||
| 		Name: "ntfy_visitors_total", | 		Name: "ntfy_visitors_total", | ||||||
| 		}), | 	}) | ||||||
| 		subscribers: prometheus.NewGauge(prometheus.GaugeOpts{ | 	metricSubscribers = prometheus.NewGauge(prometheus.GaugeOpts{ | ||||||
| 		Name: "ntfy_subscribers_total", | 		Name: "ntfy_subscribers_total", | ||||||
| 		}), | 	}) | ||||||
| 		topics: prometheus.NewGauge(prometheus.GaugeOpts{ | 	metricTopics = prometheus.NewGauge(prometheus.GaugeOpts{ | ||||||
| 		Name: "ntfy_topics_total", | 		Name: "ntfy_topics_total", | ||||||
| 		}), | 	}) | ||||||
| 		httpRequests: prometheus.NewCounterVec(prometheus.CounterOpts{ | 	metricHTTPRequests = prometheus.NewCounterVec(prometheus.CounterOpts{ | ||||||
| 		Name: "ntfy_http_requests_total", | 		Name: "ntfy_http_requests_total", | ||||||
| 		}, []string{"http_code", "ntfy_code", "http_method"}), | 	}, []string{"http_code", "ntfy_code", "http_method"}) | ||||||
| 	} |  | ||||||
| 	prometheus.MustRegister( | 	prometheus.MustRegister( | ||||||
| 		m.messagesPublishedSuccess, | 		metricMessagesPublishedSuccess, | ||||||
| 		m.messagesPublishedFailure, | 		metricMessagesPublishedFailure, | ||||||
| 		m.messagesCached, | 		metricMessagesCached, | ||||||
| 		m.firebasePublishedSuccess, | 		metricFirebasePublishedSuccess, | ||||||
| 		m.firebasePublishedFailure, | 		metricFirebasePublishedFailure, | ||||||
| 		m.emailsPublishedSuccess, | 		metricEmailsPublishedSuccess, | ||||||
| 		m.emailsPublishedFailure, | 		metricEmailsPublishedFailure, | ||||||
| 		m.emailsReceivedSuccess, | 		metricEmailsReceivedSuccess, | ||||||
| 		m.emailsReceivedFailure, | 		metricEmailsReceivedFailure, | ||||||
| 		m.unifiedPushPublishedSuccess, | 		metricUnifiedPushPublishedSuccess, | ||||||
| 		m.matrixPublishedSuccess, | 		metricMatrixPublishedSuccess, | ||||||
| 		m.matrixPublishedFailure, | 		metricMatrixPublishedFailure, | ||||||
| 		m.attachmentsTotalSize, | 		metricAttachmentsTotalSize, | ||||||
| 		m.visitors, | 		metricVisitors, | ||||||
| 		m.subscribers, | 		metricSubscribers, | ||||||
| 		m.topics, | 		metricTopics, | ||||||
| 		m.httpRequests, | 		metricHTTPRequests, | ||||||
| 	) | 	) | ||||||
| 	return m | } | ||||||
|  | 
 | ||||||
|  | // minc increments a prometheus.Counter if it is non-nil | ||||||
|  | func minc(counter prometheus.Counter) { | ||||||
|  | 	if counter != nil { | ||||||
|  | 		counter.Inc() | ||||||
|  | 	} | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | // mset sets a prometheus.Gauge if it is non-nil | ||||||
|  | func mset[T int | int64 | float64](gauge prometheus.Gauge, value T) { | ||||||
|  | 	if gauge != nil { | ||||||
|  | 		gauge.Set(float64(value)) | ||||||
|  | 	} | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -165,7 +165,7 @@ func (s *smtpSession) Data(r io.Reader) error { | ||||||
| 		s.backend.mu.Lock() | 		s.backend.mu.Lock() | ||||||
| 		s.backend.success++ | 		s.backend.success++ | ||||||
| 		s.backend.mu.Unlock() | 		s.backend.mu.Unlock() | ||||||
| 		metrics.emailsReceivedSuccess.Inc() | 		minc(metricEmailsReceivedSuccess) | ||||||
| 		return nil | 		return nil | ||||||
| 	}) | 	}) | ||||||
| } | } | ||||||
|  | @ -218,7 +218,7 @@ func (s *smtpSession) withFailCount(fn func() error) error { | ||||||
| 		// We do not want to spam the log with WARN messages. | 		// We do not want to spam the log with WARN messages. | ||||||
| 		logem(s.conn).Err(err).Debug("Incoming mail error") | 		logem(s.conn).Err(err).Debug("Incoming mail error") | ||||||
| 		s.backend.failure++ | 		s.backend.failure++ | ||||||
| 		metrics.emailsReceivedFailure.Inc() | 		minc(metricEmailsReceivedFailure) | ||||||
| 	} | 	} | ||||||
| 	return err | 	return err | ||||||
| } | } | ||||||
|  |  | ||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue