Add prometheus + grafana

main
mathan 2024-02-18 07:37:50 -08:00
parent 8b89a1544e
commit 334af033b8
9 changed files with 24566 additions and 0 deletions

View File

@ -0,0 +1,77 @@
You'll need to install node exporter for monitoring
1. Download Node Exporter
As first step, you need to download the Node Exporter binary which is available for Linux in the official Prometheus website here. In the website, you will find a table with the list of available builds. Of our interest in this case, is the node_exporter build for Linux AMD64:
Node Exporter Ubuntu Linux
In this case the latest available version is the 1.3.1. Copy the .tar.gz URL and download it somewhere in your server using wget or cURL:
`wget https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz`
2. Extract Node Exporter and move binary
After downloading the latest version of Node Exporter, proceed to extract the content of the downloaded tar using the following command:
`tar xvf node_exporter-1.3.1.linux-amd64.tar.gz`
The content of the zip will be extracted in the current directory, the extracted directory will contain 3 files:
LICENSE (license text file)
node_exporter (binary)
NOTICE (license text file)
You only need to move the binary file node_exporter to the /usr/local/bin directory of your system. Switch to the node_exporter directory:
`cd node_exporter-1.3.1.linux-amd64`
And then copy the binary file with the following command:
`sudo cp node_exporter /usr/local/bin`
Then you can remove the directory that we created after extracting the zip file content:
# Exit current directory
`cd ..`
# Remove the extracted directory
`rm -rf ./node_exporter-1.3.1.linux-amd64`
3. Create Node Exporter User
As a good practice, create an user in the system for Node Exporter:
`sudo useradd --no-create-home --shell /bin/false node_exporter`
And set the owner of the binary node_exporter to the recently created user:
`sudo chown node_exporter:node_exporter /usr/local/bin/node_exporter`
4. Create and start the Node Exporter service
The Node Exporter service should always start when the server boots so it will always be available to be scrapped for information. Create the node_exporter.service file with nano:
`sudo nano /etc/systemd/system/node_exporter.service`
And paste the following content in the file:
```
[Unit]
Description=Node Exporter
Wants=network-online.target
After=network-online.target
[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/usr/local/bin/node_exporter
Restart=always
RestartSec=3
[Install]
WantedBy=multi-user.target
```
Close nano and save the changes to the file. Proceed to reload the daemon with:
`sudo systemctl daemon-reload`
And finally enable the node_exporter service with the following command:
`sudo systemctl enable node_exporter`
And then start the service:
`sudo systemctl start node_exporter`
`sudo ufw allow 9100`
now go to localhost:9100

View File

@ -0,0 +1,24 @@
version: '3.8'
services:
prometheus:
image: prom/prometheus
# needed if mounted in custom volume
user: root
volumes:
- "./prometheus.yml:/etc/prometheus/prometheus.yml"
- "${PROMETHEUS_DATA_DIR:?specify data dir in .env file}:/prometheus"
extra_hosts:
- "host.docker.internal:host-gateway"
ports:
- 9090:9090
grafana:
build:
context: ./grafana
container_name: grafana
user: root
extra_hosts:
- "host.docker.internal:host-gateway"
ports:
- 9000:3000
volumes:
- ${GRAFANA_DATA_DIR:?specify data dir in .env file}:/var/lib/grafana

View File

@ -0,0 +1,17 @@
FROM grafana/grafana:latest
# Disable Login form or not
ENV GF_AUTH_DISABLE_LOGIN_FORM "true"
# Allow anonymous authentication or not
ENV GF_AUTH_ANONYMOUS_ENABLED "true"
# Role of anonymous user
ENV GF_AUTH_ANONYMOUS_ORG_ROLE "Admin"
# Install plugins here our in your own config file
# ENV GF_INSTALL_PLUGINS="<list of plugins seperated by ,"
# Add provisioning
ADD ./provisioning /etc/grafana/provisioning
# Add configuration file
ADD ./grafana.ini /etc/grafana/grafana.ini
# Add dashboard json files
ADD ./dashboards /etc/grafana/dashboards

View File

@ -0,0 +1,541 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 16,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMin": 0,
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "A"
},
"properties": [
{
"id": "custom.axisPlacement",
"value": "right"
},
{
"id": "custom.stacking",
"value": {
"group": "A",
"mode": "normal"
}
},
{
"id": "custom.fillOpacity",
"value": 33
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "B"
},
"properties": [
{
"id": "unit",
"value": "repos/s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Fail rate"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Success rate"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"editorMode": "builder",
"expr": "sum by(state) (indexer_queue_length)",
"legendFormat": "{{state}}",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus"
},
"editorMode": "builder",
"expr": "sum by(success) (rate(indexer_repos_indexed_count[$rate_interval]))",
"hide": false,
"legendFormat": "__auto",
"range": true,
"refId": "B"
}
],
"title": "Indexing queue",
"transformations": [
{
"id": "renameByRegex",
"options": {
"regex": "false",
"renamePattern": "Fail rate"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "true",
"renamePattern": "Success rate"
}
}
],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMin": 0,
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "records/s"
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 0
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"editorMode": "builder",
"expr": "sum(rate(indexer_records_fetched_count[$rate_interval]))",
"legendFormat": "Fetch rate",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus"
},
"editorMode": "builder",
"expr": "sum(rate(indexer_records_inserted_count[$rate_interval]))",
"hide": false,
"legendFormat": "Insert rate",
"range": true,
"refId": "B"
}
],
"title": "Record indexing",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "A"
},
"properties": [
{
"id": "custom.stacking",
"value": {
"group": "A",
"mode": "normal"
}
},
{
"id": "custom.fillOpacity",
"value": 28
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 9
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus"
},
"editorMode": "builder",
"expr": "sum by(remote, success) (rate(indexer_repos_fetched_count{success=\"true\"}[$rate_interval]))",
"legendFormat": "{{success}} {{remote}}",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus"
},
"editorMode": "builder",
"expr": "sum(rate(indexer_repos_fetched_count{success=\"false\"}[$rate_interval]))",
"hide": false,
"legendFormat": "Errors",
"range": true,
"refId": "B"
}
],
"title": "Repo fetch rate",
"transformations": [
{
"id": "renameByRegex",
"options": {
"regex": "true https://([^.]*).*",
"renamePattern": "$1"
}
},
{
"id": "renameByRegex",
"options": {
"regex": "false https://([^.]*).*",
"renamePattern": "$1:fail"
}
}
],
"type": "timeseries"
},
{
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 9
},
"id": 8,
"title": "Disk space"
}
],
"refresh": "10s",
"revision": 1,
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"auto": false,
"auto_count": 30,
"auto_min": "10s",
"current": {
"selected": false,
"text": "30m",
"value": "30m"
},
"hide": 0,
"label": "Rate interval",
"name": "rate_interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": false,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "10m",
"value": "10m"
},
{
"selected": true,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
},
{
"selected": false,
"text": "6h",
"value": "6h"
},
{
"selected": false,
"text": "12h",
"value": "12h"
},
{
"selected": false,
"text": "1d",
"value": "1d"
}
],
"query": "1m,5m,10m,30m,1h,6h,12h,1d",
"queryValue": "",
"refresh": 2,
"skipUrlSync": false,
"type": "interval"
}
]
},
"time": {
"from": "now-3h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Bluesky indexer",
"uid": "aXw6dQhSz",
"version": 7,
"weekStart": ""
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,23 @@
[paths]
provisioning = /etc/grafana/provisioning
[server]
enable_gzip = true
# To add HTTPS support:
#protocol = https
#;http_addr =
#http_port = 3000
#domain = localhost
#enforce_domain = false
#root_url = https://localhost:3000
#router_logging = false
#static_root_path = public
#cert_file = /etc/certs/cert.pem
#cert_key = /etc/certs/cert-key.pem
[security]
# If you want to embed grafana into an iframe for example
allow_embedding = true
[users]
default_theme = dark

View File

@ -0,0 +1,25 @@
# config file version
apiVersion: 1
providers:
# <string> an unique provider name
- name: My Dashboard
# <int> org id. will default to orgId 1 if not specified
org_id: 1
# <string, required> name of the dashboard folder. Required
folder: ''
# <string, required> provider type. Required
type: 'file'
# <bool> disable dashboard deletion
disableDeletion: false
# <bool> enable dashboard editing
editable: true
# <int> how often Grafana will scan for changed dashboards
updateIntervalSeconds: 5
# <bool> allow updating provisioned dashboards from the UI
allowUiUpdates: true
options:
# <string, required> path to dashboard files on disk. Required
path: /etc/grafana/dashboards
# <bool> use folder names from filesystem to create folders in Grafana
foldersFromFilesStructure: true

View File

@ -0,0 +1,25 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
# Access mode - proxy (server in the UI) or direct (browser in the UI).
url: http://prometheus:9090
jsonData:
httpMethod: POST
manageAlerts: true
prometheusType: Prometheus
prometheusVersion: 2.49.0
cacheLevel: 'High'
disableRecordingRules: false
incrementalQueryOverlapWindow: 10m
exemplarTraceIdDestinations:
# Field with internal link pointing to data source in Grafana.
# datasourceUid value can be anything, but it should be unique across all defined data source uids.
- datasourceUid: 000000001
name: traceID
# Field with external link.
- name: traceID
url: 'http://host.docker.internal:3000/explore?orgId=1&left=%5B%22now-1h%22,%22now%22,%22Jaeger%22,%7B%22query%22:%22$${__value.raw}%22%7D%5D'

View File

@ -0,0 +1,13 @@
global:
scrape_interval: 10s
scrape_configs:
- job_name: prometheus
static_configs:
- targets:
- host.docker.internal:9090
- job_name: node
static_configs:
- targets: ['host.docker.internal:9100']
- job_name: indexer
static_configs:
- targets: [ host.docker.internal:11001, host.docker.internal:11002, host.docker.internal:11003 ]