Move prometheus + grafana to metrics. Edit readme

metrics/README.md

@@ -0,0 +1,92 @@
+# To start prometheus + grafana
+
+`cd metrics`
+
+`docker compose up -d --build`
+
+### Note: remember to allow ports for Prometheus to see host.docker.internal:xxxx from within container
+
+`sudo ufw allow 11001`
+`sudo ufw allow 11002`
+`sudo ufw allow 11003`
+
+# Install Node-exporter
+
+You'll need to install node exporter for monitoring
+
+1. Download Node Exporter
+As first step, you need to download the Node Exporter binary which is available for Linux in the official Prometheus website here. In the website, you will find a table with the list of available builds. Of our interest in this case, is the node_exporter build for Linux AMD64:
+
+Node Exporter Ubuntu Linux
+
+In this case the latest available version is the 1.7.0. Copy the .tar.gz URL and download it somewhere in your server using wget or cURL:
+
+`wget https://github.com/prometheus/node_exporter/releases/download/v1.7.0/node_exporter-1.7.0.linux-amd64.tar.gz`
+
+2. Extract Node Exporter and move binary
+After downloading the latest version of Node Exporter, proceed to extract the content of the downloaded tar using the following command:
+
+`tar xvf node_exporter-1.7.0.linux-amd64.tar.gz`
+The content of the zip will be extracted in the current directory, the extracted directory will contain 3 files:
+
+LICENSE (license text file)
+node_exporter (binary)
+NOTICE (license text file)
+You only need to move the binary file node_exporter to the /usr/local/bin directory of your system. Switch to the node_exporter directory:
+
+`cd node_exporter-1.7.0.linux-amd64`
+And then copy the binary file with the following command:
+
+`sudo cp node_exporter /usr/local/bin`
+Then you can remove the directory that we created after extracting the zip file content:
+
+# Exit current directory
+`cd ..`
+
+# Remove the extracted directory
+`rm -rf ./node_exporter-1.7.0.linux-amd64`
+3. Create Node Exporter User
+As a good practice, create an user in the system for Node Exporter:
+
+`sudo useradd --no-create-home --shell /bin/false node_exporter`
+And set the owner of the binary node_exporter to the recently created user:
+
+`sudo chown node_exporter:node_exporter /usr/local/bin/node_exporter`
+4. Create and start the Node Exporter service
+The Node Exporter service should always start when the server boots so it will always be available to be scrapped for information. Create the node_exporter.service file with nano:
+
+`sudo nano /etc/systemd/system/node_exporter.service`
+And paste the following content in the file:
+
+```
+[Unit]
+Description=Node Exporter
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+User=node_exporter
+Group=node_exporter
+Type=simple
+ExecStart=/usr/local/bin/node_exporter
+Restart=always
+RestartSec=3
+
+[Install]https://github.com/prometheus/node_exporter/releases/download
+WantedBy=multi-user.target
+```
+
+Close nano and save the changes to the file. Proceed to reload the daemon with:
+
+`sudo systemctl daemon-reload`
+And finally enable the node_exporter service with the following command:
+
+`sudo systemctl enable node_exporter`
+And then start the service:
+
+`sudo systemctl start node_exporter`
+
+`sudo ufw allow 9090`
+`sudo ufw allow 9100`
+
+now go to `http://localhost:9100/metrics`

metrics/docker-compose.yml

@@ -0,0 +1,24 @@
+version: '3.8'
+services:
+  prometheus:
+    image: prom/prometheus
+    # needed if mounted in custom volume
+    user: root
+    volumes:
+      - "./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml"
+      - "${PROMETHEUS_DATA_DIR:?specify data dir in .env file}:/prometheus"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    ports:
+      - 9090:9090
+  grafana:
+    build:
+      context: ./grafana
+    container_name: grafana
+    user: root
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    ports:
+      - 9000:3000
+    volumes:
+      - ${GRAFANA_DATA_DIR:?specify data dir in .env file}:/var/lib/grafana

metrics/grafana/Dockerfile

@@ -0,0 +1,17 @@
+FROM grafana/grafana:latest
+
+# Disable Login form or not
+ENV GF_AUTH_DISABLE_LOGIN_FORM "true"
+# Allow anonymous authentication or not
+ENV GF_AUTH_ANONYMOUS_ENABLED "true"
+# Role of anonymous user
+ENV GF_AUTH_ANONYMOUS_ORG_ROLE "Admin"
+# Install plugins here our in your own config file
+# ENV GF_INSTALL_PLUGINS="<list of plugins seperated by ,"
+
+# Add provisioning
+ADD ./provisioning /etc/grafana/provisioning
+# Add configuration file
+ADD ./grafana.ini /etc/grafana/grafana.ini
+# Add dashboard json files
+ADD ./dashboards /etc/grafana/dashboards

metrics/grafana/grafana.ini

@@ -0,0 +1,23 @@
+[paths]
+provisioning = /etc/grafana/provisioning
+
+[server]
+enable_gzip = true
+# To add HTTPS support:
+#protocol = https
+#;http_addr =
+#http_port = 3000
+#domain = localhost
+#enforce_domain = false
+#root_url = https://localhost:3000
+#router_logging = false
+#static_root_path = public
+#cert_file = /etc/certs/cert.pem
+#cert_key = /etc/certs/cert-key.pem
+
+[security]
+# If you want to embed grafana into an iframe for example
+allow_embedding = true
+
+[users]
+default_theme = dark

metrics/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,25 @@
+# config file version
+apiVersion: 1
+
+providers:
+# <string> an unique provider name
+- name: My Dashboard
+  # <int> org id. will default to orgId 1 if not specified
+  org_id: 1
+  # <string, required> name of the dashboard folder. Required
+  folder: ''
+  # <string, required> provider type. Required
+  type: 'file'
+  # <bool> disable dashboard deletion
+  disableDeletion: false
+  # <bool> enable dashboard editing
+  editable: true
+  # <int> how often Grafana will scan for changed dashboards
+  updateIntervalSeconds: 5
+  # <bool> allow updating provisioned dashboards from the UI
+  allowUiUpdates: true
+  options:
+    # <string, required> path to dashboard files on disk. Required
+    path: /etc/grafana/dashboards
+    # <bool> use folder names from filesystem to create folders in Grafana
+    foldersFromFilesStructure: true

metrics/grafana/provisioning/datasources/datarource.yml

@@ -0,0 +1,25 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    # Access mode - proxy (server in the UI) or direct (browser in the UI).
+    url: http://prometheus:9090
+    jsonData:
+      httpMethod: POST
+      manageAlerts: true
+      prometheusType: Prometheus
+      prometheusVersion: 2.49.0
+      cacheLevel: 'High'
+      disableRecordingRules: false
+      incrementalQueryOverlapWindow: 10m
+      exemplarTraceIdDestinations:
+        # Field with internal link pointing to data source in Grafana.
+        # datasourceUid value can be anything, but it should be unique across all defined data source uids.
+        - datasourceUid: 000000001
+          name: traceID
+
+        # Field with external link.
+        - name: traceID
+          url: 'http://host.docker.internal:3000/explore?orgId=1&left=%5B%22now-1h%22,%22now%22,%22Jaeger%22,%7B%22query%22:%22$${__value.raw}%22%7D%5D'

metrics/prometheus/prometheus.yml

@@ -0,0 +1,13 @@
+global:
+  scrape_interval: 10s
+scrape_configs:
+ - job_name: prometheus
+   static_configs:
+    - targets:
+       - host.docker.internal:9090
+ - job_name: node
+   static_configs:
+    - targets: ['host.docker.internal:9100']
+ - job_name: indexer
+   static_configs:
+    - targets: [ host.docker.internal:11001, host.docker.internal:11002, host.docker.internal:11003 ]