From f52b2a47626ded75bef81da68543a0f97fa22a19 Mon Sep 17 00:00:00 2001 From: Cezary Olborski Date: Mon, 2 Feb 2026 08:02:40 +0800 Subject: [PATCH 01/10] poc: Rocket webhook for Grafana alerts --- docker-compose.yml | 1 + env.example | 5 +++++ grafana/provisioning/alerting/contactpoints.yml | 9 +++++++++ 3 files changed, 15 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 692b419..8576a99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,6 +101,7 @@ services: # Telegram Configuration - TELEGRAM_BOT_TOKEN=${TELEGRAM_BOT_TOKEN} - TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID} + - ROCKET_WEBHOOK_URL=${ROCKET_WEBHOOK_URL} volumes: - grafana-data:/var/lib/grafana - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro diff --git a/env.example b/env.example index d6e8b4b..677c9ee 100644 --- a/env.example +++ b/env.example @@ -62,3 +62,8 @@ TELEGRAM_CHAT_ID=123456789 # TELEGRAM_BOT_TOKEN= # TELEGRAM_CHAT_ID= +# ============================================ +# ROCKET.CHAT WEBHOOK +# ============================================ +# Full Rocket.Chat Incoming Webhook URL +ROCKET_WEBHOOK_URL= diff --git a/grafana/provisioning/alerting/contactpoints.yml b/grafana/provisioning/alerting/contactpoints.yml index 18b2387..94e80ef 100644 --- a/grafana/provisioning/alerting/contactpoints.yml +++ b/grafana/provisioning/alerting/contactpoints.yml @@ -39,3 +39,12 @@ contactPoints: parse_mode: Markdown disableResolveMessage: false + - orgId: 1 + name: Rocket Notifications + receivers: + - uid: rocket-contact-point + type: webhook + settings: + url: ${ROCKET_WEBHOOK_URL} + httpMethod: POST + disableResolveMessage: false From eb14d58a425990a3fc824dff170ea189e1ff92c3 Mon Sep 17 00:00:00 2001 From: Cezary Olborski Date: Mon, 2 Feb 2026 08:11:08 +0800 Subject: [PATCH 02/10] fix: Docker compose contactpoints --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 8576a99..1ecff48 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -106,6 +106,7 @@ services: - grafana-data:/var/lib/grafana - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro + - ./grafana/provisioning/alerting:/etc/grafana/provisioning/alerting:ro - ./grafana/dashboards:/var/lib/grafana/dashboards - ./grafana/branding:/etc/grafana/branding:ro - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro From 34cb0218c79ecf4c2faf3b8753fe9df6a00a4eef Mon Sep 17 00:00:00 2001 From: Cezary Olborski Date: Mon, 2 Feb 2026 08:24:18 +0800 Subject: [PATCH 03/10] fix: Docker compose rollback --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1ecff48..8576a99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -106,7 +106,6 @@ services: - grafana-data:/var/lib/grafana - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro - - ./grafana/provisioning/alerting:/etc/grafana/provisioning/alerting:ro - ./grafana/dashboards:/var/lib/grafana/dashboards - ./grafana/branding:/etc/grafana/branding:ro - ./grafana/grafana.ini:/etc/grafana/grafana.ini:ro From 4d0528e1f1871f57a14dfc82c9a13a6fd9e63a5b Mon Sep 17 00:00:00 2001 From: Cezary Olborski Date: Mon, 2 Feb 2026 08:30:23 +0800 Subject: [PATCH 04/10] feat: Rocket json template --- grafana/provisioning/alerting/contactpoints.yml | 1 + grafana/provisioning/alerting/templates.yml | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 grafana/provisioning/alerting/templates.yml diff --git a/grafana/provisioning/alerting/contactpoints.yml b/grafana/provisioning/alerting/contactpoints.yml index 94e80ef..c8b9ecf 100644 --- a/grafana/provisioning/alerting/contactpoints.yml +++ b/grafana/provisioning/alerting/contactpoints.yml @@ -47,4 +47,5 @@ contactPoints: settings: url: ${ROCKET_WEBHOOK_URL} httpMethod: POST + message: '{{ template "rocket_template" . }}' disableResolveMessage: false diff --git a/grafana/provisioning/alerting/templates.yml b/grafana/provisioning/alerting/templates.yml new file mode 100644 index 0000000..dd5fddc --- /dev/null +++ b/grafana/provisioning/alerting/templates.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +templates: + - name: rocket_template + template: | + { + "text": "{{ range .Alerts }}šŸšØ **{{ .Labels.alertname }}**\n\n**Status:** {{ .Status }}\n{{ if .Labels.severity }}**Severity:** {{ .Labels.severity }}\n{{ end }}{{ if .Labels.chain }}**Chain:** {{ .Labels.chain }}\n{{ end }}{{ if .Labels.instance }}**Instance:** {{ .Labels.instance }}\n{{ end }}\n{{ if .Annotations.summary }}šŸ“‹ {{ .Annotations.summary }}\n{{ end }}{{ if .Annotations.description }}\n{{ .Annotations.description }}\n{{ end }}\nšŸ”— [View in Grafana]({{ .GeneratorURL }})\n{{ end }}", + "alias": "Quantus Monitor", + "emoji": ":warning:" + } From 17a4bbd1f0bf200aa8574dc169b70580b76249cf Mon Sep 17 00:00:00 2001 From: Cezary Olborski Date: Mon, 2 Feb 2026 08:34:41 +0800 Subject: [PATCH 05/10] fix: Rocket template fix --- grafana/provisioning/alerting/contactpoints.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/grafana/provisioning/alerting/contactpoints.yml b/grafana/provisioning/alerting/contactpoints.yml index c8b9ecf..cdbf9c4 100644 --- a/grafana/provisioning/alerting/contactpoints.yml +++ b/grafana/provisioning/alerting/contactpoints.yml @@ -47,5 +47,7 @@ contactPoints: settings: url: ${ROCKET_WEBHOOK_URL} httpMethod: POST - message: '{{ template "rocket_template" . }}' + title: '{{ template "__subject" . }}' + message: '{{ range .Alerts }}šŸšØ **{{ .Labels.alertname }}** - {{ .Status }}{{ if .Labels.severity }} ({{ .Labels.severity }}){{ end }}{{ if .Annotations.summary }}\nšŸ“‹ {{ .Annotations.summary }}{{ end }}{{ if .Annotations.description }}\n{{ .Annotations.description }}{{ end }}\nšŸ”— [View in Grafana]({{ .GeneratorURL }}){{ end }}' + username: Quantus Monitor disableResolveMessage: false From 5fc885c7fa167592dbdd22e4faf332828b6772a7 Mon Sep 17 00:00:00 2001 From: Cezary Olborski Date: Mon, 2 Feb 2026 09:37:59 +0800 Subject: [PATCH 06/10] fix: Slack hack --- grafana/provisioning/alerting/contactpoints.yml | 5 +---- grafana/provisioning/alerting/templates.yml | 10 ---------- 2 files changed, 1 insertion(+), 14 deletions(-) delete mode 100644 grafana/provisioning/alerting/templates.yml diff --git a/grafana/provisioning/alerting/contactpoints.yml b/grafana/provisioning/alerting/contactpoints.yml index cdbf9c4..da9cb16 100644 --- a/grafana/provisioning/alerting/contactpoints.yml +++ b/grafana/provisioning/alerting/contactpoints.yml @@ -43,11 +43,8 @@ contactPoints: name: Rocket Notifications receivers: - uid: rocket-contact-point - type: webhook + type: slack settings: url: ${ROCKET_WEBHOOK_URL} httpMethod: POST - title: '{{ template "__subject" . }}' - message: '{{ range .Alerts }}šŸšØ **{{ .Labels.alertname }}** - {{ .Status }}{{ if .Labels.severity }} ({{ .Labels.severity }}){{ end }}{{ if .Annotations.summary }}\nšŸ“‹ {{ .Annotations.summary }}{{ end }}{{ if .Annotations.description }}\n{{ .Annotations.description }}{{ end }}\nšŸ”— [View in Grafana]({{ .GeneratorURL }}){{ end }}' - username: Quantus Monitor disableResolveMessage: false diff --git a/grafana/provisioning/alerting/templates.yml b/grafana/provisioning/alerting/templates.yml deleted file mode 100644 index dd5fddc..0000000 --- a/grafana/provisioning/alerting/templates.yml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: 1 - -templates: - - name: rocket_template - template: | - { - "text": "{{ range .Alerts }}šŸšØ **{{ .Labels.alertname }}**\n\n**Status:** {{ .Status }}\n{{ if .Labels.severity }}**Severity:** {{ .Labels.severity }}\n{{ end }}{{ if .Labels.chain }}**Chain:** {{ .Labels.chain }}\n{{ end }}{{ if .Labels.instance }}**Instance:** {{ .Labels.instance }}\n{{ end }}\n{{ if .Annotations.summary }}šŸ“‹ {{ .Annotations.summary }}\n{{ end }}{{ if .Annotations.description }}\n{{ .Annotations.description }}\n{{ end }}\nšŸ”— [View in Grafana]({{ .GeneratorURL }})\n{{ end }}", - "alias": "Quantus Monitor", - "emoji": ":warning:" - } From 76c449eea622b222d478a76f0bdf06c3a4576272 Mon Sep 17 00:00:00 2001 From: Cezary Olborski Date: Mon, 2 Feb 2026 09:51:26 +0800 Subject: [PATCH 07/10] fix: Correct slack format --- .../provisioning/alerting/contactpoints.yml | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/grafana/provisioning/alerting/contactpoints.yml b/grafana/provisioning/alerting/contactpoints.yml index da9cb16..2682665 100644 --- a/grafana/provisioning/alerting/contactpoints.yml +++ b/grafana/provisioning/alerting/contactpoints.yml @@ -46,5 +46,28 @@ contactPoints: type: slack settings: url: ${ROCKET_WEBHOOK_URL} - httpMethod: POST + username: Grafana Monitor + title: '{{ template "slack.default.title" . }}' + text: | + {{ range .Alerts -}} + šŸšØ *{{ .Labels.alertname }}* - {{ .Status | toUpper }} + {{- if .Labels.severity }} + *Severity:* {{ .Labels.severity }} + {{- end }} + {{- if .Labels.chain }} + *Chain:* {{ .Labels.chain }} + {{- end }} + {{- if .Labels.instance }} + *Instance:* {{ .Labels.instance }} + {{- end }} + {{- if .Annotations.summary }} + + šŸ“‹ {{ .Annotations.summary }} + {{- end }} + {{- if .Annotations.description }} + {{ .Annotations.description }} + {{- end }} + + šŸ”— <{{ .GeneratorURL }}|View in Grafana> | <{{ .SilenceURL }}|Silence> + {{ end -}} disableResolveMessage: false From 740332403ebde8a01b76dccf7ec8687c431e7663 Mon Sep 17 00:00:00 2001 From: Cezary Olborski Date: Mon, 2 Feb 2026 09:54:24 +0800 Subject: [PATCH 08/10] fix: Slack format - once again --- grafana/provisioning/alerting/contactpoints.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/grafana/provisioning/alerting/contactpoints.yml b/grafana/provisioning/alerting/contactpoints.yml index 2682665..cc113d1 100644 --- a/grafana/provisioning/alerting/contactpoints.yml +++ b/grafana/provisioning/alerting/contactpoints.yml @@ -47,7 +47,6 @@ contactPoints: settings: url: ${ROCKET_WEBHOOK_URL} username: Grafana Monitor - title: '{{ template "slack.default.title" . }}' text: | {{ range .Alerts -}} šŸšØ *{{ .Labels.alertname }}* - {{ .Status | toUpper }} @@ -68,6 +67,10 @@ contactPoints: {{ .Annotations.description }} {{- end }} - šŸ”— <{{ .GeneratorURL }}|View in Grafana> | <{{ .SilenceURL }}|Silence> + {{- if .GeneratorURL }} + šŸ”— <{{ .GeneratorURL }}|View in Grafana>{{ if .SilenceURL }} | <{{ .SilenceURL }}|Silence>{{ end }} + {{- else if .SilenceURL }} + šŸ”— <{{ .SilenceURL }}|Silence> + {{- end }} {{ end -}} disableResolveMessage: false From 964d1026b61f09734ebf47a28becf6ecba28f233 Mon Sep 17 00:00:00 2001 From: Cezary Olborski Date: Mon, 2 Feb 2026 09:57:24 +0800 Subject: [PATCH 09/10] feat: Telegram alerts moved to Rocket --- grafana/provisioning/alerting/policies.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/grafana/provisioning/alerting/policies.yml b/grafana/provisioning/alerting/policies.yml index 1dd4a63..023f973 100644 --- a/grafana/provisioning/alerting/policies.yml +++ b/grafana/provisioning/alerting/policies.yml @@ -9,8 +9,8 @@ policies: repeat_interval: 4h routes: # šŸ”“ DIRAC - HIGHEST PRIORITY - # Critical alerts (Faucet/Explorer/Task Master DOWN) ā†’ Telegram + Email - - receiver: Telegram Notifications + # Critical alerts (Faucet/Explorer/Task Master DOWN) ā†’ Rocket + Email + - receiver: Rocket Notifications matchers: - chain = dirac - severity = critical @@ -36,8 +36,8 @@ policies: continue: false # GENERIC ALERTS (no chain label) - # Critical (Node Down, No Blocks, Disk Full, etc.) ā†’ Telegram + Email - - receiver: Telegram Notifications + # Critical (Node Down, No Blocks, Disk Full, etc.) ā†’ Rocket + Email + - receiver: Rocket Notifications matchers: - severity = critical group_wait: 10s From a860fb310748977ca7710097b3fbbcd2c3c4db86 Mon Sep 17 00:00:00 2001 From: Cezary Olborski Date: Mon, 2 Feb 2026 10:06:42 +0800 Subject: [PATCH 10/10] feat: Business alerts grupped --- grafana/provisioning/alerting/policies.yml | 46 ++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/grafana/provisioning/alerting/policies.yml b/grafana/provisioning/alerting/policies.yml index 023f973..b9b60be 100644 --- a/grafana/provisioning/alerting/policies.yml +++ b/grafana/provisioning/alerting/policies.yml @@ -9,16 +9,37 @@ policies: repeat_interval: 4h routes: # šŸ”“ DIRAC - HIGHEST PRIORITY - # Critical alerts (Faucet/Explorer/Task Master DOWN) ā†’ Rocket + Email + # Business critical alerts (No New Blocks) ā†’ Rocket + Email, grouped by alert only - receiver: Rocket Notifications matchers: - chain = dirac - severity = critical + - alertname = "No New Blocks" + group_by: ['alertname', 'chain'] # Group by alert name only, ignore instance group_wait: 2m # First notification after 2 minutes repeat_interval: 30m # Repeat every 30 minutes continue: true # Continue to email - # All Dirac alerts ā†’ Email + - receiver: Email Notifications + matchers: + - chain = dirac + - severity = critical + - alertname = "No New Blocks" + group_by: ['alertname', 'chain'] # Same grouping as Rocket + group_wait: 2m + repeat_interval: 30m + continue: false + + # Infrastructure critical alerts (Node Down, Low Peers, etc.) ā†’ Rocket + Email, per instance + - receiver: Rocket Notifications + matchers: + - chain = dirac + - severity = critical + group_wait: 2m + repeat_interval: 30m + continue: true + + # All other Dirac alerts ā†’ Email - receiver: Email Notifications matchers: - chain = dirac @@ -36,7 +57,26 @@ policies: continue: false # GENERIC ALERTS (no chain label) - # Critical (Node Down, No Blocks, Disk Full, etc.) ā†’ Rocket + Email + # Business critical alerts (No New Blocks) ā†’ Rocket + Email, grouped + - receiver: Rocket Notifications + matchers: + - severity = critical + - alertname = "No New Blocks" + group_by: ['alertname'] # Group by alert name only, ignore instance + group_wait: 10s + repeat_interval: 1h + continue: true + + - receiver: Email Notifications + matchers: + - severity = critical + - alertname = "No New Blocks" + group_by: ['alertname'] + group_wait: 10s + repeat_interval: 1h + continue: false + + # Infrastructure critical alerts (Node Down, Disk Full, etc.) ā†’ Rocket + Email, per instance - receiver: Rocket Notifications matchers: - severity = critical