From 6c14412aa936d24a7a6c124b390bbd95fdc66cd7 Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Tue, 17 Nov 2020 15:03:01 +0100 Subject: netwatch-notify: implemented simple dependency model --- doc/netwatch-notify.md | 13 +++++++++++-- netwatch-notify | 8 ++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/doc/netwatch-notify.md b/doc/netwatch-notify.md index 557ae88..66c5c75 100644 --- a/doc/netwatch-notify.md +++ b/doc/netwatch-notify.md @@ -8,8 +8,9 @@ Description This script sends notifications about host UP and DOWN events. In comparison to just netwatch (`/ tool netwatch`) and its `up-script` and `down-script` -this script implements a simple state machine. Host down events are triggered -only if the host is down for several checks to avoid false alerts. +this script implements a simple state machine and dependency model. Host +down events are triggered only if the host is down for several checks and +optional parent host is not down to avoid false alerts. Requirements and installation ----------------------------- @@ -39,6 +40,14 @@ The count threshould (default is 5 checks) is configurable as well: / tool netwatch add comment="notify, hostname=example.com, count=10" host=104.18.144.11; +If the host is behind another checked host add a dependency, this will +suppress notification if the parent host is down: + + / tool netwatch add comment="notify, hostname=gateway" host=93.184.216.1; + / tool netwatch add comment="notify, hostname=example.com, parent=gateway" host=93.184.216.34; + +Note that a configured parent increases the check count threshould by one. + Also notification settings are required for e-mail and telegram. --- diff --git a/netwatch-notify b/netwatch-notify index 6c2a18c..2462e9b 100644 --- a/netwatch-notify +++ b/netwatch-notify @@ -48,8 +48,12 @@ :set ($Metric->"since") ($HostVal->"since"); $LogPrintExit info ("Host " . $HostName . " (" . $HostVal->"host" . ") is down for " . \ $Metric->"count" . " checks.") false; - :if ($Metric->"count" >= [ $IfThenElse ([ :typeof ($HostInfo->"count") ] != "nothing") ($HostInfo->"count") 5 ] && \ - $Metric->"notified" != true) do={ + :local Count [ $IfThenElse ([ :tonum ($HostInfo->"count") ] > 0) ($HostInfo->"count") 5 ]; + :local ParentNotified [ $IfThenElse (($NetwatchNotify->($HostInfo->"parent")->"notified") = true) true false ]; + :if ([ :len ($HostInfo->"parent") ] > 0) do={ + :set Count ($Count + 1); + } + :if ($ParentNotified = false && $Metric->"count" >= $Count && $Metric->"notified" != true) do={ $SendNotification ([ $SymbolForNotification "cross-mark" ] . "Netwatch Notify: " . $HostName . " down") \ ("Host " . $HostName . " (" . $HostVal->"host" . ") is down since " . $HostVal->"since" . "."); :set ($Metric->"notified") true; -- cgit v1.2.3-54-g00ecf From d5125b816aeef0f86e48bdf5dbd1dccfe0164c9f Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Tue, 17 Nov 2020 15:14:27 +0100 Subject: netwatch-notify: be more verbose in logs --- netwatch-notify | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/netwatch-notify b/netwatch-notify index 2462e9b..4c3586c 100644 --- a/netwatch-notify +++ b/netwatch-notify @@ -46,13 +46,15 @@ } else={ :set ($Metric->"count") ($Metric->"count" + 1); :set ($Metric->"since") ($HostVal->"since"); - $LogPrintExit info ("Host " . $HostName . " (" . $HostVal->"host" . ") is down for " . \ - $Metric->"count" . " checks.") false; :local Count [ $IfThenElse ([ :tonum ($HostInfo->"count") ] > 0) ($HostInfo->"count") 5 ]; :local ParentNotified [ $IfThenElse (($NetwatchNotify->($HostInfo->"parent")->"notified") = true) true false ]; :if ([ :len ($HostInfo->"parent") ] > 0) do={ :set Count ($Count + 1); } + $LogPrintExit info ("Host " . $HostName . " (" . $HostVal->"host" . ") is down for " . \ + $Metric->"count" . " checks, " . [ $IfThenElse ($ParentNotified = false) [ $IfThenElse \ + ($Metric->"notified" = true) ("already notified.") ($Count - $Metric->"count" . " to go.") ] \ + ("parent host " . $HostInfo->"parent" . " is down.") ]) false; :if ($ParentNotified = false && $Metric->"count" >= $Count && $Metric->"notified" != true) do={ $SendNotification ([ $SymbolForNotification "cross-mark" ] . "Netwatch Notify: " . $HostName . " down") \ ("Host " . $HostName . " (" . $HostVal->"host" . ") is down since " . $HostVal->"since" . "."); -- cgit v1.2.3-54-g00ecf From e7855b4611f754fcde4f5adaff0e1a6006ddaaac Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Tue, 17 Nov 2020 21:37:23 +0100 Subject: netwatch-notify: support parents in a chain --- netwatch-notify | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/netwatch-notify b/netwatch-notify index 4c3586c..1cfd4a6 100644 --- a/netwatch-notify +++ b/netwatch-notify @@ -42,19 +42,28 @@ } } :set ($Metric->"notified") false; + :set ($Metric->"parent") ($HostInfo->"parent"); :set ($Metric->"since"); } else={ :set ($Metric->"count") ($Metric->"count" + 1); + :set ($Metric->"parent") ($HostInfo->"parent"); :set ($Metric->"since") ($HostVal->"since"); :local Count [ $IfThenElse ([ :tonum ($HostInfo->"count") ] > 0) ($HostInfo->"count") 5 ]; - :local ParentNotified [ $IfThenElse (($NetwatchNotify->($HostInfo->"parent")->"notified") = true) true false ]; - :if ([ :len ($HostInfo->"parent") ] > 0) do={ + :local Parent ($HostInfo->"parent"); + :if ([ :len $Parent ] > 0) do={ :set Count ($Count + 1); } + :local ParentNotified false; + :while ($ParentNotified = false && [ :len $Parent ] > 0) do={ + :set ParentNotified [ $IfThenElse (($NetwatchNotify->$Parent->"notified") = true) true false ]; + :if ($ParentNotified = false) do={ + :set Parent ($NetwatchNotify->$Parent->"parent"); + } + } $LogPrintExit info ("Host " . $HostName . " (" . $HostVal->"host" . ") is down for " . \ $Metric->"count" . " checks, " . [ $IfThenElse ($ParentNotified = false) [ $IfThenElse \ ($Metric->"notified" = true) ("already notified.") ($Count - $Metric->"count" . " to go.") ] \ - ("parent host " . $HostInfo->"parent" . " is down.") ]) false; + ("parent host " . $Parent . " is down.") ]) false; :if ($ParentNotified = false && $Metric->"count" >= $Count && $Metric->"notified" != true) do={ $SendNotification ([ $SymbolForNotification "cross-mark" ] . "Netwatch Notify: " . $HostName . " down") \ ("Host " . $HostName . " (" . $HostVal->"host" . ") is down since " . $HostVal->"since" . "."); @@ -68,5 +77,6 @@ :set ($NetwatchNotify->$HostName) { "count"=($Metric->"count"); "notified"=($Metric->"notified"); + "parent"=($Metric->"parent"); "since"=($Metric->"since") }; } -- cgit v1.2.3-54-g00ecf From 23923619dd0d92e4af65060e7b51b058fc1842aa Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Tue, 17 Nov 2020 22:02:41 +0100 Subject: netwatch-notify: increase count for every parent in chain --- doc/netwatch-notify.md | 3 ++- netwatch-notify | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/netwatch-notify.md b/doc/netwatch-notify.md index 66c5c75..11371ff 100644 --- a/doc/netwatch-notify.md +++ b/doc/netwatch-notify.md @@ -46,7 +46,8 @@ suppress notification if the parent host is down: / tool netwatch add comment="notify, hostname=gateway" host=93.184.216.1; / tool netwatch add comment="notify, hostname=example.com, parent=gateway" host=93.184.216.34; -Note that a configured parent increases the check count threshould by one. +Note that every configured parent in a chain increases the check count +threshould by one. Also notification settings are required for e-mail and telegram. diff --git a/netwatch-notify b/netwatch-notify index 1cfd4a6..9a12c5a 100644 --- a/netwatch-notify +++ b/netwatch-notify @@ -50,9 +50,11 @@ :set ($Metric->"since") ($HostVal->"since"); :local Count [ $IfThenElse ([ :tonum ($HostInfo->"count") ] > 0) ($HostInfo->"count") 5 ]; :local Parent ($HostInfo->"parent"); - :if ([ :len $Parent ] > 0) do={ + :while ([ :len $Parent ] > 0) do={ :set Count ($Count + 1); + :set Parent ($NetwatchNotify->$Parent->"parent"); } + :set Parent ($HostInfo->"parent"); :local ParentNotified false; :while ($ParentNotified = false && [ :len $Parent ] > 0) do={ :set ParentNotified [ $IfThenElse (($NetwatchNotify->$Parent->"notified") = true) true false ]; -- cgit v1.2.3-54-g00ecf From 62ef70e5dd0da2eee841359b8aa0305088880e58 Mon Sep 17 00:00:00 2001 From: Christian Hesse Date: Tue, 17 Nov 2020 15:03:01 +0100 Subject: netwatch-notify: notify about changes --- global-config | 2 +- global-config-overlay | 2 +- global-config.changes | 1 + global-functions | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/global-config b/global-config index 9bbbc29..591bdda 100644 --- a/global-config +++ b/global-config @@ -8,7 +8,7 @@ # Make sure all configuration properties are up to date and this # value is in sync with value in script 'global-functions'! -:global GlobalConfigVersion 36; +:global GlobalConfigVersion 37; # This is used for DNS and backup file. :global Domain "example.com"; diff --git a/global-config-overlay b/global-config-overlay index 2777519..a0c6361 100644 --- a/global-config-overlay +++ b/global-config-overlay @@ -9,7 +9,7 @@ # Make sure all configuration properties are up to date and this # value is in sync with value in script 'global-functions'! # Comment or remove to disable change notifications. -:global GlobalConfigVersion 36; +:global GlobalConfigVersion 37; # Copy configuration from global-config here and modify it. diff --git a/global-config.changes b/global-config.changes index 85529eb..5c400a5 100644 --- a/global-config.changes +++ b/global-config.changes @@ -40,4 +40,5 @@ 34="Introduced script 'ospf-to-leds' to visualize OSPF instance state via LEDs."; 35="Implemented visual feedback for 'mode-button' with configurable LED."; 36="Added support for installing updates automatically if seen in neighbor list."; + 37="Implemented simple dependency model in 'netwatch-notify'."; }; diff --git a/global-functions b/global-functions index a4a3630..b0b27c2 100644 --- a/global-functions +++ b/global-functions @@ -8,7 +8,7 @@ # https://git.eworm.de/cgit/routeros-scripts/about/ # expected configuration version -:global ExpectedConfigVersion 36; +:global ExpectedConfigVersion 37; # global variables not to be changed by user :global GlobalFunctionsReady false; -- cgit v1.2.3-54-g00ecf