#!rsc by RouterOS # RouterOS script: netwatch-notify # Copyright (c) 2020-2022 Christian Hesse <mail@eworm.de> # https://git.eworm.de/cgit/routeros-scripts/about/COPYING.md # # monitor netwatch and send notifications # https://git.eworm.de/cgit/routeros-scripts/about/doc/netwatch-notify.md :local 0 "netwatch-notify"; :global GlobalFunctionsReady; :while ($GlobalFunctionsReady != true) do={ :delay 500ms; } :global NetwatchNotify; :global EitherOr; :global IfThenElse; :global IsDNSResolving; :global LogPrintExit2; :global ParseKeyValueStore; :global ScriptLock; :global SendNotification2; :global SymbolForNotification; :local NetwatchNotifyHook do={ :local Name [ :tostr $1 ]; :local Type [ :tostr $2 ]; :local State [ :tostr $3 ]; :local Hook [ :tostr $4 ]; :global LogPrintExit2; :global ValidateSyntax; :if ([ $ValidateSyntax $Hook ] = true) do={ :do { [ :parse $Hook ]; } on-error={ $LogPrintExit2 warning $0 ("The " . $State . "-hook for " . $Type . " '" . $Name . \ "' failed to run.") false; :return ("The hook failed to run."); } } else={ $LogPrintExit2 warning $0 ("The " . $State . "-hook for " . $Type . " '" . $Name . \ "' failed syntax validation.") false; :return ("The hook failed syntax validation."); } $LogPrintExit2 info $0 ("Ran hook on " . $Type . " '" . $Name . "' " . $State . ": " . \ $Hook) false; :return ("Ran hook:\n" . $Hook); } $ScriptLock $0; :if ([ /system/resource/get uptime ] < 5m) do={ $LogPrintExit2 info $0 ("System just booted, giving netwatch some time to settle.") true; } :if ([ :typeof $NetwatchNotify ] = "nothing") do={ :set NetwatchNotify ({}); } :foreach Host in=[ /tool/netwatch/find where comment~"notify" !disabled ] do={ :local HostVal [ /tool/netwatch/get $Host ]; :local Type [ $IfThenElse ($HostVal->"type" ~ "^(http-get|tcp-conn)\$") "service" "host" ]; :local HostInfo [ $ParseKeyValueStore ($HostVal->"comment") ]; :local HostDetails ($HostVal->"host" . \ [ $IfThenElse ([ :len ($HostInfo->"resolve") ] > 0) (", " . $HostInfo->"resolve") ]); :if ($HostInfo->"notify" = true && $HostInfo->"disabled" != true) do={ :local Name [ $EitherOr ($HostInfo->"name") ($HostVal->"name") ]; :local Metric { "count"=0; "notified"=false }; :if ([ :typeof ($NetwatchNotify->$Name) ] = "array") do={ :set $Metric ($NetwatchNotify->$Name); } :if ([ :typeof ($HostInfo->"resolve") ] = "str") do={ :if ([ $IsDNSResolving ] = true) do={ :do { :local Resolve [ :resolve ($HostInfo->"resolve") ]; :if ($Resolve != $HostVal->"host") do={ $LogPrintExit2 info $0 ("Name '" . $HostInfo->"resolve" . [ $IfThenElse \ ($HostInfo->"resolve" != $HostInfo->"name") ("' for " . $Type . " '" . \ $HostInfo->"name") "" ] . "' resolves to different address " . $Resolve . \ ", updating.") false; /tool/netwatch/set host=$Resolve $Host; :set ($Metric->"resolve-failed") false; } } on-error={ :if ($Metric->"resolve-failed" != true) do={ $LogPrintExit2 warning $0 ("Resolving name '" . $HostInfo->"resolve" . [ $IfThenElse \ ($HostInfo->"resolve" != $HostInfo->"name") ("' for " . $Type . " '" . \ $HostInfo->"name") "" ] . "' failed.") false; :set ($Metric->"resolve-failed") true; } } } } :if ($HostVal->"status" = "up") do={ :local CountDown ($Metric->"count"); :if ($CountDown > 0) do={ $LogPrintExit2 info $0 \ ("The " . $Type . " '" . $Name . "' (" . $HostDetails . ") is up.") false; :set ($Metric->"count") 0; } :if ($Metric->"notified" = true) do={ :local Message ("The " . $Type . " '" . $Name . "' (" . $HostDetails . \ ") is up since " . $HostVal->"since" . ".\n" . \ "It was down for " . $CountDown . " checks since " . ($Metric->"since") . "."); :if ([ :typeof ($HostInfo->"up-hook") ] = "str") do={ :set Message ($Message . "\n\n" . [ $NetwatchNotifyHook $Name $Type "up" \ ($HostInfo->"up-hook") ]); } $SendNotification2 ({ origin=$0; \ subject=([ $SymbolForNotification "white-heavy-check-mark" ] . "Netwatch Notify: " . \ $Name . " up"); \ message=$Message }); } :set ($Metric->"notified") false; :set ($Metric->"parent") ($HostInfo->"parent"); :set ($Metric->"since"); } else={ :set ($Metric->"count") ($Metric->"count" + 1); :set ($Metric->"parent") ($HostInfo->"parent"); :set ($Metric->"since") ($HostVal->"since"); :local CountDown [ $IfThenElse ([ :tonum ($HostInfo->"count") ] > 0) ($HostInfo->"count") 5 ]; :local Parent ($HostInfo->"parent"); :while ([ :len $Parent ] > 0) do={ :set CountDown ($CountDown + 1); :set Parent ($NetwatchNotify->$Parent->"parent"); } :set Parent ($HostInfo->"parent"); :local ParentNotified false; :while ($ParentNotified = false && [ :len $Parent ] > 0) do={ :set ParentNotified [ $IfThenElse (($NetwatchNotify->$Parent->"notified") = true) \ true false ]; :if ($ParentNotified = false) do={ :set Parent ($NetwatchNotify->$Parent->"parent"); } } $LogPrintExit2 [ $IfThenElse ($HostInfo->"no-down-notification" != true) info debug ] $0 \ ("The " . $Type . " '" . $Name . "' (" . $HostDetails . ") is down for " . \ $Metric->"count" . " checks, " . [ $IfThenElse ($ParentNotified = false) [ $IfThenElse \ ($Metric->"notified" = true) ("already notified.") ($CountDown - $Metric->"count" . \ " to go.") ] ("parent " . $Type . " " . $Parent . " is down.") ]) false; :if ((($CountDown * 2) - ($Metric->"count" * 3)) / 2 = 0 && \ [ :typeof ($HostInfo->"pre-down-hook") ] = "str") do={ $NetwatchNotifyHook $Name $Type "pre-down" ($HostInfo->"pre-down-hook"); } :if ($ParentNotified = false && $Metric->"count" >= $CountDown && \ $Metric->"notified" != true) do={ :local Message ("The " . $Type . " '" . $Name . "' (" . $HostDetails . \ ") is down since " . $HostVal->"since" . "."); :if ([ :typeof ($HostInfo->"down-hook") ] = "str") do={ :set Message ($Message . "\n\n" . [ $NetwatchNotifyHook $Name $Type "down" \ ($HostInfo->"down-hook") ]); } :if ($HostInfo->"no-down-notification" != true) do={ $SendNotification2 ({ origin=$0; \ subject=([ $SymbolForNotification "cross-mark" ] . "Netwatch Notify: " . \ $Name . " down"); \ message=$Message }); } :set ($Metric->"notified") true; } } :set ($NetwatchNotify->$Name) { "count"=($Metric->"count"); "notified"=($Metric->"notified"); "parent"=($Metric->"parent"); "resolve-failed"=($Metric->"resolve-failed"); "since"=($Metric->"since") }; } }