1 files changed, 115 insertions, 116 deletions

diff --git a/global-functions.rsc b/global-functions.rsc
index 919ffbe..f890376 100644
--- a/global-functions.rsc
+++ b/global-functions.rsc
@@ -4,7 +4,7 @@
 #                         Michael Gisbers <michael@gisbers.de>
 # https://git.eworm.de/cgit/routeros-scripts/about/COPYING.md
 #
-# requires RouterOS, version=7.12
+# requires RouterOS, version=7.13
 #
 # global functions
 # https://git.eworm.de/cgit/routeros-scripts/about/
@@ -12,7 +12,7 @@
 :local ScriptName [ :jobname ];
 
 # expected configuration version
-:global ExpectedConfigVersion 124;
+:global ExpectedConfigVersion 128;
 
 # global variables not to be changed by user
 :global GlobalFunctionsReady false;
@@ -32,7 +32,8 @@
 :global DownloadPackage;
 :global EitherOr;
 :global EscapeForRegEx;
-:global FetchUserAgent;
+:global FetchHuge;
+:global FetchUserAgentStr;
 :global FormatLine;
 :global FormatMultiLines;
 :global GetMacVendor;
@@ -49,16 +50,15 @@
 :global IsMacLocallyAdministered;
 :global IsTimeSync;
 :global LogPrint;
-:global LogPrintExit2;
 :global LogPrintOnce;
 :global MAX;
 :global MIN;
 :global MkDir;
 :global NotificationFunctions;
 :global ParseDate;
-:global ParseJson;
 :global ParseKeyValueStore;
 :global PrettyPrint;
+:global ProtocolStrip;
 :global RandomDelay;
 :global RequiredRouterOS;
 :global ScriptFromTerminal;
@@ -108,7 +108,7 @@
   }
 
   :if ([ :len [ /certificate/find where common-name=$CommonName ] ] = 0) do={
-    $LogPrint info $0 ("Certificate with CommonName \"" . $CommonName . "\" not available.");
+    $LogPrint info $0 ("Certificate with CommonName '" . $CommonName . "' not available.");
     :if ([ $CertificateDownload $CommonName ] = false) do={
       :return false;
     }
@@ -117,8 +117,8 @@
   :local CertVal [ /certificate/get [ find where common-name=$CommonName ] ];
   :while (($CertVal->"akid") != "" && ($CertVal->"akid") != ($CertVal->"skid")) do={
     :if ([ :len [ /certificate/find where skid=($CertVal->"akid") ] ] = 0) do={
-      $LogPrint info $0 ("Certificate chain for \"" . $CommonName . \
-        "\" is incomplete, missing \"" . ([ $ParseKeyValueStore ($CertVal->"issuer") ]->"CN") . "\".");
+      $LogPrint info $0 ("Certificate chain for '" . $CommonName . \
+        "' is incomplete, missing '" . ([ $ParseKeyValueStore ($CertVal->"issuer") ]->"CN") . "'.");
       :if ([ $CertificateDownload $CommonName ] = false) do={
         :return false;
       }
@@ -137,27 +137,27 @@
 
   :global CertificateNameByCN;
   :global CleanName;
-  :global FetchUserAgent;
+  :global FetchUserAgentStr;
   :global LogPrint;
   :global WaitForFile;
 
   $LogPrint info $0 ("Downloading and importing certificate with " . \
-      "CommonName \"" . $CommonName . "\".");
+      "CommonName '" . $CommonName . "'.");
   :do {
     :local FileName ([ $CleanName $CommonName ] . ".pem");
-    /tool/fetch check-certificate=yes-without-crl http-header-field=({ [ $FetchUserAgent $0 ] }) \
+    /tool/fetch check-certificate=yes-without-crl http-header-field=({ [ $FetchUserAgentStr $0 ] }) \
       ($ScriptUpdatesBaseUrl . "certs/" . $FileName . $ScriptUpdatesUrlSuffix) \
       dst-path=$FileName as-value;
     $WaitForFile $FileName;
     /certificate/import file-name=$FileName passphrase="" as-value;
     :delay 1s;
-    /file/remove $FileName;
+    /file/remove [ find where name=$FileName ];
 
     :foreach Cert in=[ /certificate/find where name~("^" . $FileName . "_[0-9]+\$") ] do={
       $CertificateNameByCN [ /certificate/get $Cert common-name ];
     }
   } on-error={
-    $LogPrint warning $0 ("Failed importing certificate with CommonName \"" . $CommonName . "\"!");
+    $LogPrint warning $0 ("Failed importing certificate with CommonName '" . $CommonName . "'!");
     :return false;
   }
   :return true;
@@ -230,11 +230,19 @@
   :for I from=0 to=([ :len $Input ] - 1) do={
     :local Char [ :pick $Input $I ];
     :if ([ :typeof [ find "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-" $Char ] ] = "nil") do={
-      :set Char "-";
-    }
-    :if ($Char != "-" || [ :pick $Return ([ :len $Return ] - 1) ] != "-") do={
-      :set Return ($Return . $Char);
+      :do {
+        :if ([ :len $Return ] = 0) do={
+          :error true;
+        }
+        :if ([ :pick $Return ([ :len $Return ] - 1) ] = "-") do={
+          :error true;
+        }
+        :set Char "-";
+      } on-error={
+        :set Char "";
+      }
     }
+    :set Return ($Return . $Char);
   }
   :return $Return;
 }
@@ -390,8 +398,55 @@
   :return $Return;
 }
 
+# fetch huge data to file, read in chunks
+:set FetchHuge do={
+  :local ScriptName [ :tostr  $1 ];
+  :local Url        [ :tostr  $2 ];
+  :local CheckCert  [ :tobool $3 ];
+
+  :global CleanName;
+  :global FetchUserAgentStr;
+  :global GetRandom20CharAlNum;
+  :global IfThenElse;
+  :global LogPrint;
+  :global MkDir;
+  :global WaitForFile;
+
+  :set CheckCert [ $IfThenElse ($CheckCert = false) "no" "yes-without-crl" ];
+
+  :local DirName ("tmpfs/" . [ $CleanName $ScriptName ]);
+  :if ([ $MkDir $DirName ] = false) do={
+    $LogPrint error $0 ("Failed creating directory!");
+    :return false;
+  }
+
+  :local FileName ($DirName . "/" . [ $CleanName $0 ] . "-" . [ $GetRandom20CharAlNum ]);
+  :do {
+    /tool/fetch check-certificate=$CheckCert $Url dst-path=$FileName \
+      http-header-field=({ [ $FetchUserAgentStr $ScriptName ] }) as-value;
+  } on-error={
+    :if ([ $WaitForFile $FileName 500ms ] = true) do={
+      /file/remove $FileName;
+    }
+    $LogPrint debug $0 ("Failed downloading from: " . $Url);
+    /file/remove $DirName;
+    :return false;
+  }
+  $WaitForFile $FileName;
+
+  :local FileSize [ /file/get $FileName size ];
+  :local Return "";
+  :local VarSize 0;
+  :while ($VarSize < $FileSize) do={
+    :set Return ($Return . ([ /file/read offset=$VarSize chunk-size=32768 file=$FileName as-value ]->"data"));
+    :set VarSize [ :len $Return ];
+  }
+  /file/remove $DirName;
+  :return $Return;
+}
+
 # generate user agent string for fetch
-:set FetchUserAgent do={
+:set FetchUserAgentStr do={
   :local Caller [ :tostr $1 ];
 
   :local Resource [ /system/resource/get ];
@@ -528,16 +583,18 @@
 }
 
 # return human readable number
-:global HumanReadableNum do={
+:set HumanReadableNum do={
   :local Input [ :tonum $1 ];
   :local Base  [ :tonum $2 ];
 
   :global EitherOr;
+  :global IfThenElse;
 
   :local Prefix "kMGTPE";
   :local Pow 1;
 
   :set Base [ $EitherOr $Base 1024 ];
+  :local Bin [ $IfThenElse ($Base = 1024) "i" "" ];
 
   :if ($Input < $Base) do={
     :return $Input;
@@ -550,9 +607,11 @@
       :local Tmp1 ($Input * 100 / $Pow);
       :local Tmp2 ($Tmp1 / 100);
       :if ($Tmp2 >= 100) do={
-        :return ($Tmp2 . $Prefix);
+        :return ($Tmp2 . $Prefix . $Bin);
       }
-      :return ($Tmp2 . "." . [ :pick $Tmp1 [ :len $Tmp2 ] ([ :len $Tmp1 ] - [ :len $Tmp2 ] + 1) ] . $Prefix);
+      :return ($Tmp2 . "." . \
+          [ :pick $Tmp1 [ :len $Tmp2 ] ([ :len $Tmp1 ] - [ :len $Tmp2 ] + 1) ] . \
+          $Prefix . $Bin);
     }
   }
 }
@@ -614,7 +673,7 @@
   :global IsTimeSyncCached;
   :global IsTimeSyncResetNtp;
 
-  :global LogPrint;
+  :global LogPrintOnce;
 
   :if ($IsTimeSyncCached = true) do={
     :return true;
@@ -634,6 +693,7 @@
       :return false;
     }
 
+    $LogPrintOnce warning $0 ("The ntp client is configured, but did not sync.");
     :set IsTimeSyncResetNtp $Uptime;
     /system/ntp/client/set enabled=no;
     :delay 20ms;
@@ -643,7 +703,7 @@
 
   :if ([ /system/license/get ]->"level" = "free" || \
        [ /system/resource/get ]->"board-name" = "x86") do={
-    $LogPrint debug $0 ("No ntp client configured, relying on RTC for CHR free license and x86.");
+    $LogPrintOnce debug $0 ("No ntp client configured, relying on RTC for CHR free license and x86.");
     :return true;
   }
 
@@ -655,7 +715,7 @@
     :return false;
   }
 
-  $LogPrint debug $0 ("No time source configured! Returning gracefully...");
+  $LogPrintOnce debug $0 ("No time source configured! Returning gracefully...");
   :return true;
 }
 
@@ -698,27 +758,6 @@
   }
 }
 
-# log and print with same text, optionally exit
-# Deprectated! - TODO: remove later
-:set LogPrintExit2 do={
-  :local Severity [ :tostr $1 ];
-  :local Name     [ :tostr $2 ];
-  :local Message  [ :tostr $3 ];
-  :local Exit     [ :tostr $4 ];
-
-  :global LogPrint;
-  :global LogPrintOnce;
-
-  $LogPrintOnce warning $0 \
-    ("This function is deprecated and will be removed. Please make your adjustments!");
-
-  $LogPrint $1 $2 $3;
-
-  :if ($Exit = "true") do={
-    :error ("Hard error to exit.");
-  }
-}
-
 # log and print, once until reboot
 :set LogPrintOnce do={
   :local Severity [ :tostr $1 ];
@@ -830,66 +869,6 @@
               "day"=[ :tonum [ :pick $Date 8 10 ] ] });
 }
 
-# parse JSON into array
-# Warning: This is not a complete parser!
-:set ParseJson do={
-  :local Input [ :tostr $1 ];
-
-  :local InLen;
-  :local Return ({});
-  :local Skip 0;
-
-  :if ([ :pick $Input 0 ] = "{") do={
-    :set Input [ :pick $Input 1 ([ :len $Input ] - 1) ];
-  }
-  :set Input [ :toarray $Input ];
-  :set InLen [ :len $Input ];
-
-  :for I from=0 to=$InLen do={
-    :if ($Skip > 0 || $Input->$I = "\n" || $Input->$I = "\r\n") do={
-      :if ($Skip > 0) do={
-        :set $Skip ($Skip - 1);
-      }
-    } else={
-      :local Done false;
-      :local Key ($Input->$I);
-      :local Val1 ($Input->($I + 1));
-      :local Val2 ($Input->($I + 2));
-      :if ($Val1 = ":") do={
-        :set Skip 2;
-        :set ($Return->$Key) $Val2;
-        :set Done true;
-      }
-      :if ($Done = false && $Val1 = ":[") do={
-        :local Last false;
-        :set Skip 1;
-        :set ($Return->$Key) ({});
-        :do {
-          :set Skip ($Skip + 1);
-          :local ValX ($Input->($I + $Skip));
-          :if ([ :pick $ValX ([ :len $ValX ] - 1) ] = "]") do={
-            :set Last true;
-            :set ValX [ :pick $ValX 0 ([ :len $ValX ] - 1) ];
-          }
-          :set ($Return->$Key) (($Return->$Key), $ValX);
-        } while=($Last = false && $I + $Skip < $InLen);
-        :set Done true;
-      }
-      :if ($Done = false && $Val1 = ":[]") do={
-        :set Skip 1;
-        :set ($Return->$Key) ({});
-        :set Done true;
-      }
-      :if ($Done = false) do={
-        :set Skip 1;
-        :set ($Return->$Key) [ :pick $Val1 1 [ :len $Val1 ] ];
-      }
-    }
-  }
-
-  :return $Return;
-}
-
 # parse key value store
 :set ParseKeyValueStore do={
   :local Source $1;
@@ -917,6 +896,17 @@
   :put [ $Unix2Dos $Input ];
 }
 
+# strip protocol from from url string
+:set ProtocolStrip do={
+  :local Input [ :tostr $1 ];
+
+  :local Pos [ :find $Input "://" ];
+  :if ([ :typeof $Pos ] = "nil") do={
+    :return $Input;
+  }
+  :return [ :pick $Input ($Pos + 3) [ :len $Input ] ];
+}
+
 # delay a random amount of seconds
 :set RandomDelay do={
   :local Time [ :tonum $1 ];
@@ -963,6 +953,11 @@
   :local Script [ :tostr $1 ];
 
   :global LogPrint;
+  :global ScriptLock;
+
+  :if ([ $ScriptLock $Script ] = false) do={
+    :return false;
+  }
 
   :foreach Job in=[ /system/script/job/find where script=$Script ] do={
     :set Job [ /system/script/job/get $Job ];
@@ -974,8 +969,8 @@
       :return true;
     }
   }
-  $LogPrint debug $0 ("Script " . $Script . " NOT started from terminal.");
 
+  $LogPrint debug $0 ("Script " . $Script . " NOT started from terminal.");
   :return false;
 }
 
@@ -993,7 +988,7 @@
 
   :global CertificateAvailable;
   :global EitherOr;
-  :global FetchUserAgent;
+  :global FetchUserAgentStr;
   :global Grep;
   :global IfThenElse;
   :global LogPrint;
@@ -1039,7 +1034,7 @@
         :local Url ($BaseUrl . $ScriptVal->"name" . ".rsc" . $UrlSuffix);
         $LogPrint debug $0 ("Fetching script '" . $ScriptVal->"name" . "' from url: " . $Url);
         :local Result [ /tool/fetch check-certificate=yes-without-crl \
-          http-header-field=({ [ $FetchUserAgent $0 ] }) $Url output=user as-value ];
+          http-header-field=({ [ $FetchUserAgentStr $0 ] }) $Url output=user as-value ];
         :if ($Result->"status" = "finished") do={
           :set SourceNew ($Result->"data");
         }
@@ -1122,7 +1117,7 @@
       :local Url ($ScriptUpdatesBaseUrl . "news-and-changes.rsc" . $ScriptUpdatesUrlSuffix);
       $LogPrint debug $0 ("Fetching news, changes and migration: " . $Url);
       :local Result [ /tool/fetch check-certificate=yes-without-crl \
-        http-header-field=({ [ $FetchUserAgent $0 ] }) $Url output=user as-value ];
+        http-header-field=({ [ $FetchUserAgentStr $0 ] }) $Url output=user as-value ];
       :if ($Result->"status" = "finished") do={
         :set ChangeLogCode ($Result->"data");
       }
@@ -1310,20 +1305,23 @@
   $AddTicket $Script $MyTicket;
 
   :local WaitCount 0;
-  :while ($WaitMax > $WaitCount && ([ $IsFirstTicket $Script $MyTicket ] = false || [ $TicketCount $Script ] < [ $JobCount $Script ])) do={
+  :while ($WaitMax > $WaitCount && \
+      ([ $IsFirstTicket $Script $MyTicket ] = false || \
+      [ $TicketCount $Script ] < [ $JobCount $Script ])) do={
     :set WaitCount ($WaitCount + 1);
     :delay 100ms;
   }
 
-  :if ([ $IsFirstTicket $Script $MyTicket ] = true && [ $TicketCount $Script ] = [ $JobCount $Script ]) do={
+  :if ([ $IsFirstTicket $Script $MyTicket ] = true && \
+      [ $TicketCount $Script ] = [ $JobCount $Script ]) do={
     $RemoveTicket $Script $MyTicket;
     $CleanupTickets $Script;
     :return true;
   }
 
   $RemoveTicket $Script $MyTicket;
-  $LogPrint info $0 ("Script '" . $Script . "' started more than once" . [ $IfThenElse ($WaitCount > 0) \
-    " and timed out waiting for lock" "" ] . "...");
+  $LogPrint debug $0 ("Script '" . $Script . "' started more than once" . \
+    [ $IfThenElse ($WaitCount > 0) " and timed out waiting for lock" "" ] . "...");
   :return false;
 }
 
@@ -1331,7 +1329,7 @@
 :set SendNotification do={
   :global SendNotification2;
 
-  $SendNotification2 ({ subject=$1; message=$2; link=$3; silent=$4 });
+  $SendNotification2 ({ origin=$0; subject=$1; message=$2; link=$3; silent=$4 });
 }
 
 # send notification via NotificationFunctions - expects one array argument
@@ -1520,13 +1518,14 @@
 
   :global CleanFilePath;
   :global EitherOr;
+  :global MAX;
 
   :set FileName [ $CleanFilePath $FileName ];
   :local I 1;
-  :local Delay ([ :totime [ $EitherOr $WaitTime 2s ] ] / 20);
+  :local Delay ([ $MAX [ $EitherOr $WaitTime 2s ] 100ms ] / 10);
 
   :while ([ :len [ /file/find where name=$FileName ] ] = 0) do={
-    :if ($I >= 20) do={
+    :if ($I >= 10) do={
       :return false;
     }
     :delay $Delay;