alpine: tweaks and fixes

nixos/services/prometheus.nix

@@ -2,7 +2,7 @@
   inherit (config.services) prometheus;
   nodes = [
     "alpine"
-    "fcs-tristan-nixbook"
+    "framework-13"
     "zenix"
   ];
   addPort = ip: "${ip}:${toString prometheus.exporters.node.port}";
@@ -27,12 +27,66 @@ in {
         ];
       }
     ];
+    rules = [
+      (builtins.toJSON {
+        groups = [{
+          name = "node";
+          rules = [
+            {
+              alert = "io error";
+              expr = ''node_filesystem_device_error{device_error!="permission denied"} > 0'';
+            }
+            {
+              alert = "disk full";
+              expr = ''node_filesystem_avail_bytes{fstype=~"ext4|btrfs"} < ${toString (50 * 1024 * 1024 * 1024)}'';
+            }
+          ];
+        }];
+      })
+    ];
+    alertmanagers = [ {
+      static_configs = [ {
+        targets = [
+          "localhost:9093"
+        ];
+      } ];
+    } ];
     exporters = {
       postgres = {
         enable = true;
         runAsLocalSuperUser = true;
       };
     };
+    alertmanager = {
+      enable = true;
+      configuration = {
+        receivers = [{
+          name = "ntfy";
+          webhook_configs = [{
+            url = "http://localhost${config.services.ntfy-sh.settings.listen-http}/alert/trigger";
+          }];
+        }];
+        route = {
+          receiver = "ntfy";
+          # routes = [{
+          #   matchers = [
+          #     ''node_filesystem_device_error != 0''
+          #   ];
+          # }];
+        };
+      };
+    };
+    # alertmanager-ntfy = {
+    #   enable = true;
+    #   settings = {
+    #     ntfy = {
+    #       baseurl = "https://up.tristans.cloud";
+    #       notification = {
+    #         topic = "alertmanager";
+    #       };
+    #     };
+    #   };
+    # };
   };
   services.grafana.provision.datasources.settings.datasources = [{
     name = "Prometheus";