{config, ...}: let
  inherit (config.services) prometheus;
  nodes = [
    "alpine"
    "100.106.49.128" # laptop
    "100.65.29.110" # gaming pc
  ];
  addPort = ip: "${ip}:${toString prometheus.exporters.node.port}";
in {
  services.prometheus = {
    enable = true;
    scrapeConfigs = [
      {
        job_name = "nodes";
        static_configs = [
          {
            targets = builtins.map addPort nodes;
          }
        ];
      }
      {
        job_name = "prometheus";
        static_configs = [
          {
            targets = ["localhost:${toString config.services.prometheus.exporters.postgres.port}"];
          }
        ];
      }
    ];
    rules = [
      (builtins.toJSON {
        groups = [{
          name = "node";
          rules = [
            {
              alert = "io error";
              expr = ''node_filesystem_device_error{device_error!="permission denied"} > 0'';
            }
            {
              alert = "disk full";
              expr = ''node_filesystem_avail_bytes{fstype=~"ext4|btrfs"} < ${toString (50 * 1024 * 1024 * 1024)}'';
            }
          ];
        }];
      })
    ];
    alertmanagers = [ {
      static_configs = [ {
        targets = [
          "localhost:9093"
        ];
      } ];
    } ];
    exporters = {
      postgres = {
        enable = true;
        runAsLocalSuperUser = true;
      };
    };
    alertmanager = {
      enable = true;
      configuration = {
        receivers = [{
          name = "ntfy";
          webhook_configs = [{
            url = "http://localhost${config.services.ntfy-sh.settings.listen-http}/alert/trigger";
          }];
        }];
        route = {
          receiver = "ntfy";
          # routes = [{
          #   matchers = [
          #     ''node_filesystem_device_error != 0''
          #   ];
          # }];
        };
      };
    };
    # alertmanager-ntfy = {
    #   enable = true;
    #   settings = {
    #     ntfy = {
    #       baseurl = "https://up.tristans.cloud";
    #       notification = {
    #         topic = "alert";
    #       };
    #     };
    #   };
    # };
  };
  services.grafana.provision.datasources.settings.datasources = [{
    name = "Prometheus";
    type = "prometheus";
    url = "http://localhost:${toString prometheus.port}";
  }];
}