{config, ...}: let
  inherit (config.services) prometheus;
  nodes = [
    "alpine"
    "100.106.49.128" # laptop
    "100.65.29.110" # gaming pc
  ];
  addPort = ip: "${ip}:${toString prometheus.exporters.node.port}";
in {
  services.prometheus = {
    enable = true;
    scrapeConfigs = [
      {
        job_name = "nodes";
        static_configs = [
          {
            targets = builtins.map addPort nodes;
          }
        ];
      }
      {
        job_name = "prometheus";
        static_configs = [
          {
            targets = ["localhost:${toString config.services.prometheus.exporters.postgres.port}"];
          }
        ];
      }
    ];
    rules = [
      (builtins.toJSON {
        groups = [
          {
            name = "node";
            rules = [
              {
                alert = "io error";
                expr = ''node_filesystem_device_error{device_error!="permission denied"} > 0'';
              }
              {
                alert = "disk full";
                expr = ''node_filesystem_avail_bytes{fstype=~"ext4|btrfs"} < ${toString (50 * 1024 * 1024 * 1024)}'';
              }
            ];
          }
        ];
      })
    ];
    alertmanagers = [
      {
        static_configs = [
          {
            targets = [
              "localhost:9093"
            ];
          }
        ];
      }
    ];
    exporters = {
      postgres = {
        enable = true;
        runAsLocalSuperUser = true;
      };
    };
    alertmanager = {
      enable = false;
      configuration = {
        route = {
          receiver = "alertmanager-ntfy";
          routes = [{
            matchers = [
              ''node_filesystem_device_error != 0''
            ];
          }];
        };
      };
    };
    alertmanager-ntfy = {
      enable = false;
      settings = {
        ntfy = {
          baseurl = "https://up.tristans.cloud";
          notification = {
            topic = "alert";
          };
        };
      };
    };
  };
  services.grafana.provision.datasources.settings.datasources = [
    {
      name = "Prometheus";
      type = "prometheus";
      url = "http://localhost:${toString prometheus.port}";
    }
  ];
}