98 lines
2.1 KiB
Nix
98 lines
2.1 KiB
Nix
{config, ...}: let
|
|
inherit (config.services) prometheus;
|
|
nodes = [
|
|
"alpine"
|
|
"100.106.49.128" # laptop
|
|
"100.65.29.110" # gaming pc
|
|
];
|
|
addPort = ip: "${ip}:${toString prometheus.exporters.node.port}";
|
|
in {
|
|
services.prometheus = {
|
|
enable = true;
|
|
scrapeConfigs = [
|
|
{
|
|
job_name = "nodes";
|
|
static_configs = [
|
|
{
|
|
targets = builtins.map addPort nodes;
|
|
}
|
|
];
|
|
}
|
|
{
|
|
job_name = "prometheus";
|
|
static_configs = [
|
|
{
|
|
targets = ["localhost:${toString config.services.prometheus.exporters.postgres.port}"];
|
|
}
|
|
];
|
|
}
|
|
];
|
|
rules = [
|
|
(builtins.toJSON {
|
|
groups = [
|
|
{
|
|
name = "node";
|
|
rules = [
|
|
{
|
|
alert = "io error";
|
|
expr = ''node_filesystem_device_error{device_error!="permission denied"} > 0'';
|
|
}
|
|
{
|
|
alert = "disk full";
|
|
expr = ''node_filesystem_avail_bytes{fstype=~"ext4|btrfs"} < ${toString (50 * 1024 * 1024 * 1024)}'';
|
|
}
|
|
];
|
|
}
|
|
];
|
|
})
|
|
];
|
|
alertmanagers = [
|
|
{
|
|
static_configs = [
|
|
{
|
|
targets = [
|
|
"localhost:9093"
|
|
];
|
|
}
|
|
];
|
|
}
|
|
];
|
|
exporters = {
|
|
postgres = {
|
|
enable = true;
|
|
runAsLocalSuperUser = true;
|
|
};
|
|
};
|
|
alertmanager = {
|
|
enable = false;
|
|
configuration = {
|
|
route = {
|
|
receiver = "alertmanager-ntfy";
|
|
routes = [{
|
|
matchers = [
|
|
''node_filesystem_device_error != 0''
|
|
];
|
|
}];
|
|
};
|
|
};
|
|
};
|
|
alertmanager-ntfy = {
|
|
enable = false;
|
|
settings = {
|
|
ntfy = {
|
|
baseurl = "https://up.tristans.cloud";
|
|
notification = {
|
|
topic = "alert";
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
services.grafana.provision.datasources.settings.datasources = [
|
|
{
|
|
name = "Prometheus";
|
|
type = "prometheus";
|
|
url = "http://localhost:${toString prometheus.port}";
|
|
}
|
|
];
|
|
}
|