96 lines
2.2 KiB
Nix
96 lines
2.2 KiB
Nix
{config, ...}: let
|
|
inherit (config.services) prometheus;
|
|
nodes = [
|
|
"alpine"
|
|
"framework-13"
|
|
"zenix"
|
|
];
|
|
addPort = ip: "${ip}:${toString prometheus.exporters.node.port}";
|
|
in {
|
|
services.prometheus = {
|
|
enable = true;
|
|
scrapeConfigs = [
|
|
{
|
|
job_name = "nodes";
|
|
static_configs = [
|
|
{
|
|
targets = builtins.map addPort nodes;
|
|
}
|
|
];
|
|
}
|
|
{
|
|
job_name = "prometheus";
|
|
static_configs = [
|
|
{
|
|
targets = ["localhost:${toString config.services.prometheus.exporters.postgres.port}"];
|
|
}
|
|
];
|
|
}
|
|
];
|
|
rules = [
|
|
(builtins.toJSON {
|
|
groups = [{
|
|
name = "node";
|
|
rules = [
|
|
{
|
|
alert = "io error";
|
|
expr = ''node_filesystem_device_error{device_error!="permission denied"} > 0'';
|
|
}
|
|
{
|
|
alert = "disk full";
|
|
expr = ''node_filesystem_avail_bytes{fstype=~"ext4|btrfs"} < ${toString (50 * 1024 * 1024 * 1024)}'';
|
|
}
|
|
];
|
|
}];
|
|
})
|
|
];
|
|
alertmanagers = [ {
|
|
static_configs = [ {
|
|
targets = [
|
|
"localhost:9093"
|
|
];
|
|
} ];
|
|
} ];
|
|
exporters = {
|
|
postgres = {
|
|
enable = true;
|
|
runAsLocalSuperUser = true;
|
|
};
|
|
};
|
|
alertmanager = {
|
|
enable = true;
|
|
configuration = {
|
|
receivers = [{
|
|
name = "ntfy";
|
|
webhook_configs = [{
|
|
url = "http://localhost${config.services.ntfy-sh.settings.listen-http}/alert/trigger";
|
|
}];
|
|
}];
|
|
route = {
|
|
receiver = "ntfy";
|
|
# routes = [{
|
|
# matchers = [
|
|
# ''node_filesystem_device_error != 0''
|
|
# ];
|
|
# }];
|
|
};
|
|
};
|
|
};
|
|
# alertmanager-ntfy = {
|
|
# enable = true;
|
|
# settings = {
|
|
# ntfy = {
|
|
# baseurl = "https://up.tristans.cloud";
|
|
# notification = {
|
|
# topic = "alertmanager";
|
|
# };
|
|
# };
|
|
# };
|
|
# };
|
|
};
|
|
services.grafana.provision.datasources.settings.datasources = [{
|
|
name = "Prometheus";
|
|
type = "prometheus";
|
|
url = "http://localhost:${toString prometheus.port}";
|
|
}];
|
|
}
|