nixos-config/services/alertmanager.nix

{ config, lib, my, pkgs, secrets, ... }:

let
  toConfigFile = name: cfg: pkgs.writeText name (lib.generators.toYAML { } cfg);
  blackboxExporterCfg = config.services.prometheus.exporters.blackbox;
in {
  sops.secrets."alertmanager/env" = {
    sopsFile = ../secrets/sops/alertmanager.yaml;
    restartUnits = [ "alertmanager.service" ];
  };

  services.prometheus = {
    enable = true;
    checkConfig = "syntax-only";
    listenAddress = "[::1]";
    scrapeConfigs = [
      {
        job_name = "blackbox-exporter-https-simple";
        scrape_interval = "30s";
        metrics_path = "/probe";
        params.module = [ "https_nocookies_hsts" ];
        static_configs =
          [{ targets = map (x: "https://${x}") my.monitoring.targets.https; }];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__address__" ];
            target_label = "__param_hostname";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "[::1]:${toString blackboxExporterCfg.port}";
          }
        ];
      }
      {
        job_name = "blackbox-exporter-https-redirect";
        scrape_interval = "30s";
        metrics_path = "/probe";
        params.module = [ "http_redirect" ];
        static_configs =
          [{ targets = map (x: "http://${x}") my.monitoring.targets.https; }];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__address__" ];
            target_label = "__param_hostname";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "[::1]:${toString blackboxExporterCfg.port}";
          }
        ];
      }
      {
        job_name = "blackbox-exporter-icmp";
        scrape_interval = "10s";
        metrics_path = "/probe";
        params.module = [ "ping" ];
        static_configs = [{ targets = my.monitoring.targets.icmp; }];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "[::1]:${toString blackboxExporterCfg.port}";
          }
        ];
      }
      {
        job_name = "blackbox-exporter-icmp-airlab";
        scrape_interval = "10s";
        metrics_path = "/probe";
        params.module = [ "ping" ];
        static_configs = [{ targets = my.monitoring.targets.airlab.icmp; }];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "[::1]:${toString blackboxExporterCfg.port}";
          }
        ];
      }
      # collect blackbox-exporter metrics of itself
      {
        job_name = "blackbox-exporter";
        scrape_interval = "60s";
        static_configs =
          [{ targets = [ "[::1]:${toString blackboxExporterCfg.port}" ]; }];
      }
    ];
    alertmanagers = [{
      scheme = "https";
      # TODO: authentication/authorization
      static_configs = [{ targets = [ "[::1]" ]; }];
    }];
  };

  services.prometheus.alertmanager = {
    enable = true;
    listenAddress = "[::1]";
    logLevel = "info";
    webExternalUrl = "https://alertmanager.${my.domain}";
    environmentFile = secrets."alertmanager/env".path;
    checkConfig = false;
    configuration = {
      route = {
        receiver = "matrix-alertmanager"; # default receiver
        group_by = [ "alertname" "cluster" "service" ];
        routes = [{
          matchers = [ ''job =~ ".*-airlab"'' ];
          receiver = "matrix-alertmanager-airlab";
        }];
      };
      inhibit_rules = [{
        # inhibit warnings if there also is a critical notification for the same
        # alert
        source_matchers = [ ''severity="critical"'' ];
        target_matchers = [ ''severity="warning"'' ];
        equal = [ "alertname" "cluster" "service" ];
      }];
      receivers = [
        {
          name = "matrix-alertmanager";
          webhook_configs = [{
            # will be replaced by `envsubst`
            url_file = "$MATRIX_ALERTMANAGER_WEBHOOK_URL";
          }];
        }
        {
          name = "matrix-alertmanager-airlab";
          webhook_configs = [{
            # will be replaced by `envsubst`
            url_file = "$MATRIX_ALERTMANAGER_AIRLAB_WEBHOOK_URL";
          }];
        }
      ];
    };
  };

  services.prometheus.exporters.blackbox = {
    enable = true;
    listenAddress = "[::1]";
    configFile = toConfigFile "blackbox-exporter-config.yaml" {
      modules = {
        http_redirect = {
          prober = "http";
          timeout = "5s";
          http = {
            valid_http_versions = [ "HTTP/1.1" "HTTP/2.0" ];
            valid_status_codes = [ 301 308 ];
            follow_redirects = true;
            fail_if_ssl = true;
            fail_if_not_ssl = false;
          };
        };
        https_nocookies_hsts = {
          prober = "http";
          timeout = "5s";
          http = {
            valid_http_versions = [ "HTTP/1.1" "HTTP/2.0" ];
            valid_status_codes = [ 200 ];
            follow_redirects = true;
            fail_if_ssl = false;
            fail_if_not_ssl = true;
            fail_if_header_matches = [
              # Verifies that no cookies are set
              {
                header = "Set-Cookie";
                allow_missing = true;
                regexp = ".*";
              }
            ];
            fail_if_header_not_matches = [{
              header = "Strict-Transport-Security";
              regexp = "max-age=31536000; includeSubDomains";
            }];
            tls_config.insecure_skip_verify = false;
          };
        };
        ping = {
          prober = "icmp";
          timeout = "5s";
        };
      };
    };
  };
}