big refactor

nixos/services/default.nix

@@ -0,0 +1,7 @@
+{ ... }:
+{
+  imports = [
+    ./promtail.nix
+    ./telegraf.nix
+  ];
+}

nixos/services/promtail.nix

@@ -0,0 +1,86 @@
+{ config, pkgs, ... }:
+let
+  lokiHost = "192.168.1.174:3100";
+in
+{
+  systemd.services.promtail.serviceConfig = {
+    SupplementaryGroups = [ "docker" ];
+  };
+
+  environment.systemPackages = with pkgs; [
+    (pkgs.writeShellScriptBin "pc" "systemctl status promtail.service")
+    (pkgs.writeShellScriptBin "pw" "journalctl -u promtail.service -b -n 25 -f")
+  ];
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 3031;
+        grpc_listen_port = 0;
+      };
+      positions = {
+        filename = "/tmp/positions.yaml";
+      };
+      clients = [{url = "http://${lokiHost}/loki/api/v1/push";}];
+      scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            max_age = "24h";
+            path = "/var/log/journal";
+            json = true;
+            # matches: _TRANSPORT=kernel;
+            labels = {
+              job = "systemd-journal";
+              host = config.networking.hostName; # Dynamically fetch the hostname
+            };
+          };
+          relabel_configs = [
+            {
+              source_labels = [ "__journal__systemd_unit" ];
+              target_label = "unit";
+            }
+          ];
+        }
+        {
+          job_name = "flog_scrape";
+          docker_sd_configs = [
+            {
+              host = "unix:///var/run/docker.sock";
+              refresh_interval = "5s";
+            }
+          ];
+          relabel_configs = [
+            {
+              source_labels = [ "__meta_docker_container_name" ];
+              regex = "/(.*)";
+              target_label = "container";
+            }
+            {
+              source_labels = [ "__meta_docker_container_label_com_docker_compose_oneoff" ];
+              target_label = "oneoff";
+            }
+            {
+              source_labels = [ "__meta_docker_container_label_com_docker_compose_project_config_files" ];
+              target_label = "compose_file";
+            }
+            {
+              source_labels = [ "__meta_docker_container_label_com_docker_compose_project" ];
+              target_label = "project_name";
+            }
+            {
+              source_labels = [ "__meta_docker_container_label_com_docker_compose_service" ];
+              target_label = "service";
+            }
+            {
+              target_label = "host";
+              replacement = "${config.networking.hostName}";
+            }
+          ];
+        }
+      ];
+    };
+    # extraFlags
+  };
+}

nixos/services/telegraf.nix

@@ -0,0 +1,60 @@
+{ config, pkgs, ... }:
+let
+  influxURL = "http://panoptes.john-stream.com:8086";
+  organization = "homelab";
+  bucket = "docker";
+  token = "${builtins.readFile config.sops.secrets."telegraf_influx_token".path}";
+in
+{
+  sops.secrets."telegraf_influx_token" = { };
+
+  environment.systemPackages = with pkgs; [
+    (pkgs.writeShellScriptBin "tc" "systemctl status telegraf.service")
+    (pkgs.writeShellScriptBin "tw" "journalctl -u telegraf.service -b -n 25 -f")
+  ];
+
+  systemd.services.telegraf = {
+    environment = {
+      INFLUX_WRITE_TOKEN = token;
+    };
+    serviceConfig.SupplementaryGroups = [ "docker" ];
+  };
+
+  services.telegraf = {
+    enable = true;
+    extraConfig = {
+      agent = {
+        interval = "10s";
+        round_interval = true;
+        metric_batch_size = 1000;
+        metric_buffer_limit = 10000;
+        collection_jitter = "0s";
+        flush_interval = "10s";
+        flush_jitter = "0s";
+        precision = "";
+        hostname = "";
+        omit_hostname = false;
+      };
+      inputs = {
+        docker = {
+          endpoint = "unix:///var/run/docker.sock";
+          gather_services = false;
+          source_tag = false;
+          container_name_include = [];
+          timeout = "5s";
+          perdevice_include = ["cpu" "blkio" "network"];
+          total = false;
+          docker_label_include = [];
+        };
+      };
+      outputs = {
+        influxdb_v2 = {
+          urls = ["${influxURL}"];
+          token = "$INFLUX_WRITE_TOKEN";
+          organization = "${organization}";
+          bucket = "${bucket}";
+        };
+      };
+    };
+  };
+}