nixos-config/machines/chungus/telemetry/loki.nix

{ config, pkgs, ... }:
{

  services.opentelemetry-collector.settings = {
    exporters.loki = {
      endpoint = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
      default_labels_enabled = {
        exporter = true;
        job = true;
        instance = true;
        level = true;
      };
    };
    processors = {
      attributes.actions = [
        {
          action = "insert";
          key = "loki.attribute.labels";
          value = "job, unit, boot_id, instance, facility, facility_label, priority, priority_label";
        }
      ];
      resource.attributes = [
        {
          action = "insert";
          key = "loki.resource.labels";
          value = "host.name";
        }
        {
          action = "insert";
          key = "loki.format";
          value = "raw";
        }
      ];
    };
    service.pipelines.logs.exporters = [ "loki" ];
    service.pipelines.logs.processors = [ "resource" "attributes" ];
  };

  services.loki = {
    enable = true;
    # https://grafana.com/docs/loki/latest/configure/#supported-contents-and-default-values-of-lokiyaml
    configuration = {

      server = {
        http_listen_port = 3100;
        log_level = "warn";
      };
      auth_enabled = false;

      ingester = {
        lifecycler = {
          address = "127.0.0.1";
          ring = {
            kvstore = {
              store = "inmemory";
            };
            replication_factor = 1;
          };
        };
        chunk_idle_period = "1h";
        max_chunk_age = "1h";
        chunk_target_size = 999999;
        chunk_retain_period = "30s";
      };

      schema_config = {
        configs = [{
          from = "2024-05-28";
          store = "tsdb";
          object_store = "filesystem";
          schema = "v13";
          index = {
            prefix = "index_";
            period = "24h";
          };
        }];
      };

      storage_config = {
        tsdb_shipper = {
          active_index_directory = "/var/lib/loki/tsdb-shipper-active";
          cache_location = "/var/lib/loki/tsdb-shipper-cache";
          cache_ttl = "24h";
        };

        filesystem = {
          directory = "/var/lib/loki/chunks";
        };
      };

      limits_config = {
        reject_old_samples = true;
        reject_old_samples_max_age = "168h";
      };

      table_manager = {
        retention_deletes_enabled = false;
        retention_period = "0s";
      };

      compactor = {
        working_directory = "/var/lib/loki";
        compactor_ring = {
          kvstore = {
            store = "inmemory";
          };
        };
      };

      # The query_range block configures the query splitting and caching in the Loki query-frontend.
      query_range = {
        # Perform query parallelisations based on storage sharding configuration and
        # query ASTs. This feature is supported only by the chunks storage engine.
        parallelise_shardable_queries = false; # false because of https://github.com/grafana/loki/issues/7649#issuecomment-1625645403
      };
    };

    # user, group, dataDir, extraFlags, (configFile)
  };

  # https://grafana.com/docs/grafana/latest/datasources/loki/#provision-the-loki-data-source
  services.grafana.provision.datasources.settings = {
    apiVersion = 1;
    datasources = [
      {
        name = "Loki";
        type = "loki";
        uid = "loki01";
        url = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
        jsonData = {
          timeout = 360;
          maxLines = 1000;
        };
      }
    ];
  };

}