feat: add HAProxy log ingestion server

- Introduced a new Rust-based service for ingesting HAProxy logs, including the necessary database schema and API endpoints.
- Implemented database interactions for managing source status, inserting events, and handling ingestion batches.
- Added validation for incoming log batches to ensure data integrity.
- Configured systemd service for the HAProxy ingestion server with appropriate security settings.
- Updated firewall rules to allow traffic on the ingestion server's port (8444).
- Created necessary PostgreSQL database and user configurations for the ingestion service.

rust/haproxy_ingest_server/migrations/001_initial.sql

@@ -0,0 +1,102 @@
+CREATE TABLE IF NOT EXISTS ingest_source_state (
+    machine_id                TEXT PRIMARY KEY,
+    hostname                  TEXT NOT NULL,
+    last_cursor               TEXT,
+    last_journal_timestamp    TIMESTAMPTZ,
+    last_source_ip            INET,
+    last_commit_time          TIMESTAMPTZ
+);
+
+CREATE TABLE IF NOT EXISTS haproxy_logs (
+    id                        BIGINT GENERATED ALWAYS AS IDENTITY,
+    machine_id                TEXT NOT NULL,
+    hostname                  TEXT NOT NULL,
+    source_ip                 INET NOT NULL,
+    journal_timestamp         TIMESTAMPTZ NOT NULL,
+    request_time              TIMESTAMPTZ,
+    client_ip                 INET,
+    client_port               INTEGER,
+    frontend_name             TEXT,
+    backend_name              TEXT,
+    server_name               TEXT,
+    http_method               TEXT,
+    request_path              TEXT,
+    http_version              TEXT,
+    status_code               INTEGER NOT NULL,
+    bytes_sent                BIGINT NOT NULL,
+    time_request_ms           INTEGER NOT NULL,
+    time_wait_ms              INTEGER NOT NULL,
+    time_connect_ms           INTEGER NOT NULL,
+    time_response_ms          INTEGER NOT NULL,
+    time_total_ms             INTEGER NOT NULL,
+    termination_state         TEXT,
+    retries                   INTEGER NOT NULL,
+    tls_sni                   TEXT,
+    tls_version               TEXT,
+    tls_cipher                TEXT,
+    tls_resumed               BOOLEAN,
+    normalization_flags       TEXT[] NOT NULL DEFAULT '{}',
+    active_connections        INTEGER,
+    frontend_connections      INTEGER,
+    backend_connections       INTEGER,
+    server_connections        INTEGER,
+    server_queue              INTEGER,
+    backend_queue             INTEGER,
+    ingest_time               TIMESTAMPTZ NOT NULL DEFAULT now(),
+    PRIMARY KEY (journal_timestamp, id)
+) PARTITION BY RANGE (journal_timestamp);
+
+CREATE INDEX IF NOT EXISTS haproxy_logs_machine_time_idx
+ON haproxy_logs (machine_id, journal_timestamp);
+
+CREATE INDEX IF NOT EXISTS haproxy_logs_status_time_idx
+ON haproxy_logs (status_code, journal_timestamp);
+
+CREATE TABLE IF NOT EXISTS ingest_rejected_events (
+    id                        BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    machine_id                TEXT NOT NULL,
+    hostname                  TEXT NOT NULL,
+    journal_timestamp         TIMESTAMPTZ NOT NULL,
+    reason                    TEXT NOT NULL,
+    message_sha256            BYTEA NOT NULL,
+    raw_message_prefix        TEXT,
+    raw_message_truncated     BOOLEAN NOT NULL,
+    source_ip                 INET NOT NULL,
+    ingest_time               TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS ingest_rejected_events_machine_time_idx
+ON ingest_rejected_events (machine_id, journal_timestamp);
+
+CREATE TABLE IF NOT EXISTS ingest_journal_gaps (
+    id                         BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    machine_id                 TEXT NOT NULL,
+    hostname                   TEXT NOT NULL,
+    expected_cursor            TEXT NOT NULL,
+    first_available_cursor     TEXT NOT NULL,
+    first_available_timestamp  TIMESTAMPTZ,
+    reason                     TEXT NOT NULL,
+    source_ip                  INET NOT NULL,
+    detected_at                TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS ingest_journal_gaps_machine_time_idx
+ON ingest_journal_gaps (machine_id, detected_at);
+
+CREATE TABLE IF NOT EXISTS ingest_batches (
+    batch_id                   TEXT PRIMARY KEY,
+    machine_id                 TEXT NOT NULL,
+    hostname                   TEXT NOT NULL,
+    source_ip                  INET NOT NULL,
+    previous_cursor            TEXT,
+    last_cursor                TEXT NOT NULL,
+    first_journal_timestamp    TIMESTAMPTZ,
+    last_journal_timestamp     TIMESTAMPTZ,
+    record_count               INTEGER NOT NULL,
+    compressed_bytes           BIGINT NOT NULL,
+    uncompressed_bytes         BIGINT NOT NULL,
+    received_at                TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS ingest_batches_machine_time_idx
+ON ingest_batches (machine_id, received_at);