Richie
438b4e09e6
- Introduced a new Rust-based service for ingesting HAProxy logs, including the necessary database schema and API endpoints. - Implemented database interactions for managing source status, inserting events, and handling ingestion batches. - Added validation for incoming log batches to ensure data integrity. - Configured systemd service for the HAProxy ingestion server with appropriate security settings. - Updated firewall rules to allow traffic on the ingestion server's port (8444). - Created necessary PostgreSQL database and user configurations for the ingestion service.
103 lines
4.1 KiB
SQL
103 lines
4.1 KiB
SQL
CREATE TABLE IF NOT EXISTS ingest_source_state (
|
|
machine_id TEXT PRIMARY KEY,
|
|
hostname TEXT NOT NULL,
|
|
last_cursor TEXT,
|
|
last_journal_timestamp TIMESTAMPTZ,
|
|
last_source_ip INET,
|
|
last_commit_time TIMESTAMPTZ
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS haproxy_logs (
|
|
id BIGINT GENERATED ALWAYS AS IDENTITY,
|
|
machine_id TEXT NOT NULL,
|
|
hostname TEXT NOT NULL,
|
|
source_ip INET NOT NULL,
|
|
journal_timestamp TIMESTAMPTZ NOT NULL,
|
|
request_time TIMESTAMPTZ,
|
|
client_ip INET,
|
|
client_port INTEGER,
|
|
frontend_name TEXT,
|
|
backend_name TEXT,
|
|
server_name TEXT,
|
|
http_method TEXT,
|
|
request_path TEXT,
|
|
http_version TEXT,
|
|
status_code INTEGER NOT NULL,
|
|
bytes_sent BIGINT NOT NULL,
|
|
time_request_ms INTEGER NOT NULL,
|
|
time_wait_ms INTEGER NOT NULL,
|
|
time_connect_ms INTEGER NOT NULL,
|
|
time_response_ms INTEGER NOT NULL,
|
|
time_total_ms INTEGER NOT NULL,
|
|
termination_state TEXT,
|
|
retries INTEGER NOT NULL,
|
|
tls_sni TEXT,
|
|
tls_version TEXT,
|
|
tls_cipher TEXT,
|
|
tls_resumed BOOLEAN,
|
|
normalization_flags TEXT[] NOT NULL DEFAULT '{}',
|
|
active_connections INTEGER,
|
|
frontend_connections INTEGER,
|
|
backend_connections INTEGER,
|
|
server_connections INTEGER,
|
|
server_queue INTEGER,
|
|
backend_queue INTEGER,
|
|
ingest_time TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
PRIMARY KEY (journal_timestamp, id)
|
|
) PARTITION BY RANGE (journal_timestamp);
|
|
|
|
CREATE INDEX IF NOT EXISTS haproxy_logs_machine_time_idx
|
|
ON haproxy_logs (machine_id, journal_timestamp);
|
|
|
|
CREATE INDEX IF NOT EXISTS haproxy_logs_status_time_idx
|
|
ON haproxy_logs (status_code, journal_timestamp);
|
|
|
|
CREATE TABLE IF NOT EXISTS ingest_rejected_events (
|
|
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
machine_id TEXT NOT NULL,
|
|
hostname TEXT NOT NULL,
|
|
journal_timestamp TIMESTAMPTZ NOT NULL,
|
|
reason TEXT NOT NULL,
|
|
message_sha256 BYTEA NOT NULL,
|
|
raw_message_prefix TEXT,
|
|
raw_message_truncated BOOLEAN NOT NULL,
|
|
source_ip INET NOT NULL,
|
|
ingest_time TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS ingest_rejected_events_machine_time_idx
|
|
ON ingest_rejected_events (machine_id, journal_timestamp);
|
|
|
|
CREATE TABLE IF NOT EXISTS ingest_journal_gaps (
|
|
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
machine_id TEXT NOT NULL,
|
|
hostname TEXT NOT NULL,
|
|
expected_cursor TEXT NOT NULL,
|
|
first_available_cursor TEXT NOT NULL,
|
|
first_available_timestamp TIMESTAMPTZ,
|
|
reason TEXT NOT NULL,
|
|
source_ip INET NOT NULL,
|
|
detected_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS ingest_journal_gaps_machine_time_idx
|
|
ON ingest_journal_gaps (machine_id, detected_at);
|
|
|
|
CREATE TABLE IF NOT EXISTS ingest_batches (
|
|
batch_id TEXT PRIMARY KEY,
|
|
machine_id TEXT NOT NULL,
|
|
hostname TEXT NOT NULL,
|
|
source_ip INET NOT NULL,
|
|
previous_cursor TEXT,
|
|
last_cursor TEXT NOT NULL,
|
|
first_journal_timestamp TIMESTAMPTZ,
|
|
last_journal_timestamp TIMESTAMPTZ,
|
|
record_count INTEGER NOT NULL,
|
|
compressed_bytes BIGINT NOT NULL,
|
|
uncompressed_bytes BIGINT NOT NULL,
|
|
received_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS ingest_batches_machine_time_idx
|
|
ON ingest_batches (machine_id, received_at);
|