refactor(jeeves): remove startup validation

python/system_tests/__init__.py

@@ -1 +0,0 @@
-"""system_tests."""

python/system_tests/components.py

@@ -1,99 +0,0 @@
-"""Validate Jeeves."""
-
-from __future__ import annotations
-
-import logging
-from copy import copy
-from re import search
-from time import sleep
-from typing import TYPE_CHECKING
-
-from python.common import bash_wrapper
-from python.zfs import Zpool
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-logger = logging.getLogger(__name__)
-
-
-def zpool_tests(pool_names: Sequence[str], zpool_capacity_threshold: int = 90) -> list[str] | None:
-    """Check the zpool health and capacity.
-
-    Args:
-        pool_names (Sequence[str]): A list of pool names to test.
-        zpool_capacity_threshold (int, optional): The threshold for the zpool capacity. Defaults to 90.
-
-    Returns:
-        list[str] | None: A list of errors if any.
-    """
-    logger.info("Testing zpool")
-
-    errors: list[str] = []
-    for pool_name in pool_names:
-        pool = Zpool(pool_name)
-        if pool.health != "ONLINE":
-            errors.append(f"{pool.name} is {pool.health}")
-        if pool.capacity >= zpool_capacity_threshold:
-            errors.append(f"{pool.name} is low on space")
-
-    upgrade_status, _ = bash_wrapper("zpool upgrade")
-    if not search(r"Every feature flags pool has all supported and requested features enabled.", upgrade_status):
-        errors.append("ZPool out of date run `sudo zpool upgrade -a`")
-
-    return errors
-
-
-def systemd_tests(
-    service_names: Sequence[str],
-    max_retries: int = 30,
-    retry_delay_secs: int = 1,
-    retryable_statuses: Sequence[str] | None = None,
-    valid_statuses: Sequence[str] | None = None,
-) -> list[str] | None:
-    """Tests a systemd services.
-
-    Args:
-        service_names (Sequence[str]): A list of service names to test.
-        max_retries (int, optional): The maximum number of retries. Defaults to 30.
-            minimum value is 1.
-        retry_delay_secs (int, optional): The delay between retries in seconds. Defaults to 1.
-            minimum value is 1.
-        retryable_statuses (Sequence[str] | None, optional): A list of retryable statuses. Defaults to None.
-        valid_statuses (Sequence[str] | None, optional): A list of valid statuses. Defaults to None.
-
-    Returns:
-        list[str] | None: A list of errors if any.
-    """
-    logger.info("Testing systemd service")
-
-    max_retries = max(max_retries, 1)
-    retry_delay_secs = max(retry_delay_secs, 1)
-    last_try = max_retries - 1
-
-    if retryable_statuses is None:
-        retryable_statuses = ("inactive\n", "activating\n")
-
-    if valid_statuses is None:
-        valid_statuses = ("active\n",)
-
-    service_names_set = set(service_names)
-
-    errors: set[str] = set()
-    for retry in range(max_retries):
-        if not service_names_set:
-            break
-        logger.info(f"Testing systemd service in {retry + 1} of {max_retries}")
-        service_names_to_test = copy(service_names_set)
-        for service_name in service_names_to_test:
-            service_status, _ = bash_wrapper(f"systemctl is-active {service_name}")
-            if service_status in valid_statuses:
-                service_names_set.remove(service_name)
-                continue
-            if service_status in retryable_statuses and retry < last_try:
-                continue
-            errors.add(f"{service_name} is {service_status.strip()}")
-
-        sleep(retry_delay_secs)
-
-    return list(errors)

python/system_tests/validate_system.py

@@ -1,67 +0,0 @@
-"""Validate {server_name}."""
-
-import logging
-import sys
-import tomllib
-from os import environ
-from pathlib import Path  # noqa: TC003 This is required for the typer CLI
-from socket import gethostname
-
-import typer
-
-from python.common import configure_logger
-from python.signal_alert import signal_alert
-from python.system_tests.components import systemd_tests, zpool_tests
-
-logger = logging.getLogger(__name__)
-
-
-def load_config_data(config_file: Path) -> dict[str, list[str]]:
-    """Load a TOML configuration file.
-
-    Args:
-        config_file (Path): The path to the configuration file.
-
-    Returns:
-        dict: The configuration data.
-    """
-    return tomllib.loads(config_file.read_text())
-
-
-def main(config_file: Path) -> None:
-    """Main."""
-    configure_logger(level=environ.get("LOG_LEVEL", "INFO"))
-
-    server_name = gethostname()
-    logger.info(f"Starting {server_name} validation")
-
-    config_data = load_config_data(config_file)
-
-    errors: list[str] = []
-    try:
-        if config_data.get("zpools") and (zpool_errors := zpool_tests(config_data["zpools"])):
-            errors.extend(zpool_errors)
-
-        if config_data.get("services") and (systemd_errors := systemd_tests(config_data["services"])):
-            errors.extend(systemd_errors)
-
-    except Exception as error:
-        logger.exception(f"{server_name} validation failed")
-        errors.append(f"{server_name} validation failed: {error}")
-
-    if errors:
-        logger.error(f"{server_name} validation failed: \n{'\n'.join(errors)}")
-        signal_alert(f"{server_name} validation failed {errors}")
-
-        sys.exit(1)
-
-    logger.info(f"{server_name} validation passed")
-
-
-def cli() -> None:
-    """CLI."""
-    typer.run(main)
-
-
-if __name__ == "__main__":
-    cli()

systems/jeeves/services/systemd.nix

@@ -1,11 +1,7 @@
 {
   pkgs,
-  inputs,
   ...
 }:
-let
-  vars = import ../vars.nix;
-in
 {
   systemd = {
     services = {
@@ -30,21 +26,6 @@ in
           ExecStart = "${pkgs.bash}/bin/bash -c 'echo 1 > /sys/bus/pci/devices/0000:61:00.0/remove'";
         };
       };
-      startup_validation = {
-        requires = [ "network-online.target" ];
-        after = [ "network-online.target" ];
-        wantedBy = [ "multi-user.target" ];
-        description = "validates startup";
-        path = [ pkgs.zfs ];
-        environment = {
-          PYTHONPATH = "${inputs.self}/";
-        };
-        serviceConfig = {
-          EnvironmentFile = "${vars.secrets}/services/server-validation";
-          Type = "oneshot";
-          ExecStart = "${pkgs.my_python}/bin/python -m python.system_tests.validate_system '${./validate_system.toml}'";
-        };
-      };
     };
     timers = {
       plex_permission = {
@@ -55,13 +36,6 @@ in
           Unit = "plex_permission.service";
         };
       };
-      startup_validation = {
-        wantedBy = [ "timers.target" ];
-        timerConfig = {
-          OnBootSec = "10min";
-          Unit = "startup_validation.service";
-        };
-      };
     };
   };
 }

systems/jeeves/services/validate_system.toml

@@ -1,6 +0,0 @@
-zpool = ["root_pool", "storage", "media"]
-services = [
-    "audiobookshelf",
-    "docker",
-    "jellyfin",
-]

tests/test_components.py

@@ -1,104 +0,0 @@
-"""test_components."""
-
-from pytest_mock import MockerFixture
-
-from python.system_tests.components import systemd_tests, zpool_tests
-from python.zfs import Zpool
-
-temp = "Every feature flags pool has all supported and requested features enabled.\n"
-
-SYSTEM_TESTS_COMPONENTS = "python.system_tests.components"
-
-
-def test_zpool_tests(mocker: MockerFixture) -> None:
-    """test_zpool_tests."""
-    mock_zpool = mocker.MagicMock(spec=Zpool)
-    mock_zpool.health = "ONLINE"
-    mock_zpool.capacity = 70
-    mock_zpool.name = "Main"
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
-    errors = zpool_tests(("Main",))
-    assert errors == []
-
-
-def test_zpool_tests_out_of_date(mocker: MockerFixture) -> None:
-    """test_zpool_tests_out_of_date."""
-    mock_zpool = mocker.MagicMock(spec=Zpool)
-    mock_zpool.health = "ONLINE"
-    mock_zpool.capacity = 70
-    mock_zpool.name = "Main"
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("", ""))
-    errors = zpool_tests(("Main",))
-    assert errors == ["ZPool out of date run `sudo zpool upgrade -a`"]
-
-
-def test_zpool_tests_out_of_space(mocker: MockerFixture) -> None:
-    """test_zpool_tests_out_of_space."""
-    mock_zpool = mocker.MagicMock(spec=Zpool)
-    mock_zpool.health = "ONLINE"
-    mock_zpool.capacity = 100
-    mock_zpool.name = "Main"
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
-    errors = zpool_tests(("Main",))
-    assert errors == ["Main is low on space"]
-
-
-def test_zpool_tests_offline(mocker: MockerFixture) -> None:
-    """test_zpool_tests_offline."""
-    mock_zpool = mocker.MagicMock(spec=Zpool)
-    mock_zpool.health = "OFFLINE"
-    mock_zpool.capacity = 70
-    mock_zpool.name = "Main"
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool)
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, ""))
-    errors = zpool_tests(("Main",))
-    assert errors == ["Main is OFFLINE"]
-
-
-def test_systemd_tests(mocker: MockerFixture) -> None:
-    """test_systemd_tests."""
-    mocker.patch(
-        f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper",
-        side_effect=[
-            ("inactive\n", ""),
-            ("active\n", ""),
-        ],
-    )
-    errors = systemd_tests(("docker",))
-    assert errors == []
-    """test_systemd_tests."""
-
-
-def test_systemd_tests_multiple_negative_retries(mocker: MockerFixture) -> None:
-    """test_systemd_tests_fail."""
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("active\n", ""))
-    errors = systemd_tests(("docker",), max_retries=-1, retry_delay_secs=-1)
-    assert errors == []
-
-
-def test_systemd_tests_multiple_pass(mocker: MockerFixture) -> None:
-    """test_systemd_tests_fail."""
-    mocker.patch(
-        f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper",
-        side_effect=[
-            ("inactive\n", ""),
-            ("activating\n", ""),
-            ("active\n", ""),
-        ],
-    )
-    errors = systemd_tests(
-        ("docker",),
-        retryable_statuses=("inactive\n", "activating\n"),
-        valid_statuses=("active\n",),
-    )
-    assert errors == []
-
-
-def test_systemd_tests_fail(mocker: MockerFixture) -> None:
-    """test_systemd_tests_fail."""
-    mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("inactive\n", ""))
-    errors = systemd_tests(("docker",), max_retries=5)
-    assert errors == ["docker is inactive"]

tests/test_server_validate_scripts.py

@@ -1,63 +0,0 @@
-"""test_server_validate_scripts."""
-
-from __future__ import annotations
-
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-import pytest
-from pytest_mock import MockerFixture
-
-from python.system_tests.validate_system import main
-
-if TYPE_CHECKING:
-    from pyfakefs.fake_filesystem import FakeFilesystem
-    from pytest_mock import MockerFixture
-
-VALIDATE_SYSTEM = "python.system_tests.validate_system"
-
-
-def test_validate_system(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """test_validate_system."""
-    fs.create_file(
-        "/mock_snapshot_config.toml",
-        contents='zpools = ["root_pool", "storage", "media"]\nservices = ["docker"]\n',
-    )
-
-    mocker.patch(f"{VALIDATE_SYSTEM}.systemd_tests", return_value=None)
-    mocker.patch(f"{VALIDATE_SYSTEM}.zpool_tests", return_value=None)
-    main(Path("/mock_snapshot_config.toml"))
-
-
-def test_validate_system_errors(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """test_validate_system_errors."""
-    fs.create_file(
-        "/mock_snapshot_config.toml",
-        contents='zpools = ["root_pool", "storage", "media"]\nservices = ["docker"]\n',
-    )
-
-    mocker.patch(f"{VALIDATE_SYSTEM}.signal_alert")
-    mocker.patch(f"{VALIDATE_SYSTEM}.systemd_tests", return_value=["systemd_tests error"])
-    mocker.patch(f"{VALIDATE_SYSTEM}.zpool_tests", return_value=["zpool_tests error"])
-
-    with pytest.raises(SystemExit) as exception_info:
-        main(Path("/mock_snapshot_config.toml"))
-
-    assert exception_info.value.code == 1
-
-
-def test_validate_system_execution(mocker: MockerFixture, fs: FakeFilesystem) -> None:
-    """test_validate_system_execution."""
-    fs.create_file(
-        "/mock_snapshot_config.toml",
-        contents='zpools = ["root_pool", "storage", "media"]\nservices = ["docker"]\n',
-    )
-
-    mocker.patch(f"{VALIDATE_SYSTEM}.signal_alert")
-    mocker.patch(f"{VALIDATE_SYSTEM}.systemd_tests", return_value=None)
-    mocker.patch(f"{VALIDATE_SYSTEM}.zpool_tests", side_effect=RuntimeError("zpool_tests error"))
-
-    with pytest.raises(SystemExit) as exception_info:
-        main(Path("/mock_snapshot_config.toml"))
-
-    assert exception_info.value.code == 1