diff --git a/python/system_tests/__init__.py b/python/system_tests/__init__.py deleted file mode 100644 index e022647..0000000 --- a/python/system_tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""system_tests.""" diff --git a/python/system_tests/components.py b/python/system_tests/components.py deleted file mode 100644 index 239c997..0000000 --- a/python/system_tests/components.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Validate Jeeves.""" - -from __future__ import annotations - -import logging -from copy import copy -from re import search -from time import sleep -from typing import TYPE_CHECKING - -from python.common import bash_wrapper -from python.zfs import Zpool - -if TYPE_CHECKING: - from collections.abc import Sequence - -logger = logging.getLogger(__name__) - - -def zpool_tests(pool_names: Sequence[str], zpool_capacity_threshold: int = 90) -> list[str] | None: - """Check the zpool health and capacity. - - Args: - pool_names (Sequence[str]): A list of pool names to test. - zpool_capacity_threshold (int, optional): The threshold for the zpool capacity. Defaults to 90. - - Returns: - list[str] | None: A list of errors if any. - """ - logger.info("Testing zpool") - - errors: list[str] = [] - for pool_name in pool_names: - pool = Zpool(pool_name) - if pool.health != "ONLINE": - errors.append(f"{pool.name} is {pool.health}") - if pool.capacity >= zpool_capacity_threshold: - errors.append(f"{pool.name} is low on space") - - upgrade_status, _ = bash_wrapper("zpool upgrade") - if not search(r"Every feature flags pool has all supported and requested features enabled.", upgrade_status): - errors.append("ZPool out of date run `sudo zpool upgrade -a`") - - return errors - - -def systemd_tests( - service_names: Sequence[str], - max_retries: int = 30, - retry_delay_secs: int = 1, - retryable_statuses: Sequence[str] | None = None, - valid_statuses: Sequence[str] | None = None, -) -> list[str] | None: - """Tests a systemd services. - - Args: - service_names (Sequence[str]): A list of service names to test. - max_retries (int, optional): The maximum number of retries. Defaults to 30. - minimum value is 1. - retry_delay_secs (int, optional): The delay between retries in seconds. Defaults to 1. - minimum value is 1. - retryable_statuses (Sequence[str] | None, optional): A list of retryable statuses. Defaults to None. - valid_statuses (Sequence[str] | None, optional): A list of valid statuses. Defaults to None. - - Returns: - list[str] | None: A list of errors if any. - """ - logger.info("Testing systemd service") - - max_retries = max(max_retries, 1) - retry_delay_secs = max(retry_delay_secs, 1) - last_try = max_retries - 1 - - if retryable_statuses is None: - retryable_statuses = ("inactive\n", "activating\n") - - if valid_statuses is None: - valid_statuses = ("active\n",) - - service_names_set = set(service_names) - - errors: set[str] = set() - for retry in range(max_retries): - if not service_names_set: - break - logger.info(f"Testing systemd service in {retry + 1} of {max_retries}") - service_names_to_test = copy(service_names_set) - for service_name in service_names_to_test: - service_status, _ = bash_wrapper(f"systemctl is-active {service_name}") - if service_status in valid_statuses: - service_names_set.remove(service_name) - continue - if service_status in retryable_statuses and retry < last_try: - continue - errors.add(f"{service_name} is {service_status.strip()}") - - sleep(retry_delay_secs) - - return list(errors) diff --git a/python/system_tests/validate_system.py b/python/system_tests/validate_system.py deleted file mode 100644 index ef75525..0000000 --- a/python/system_tests/validate_system.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Validate {server_name}.""" - -import logging -import sys -import tomllib -from os import environ -from pathlib import Path # noqa: TC003 This is required for the typer CLI -from socket import gethostname - -import typer - -from python.common import configure_logger -from python.signal_alert import signal_alert -from python.system_tests.components import systemd_tests, zpool_tests - -logger = logging.getLogger(__name__) - - -def load_config_data(config_file: Path) -> dict[str, list[str]]: - """Load a TOML configuration file. - - Args: - config_file (Path): The path to the configuration file. - - Returns: - dict: The configuration data. - """ - return tomllib.loads(config_file.read_text()) - - -def main(config_file: Path) -> None: - """Main.""" - configure_logger(level=environ.get("LOG_LEVEL", "INFO")) - - server_name = gethostname() - logger.info(f"Starting {server_name} validation") - - config_data = load_config_data(config_file) - - errors: list[str] = [] - try: - if config_data.get("zpools") and (zpool_errors := zpool_tests(config_data["zpools"])): - errors.extend(zpool_errors) - - if config_data.get("services") and (systemd_errors := systemd_tests(config_data["services"])): - errors.extend(systemd_errors) - - except Exception as error: - logger.exception(f"{server_name} validation failed") - errors.append(f"{server_name} validation failed: {error}") - - if errors: - logger.error(f"{server_name} validation failed: \n{'\n'.join(errors)}") - signal_alert(f"{server_name} validation failed {errors}") - - sys.exit(1) - - logger.info(f"{server_name} validation passed") - - -def cli() -> None: - """CLI.""" - typer.run(main) - - -if __name__ == "__main__": - cli() diff --git a/systems/jeeves/services/systemd.nix b/systems/jeeves/services/systemd.nix index f56bad4..525d825 100644 --- a/systems/jeeves/services/systemd.nix +++ b/systems/jeeves/services/systemd.nix @@ -1,11 +1,7 @@ { pkgs, - inputs, ... }: -let - vars = import ../vars.nix; -in { systemd = { services = { @@ -30,21 +26,6 @@ in ExecStart = "${pkgs.bash}/bin/bash -c 'echo 1 > /sys/bus/pci/devices/0000:61:00.0/remove'"; }; }; - startup_validation = { - requires = [ "network-online.target" ]; - after = [ "network-online.target" ]; - wantedBy = [ "multi-user.target" ]; - description = "validates startup"; - path = [ pkgs.zfs ]; - environment = { - PYTHONPATH = "${inputs.self}/"; - }; - serviceConfig = { - EnvironmentFile = "${vars.secrets}/services/server-validation"; - Type = "oneshot"; - ExecStart = "${pkgs.my_python}/bin/python -m python.system_tests.validate_system '${./validate_system.toml}'"; - }; - }; }; timers = { plex_permission = { @@ -55,13 +36,6 @@ in Unit = "plex_permission.service"; }; }; - startup_validation = { - wantedBy = [ "timers.target" ]; - timerConfig = { - OnBootSec = "10min"; - Unit = "startup_validation.service"; - }; - }; }; }; } diff --git a/systems/jeeves/services/validate_system.toml b/systems/jeeves/services/validate_system.toml deleted file mode 100644 index 11214b8..0000000 --- a/systems/jeeves/services/validate_system.toml +++ /dev/null @@ -1,6 +0,0 @@ -zpool = ["root_pool", "storage", "media"] -services = [ - "audiobookshelf", - "docker", - "jellyfin", -] diff --git a/tests/test_components.py b/tests/test_components.py deleted file mode 100644 index 8154fb3..0000000 --- a/tests/test_components.py +++ /dev/null @@ -1,104 +0,0 @@ -"""test_components.""" - -from pytest_mock import MockerFixture - -from python.system_tests.components import systemd_tests, zpool_tests -from python.zfs import Zpool - -temp = "Every feature flags pool has all supported and requested features enabled.\n" - -SYSTEM_TESTS_COMPONENTS = "python.system_tests.components" - - -def test_zpool_tests(mocker: MockerFixture) -> None: - """test_zpool_tests.""" - mock_zpool = mocker.MagicMock(spec=Zpool) - mock_zpool.health = "ONLINE" - mock_zpool.capacity = 70 - mock_zpool.name = "Main" - mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool) - mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, "")) - errors = zpool_tests(("Main",)) - assert errors == [] - - -def test_zpool_tests_out_of_date(mocker: MockerFixture) -> None: - """test_zpool_tests_out_of_date.""" - mock_zpool = mocker.MagicMock(spec=Zpool) - mock_zpool.health = "ONLINE" - mock_zpool.capacity = 70 - mock_zpool.name = "Main" - mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool) - mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("", "")) - errors = zpool_tests(("Main",)) - assert errors == ["ZPool out of date run `sudo zpool upgrade -a`"] - - -def test_zpool_tests_out_of_space(mocker: MockerFixture) -> None: - """test_zpool_tests_out_of_space.""" - mock_zpool = mocker.MagicMock(spec=Zpool) - mock_zpool.health = "ONLINE" - mock_zpool.capacity = 100 - mock_zpool.name = "Main" - mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool) - mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, "")) - errors = zpool_tests(("Main",)) - assert errors == ["Main is low on space"] - - -def test_zpool_tests_offline(mocker: MockerFixture) -> None: - """test_zpool_tests_offline.""" - mock_zpool = mocker.MagicMock(spec=Zpool) - mock_zpool.health = "OFFLINE" - mock_zpool.capacity = 70 - mock_zpool.name = "Main" - mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.Zpool", return_value=mock_zpool) - mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=(temp, "")) - errors = zpool_tests(("Main",)) - assert errors == ["Main is OFFLINE"] - - -def test_systemd_tests(mocker: MockerFixture) -> None: - """test_systemd_tests.""" - mocker.patch( - f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", - side_effect=[ - ("inactive\n", ""), - ("active\n", ""), - ], - ) - errors = systemd_tests(("docker",)) - assert errors == [] - """test_systemd_tests.""" - - -def test_systemd_tests_multiple_negative_retries(mocker: MockerFixture) -> None: - """test_systemd_tests_fail.""" - mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("active\n", "")) - errors = systemd_tests(("docker",), max_retries=-1, retry_delay_secs=-1) - assert errors == [] - - -def test_systemd_tests_multiple_pass(mocker: MockerFixture) -> None: - """test_systemd_tests_fail.""" - mocker.patch( - f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", - side_effect=[ - ("inactive\n", ""), - ("activating\n", ""), - ("active\n", ""), - ], - ) - errors = systemd_tests( - ("docker",), - retryable_statuses=("inactive\n", "activating\n"), - valid_statuses=("active\n",), - ) - assert errors == [] - - -def test_systemd_tests_fail(mocker: MockerFixture) -> None: - """test_systemd_tests_fail.""" - mocker.patch(f"{SYSTEM_TESTS_COMPONENTS}.bash_wrapper", return_value=("inactive\n", "")) - errors = systemd_tests(("docker",), max_retries=5) - assert errors == ["docker is inactive"] diff --git a/tests/test_server_validate_scripts.py b/tests/test_server_validate_scripts.py deleted file mode 100644 index 07e3b99..0000000 --- a/tests/test_server_validate_scripts.py +++ /dev/null @@ -1,63 +0,0 @@ -"""test_server_validate_scripts.""" - -from __future__ import annotations - -from pathlib import Path -from typing import TYPE_CHECKING - -import pytest -from pytest_mock import MockerFixture - -from python.system_tests.validate_system import main - -if TYPE_CHECKING: - from pyfakefs.fake_filesystem import FakeFilesystem - from pytest_mock import MockerFixture - -VALIDATE_SYSTEM = "python.system_tests.validate_system" - - -def test_validate_system(mocker: MockerFixture, fs: FakeFilesystem) -> None: - """test_validate_system.""" - fs.create_file( - "/mock_snapshot_config.toml", - contents='zpools = ["root_pool", "storage", "media"]\nservices = ["docker"]\n', - ) - - mocker.patch(f"{VALIDATE_SYSTEM}.systemd_tests", return_value=None) - mocker.patch(f"{VALIDATE_SYSTEM}.zpool_tests", return_value=None) - main(Path("/mock_snapshot_config.toml")) - - -def test_validate_system_errors(mocker: MockerFixture, fs: FakeFilesystem) -> None: - """test_validate_system_errors.""" - fs.create_file( - "/mock_snapshot_config.toml", - contents='zpools = ["root_pool", "storage", "media"]\nservices = ["docker"]\n', - ) - - mocker.patch(f"{VALIDATE_SYSTEM}.signal_alert") - mocker.patch(f"{VALIDATE_SYSTEM}.systemd_tests", return_value=["systemd_tests error"]) - mocker.patch(f"{VALIDATE_SYSTEM}.zpool_tests", return_value=["zpool_tests error"]) - - with pytest.raises(SystemExit) as exception_info: - main(Path("/mock_snapshot_config.toml")) - - assert exception_info.value.code == 1 - - -def test_validate_system_execution(mocker: MockerFixture, fs: FakeFilesystem) -> None: - """test_validate_system_execution.""" - fs.create_file( - "/mock_snapshot_config.toml", - contents='zpools = ["root_pool", "storage", "media"]\nservices = ["docker"]\n', - ) - - mocker.patch(f"{VALIDATE_SYSTEM}.signal_alert") - mocker.patch(f"{VALIDATE_SYSTEM}.systemd_tests", return_value=None) - mocker.patch(f"{VALIDATE_SYSTEM}.zpool_tests", side_effect=RuntimeError("zpool_tests error")) - - with pytest.raises(SystemExit) as exception_info: - main(Path("/mock_snapshot_config.toml")) - - assert exception_info.value.code == 1