feat(snapshot-manager): migrate snapshot management to Rust
build_systems / prebuild-common-x86-64-v3 (pull_request) Successful in 20s
build_systems / build-portal-1 (pull_request) Successful in 16s
test ebook search / test-ebook-search (pull_request) Successful in 1m8s
build_systems / build-bob (pull_request) Successful in 1m42s
build_systems / build-rhapsody-in-green (pull_request) Successful in 1m41s
build_systems / build-jeeves (pull_request) Successful in 1m44s
build_systems / build-brain (pull_request) Successful in 1m57s
treefmt / nix fmt (pull_request) Successful in 6s
treefmt / nix fmt (push) Successful in 3s
build_systems / prebuild-common-x86-64-v3 (push) Successful in 8s
build_systems / build-portal-1 (push) Successful in 21s
build_systems / build-bob (push) Successful in 30s
build_systems / build-brain (push) Successful in 38s
build_systems / build-rhapsody-in-green (push) Successful in 39s
test ebook search / test-ebook-search (push) Successful in 1m14s
build_systems / build-jeeves (push) Successful in 1m8s

This commit was merged in pull request #95.
This commit is contained in:
Richie committed 2026-10-11 20:11:21 -04:00
1 parent 704999b94c
commit 1ccba91d2a
15 files changed
+958 -424

No files matched your search

+2 -2
View File
@@ -16,8 +16,8 @@ zfs = { path = "rust/zfs" }
[workspace.lints.clippy] [workspace.lints.clippy]
all = { level = "deny", priority = -1 } all = { level = "deny", priority = -1 }
pedantic = { level = "deny", priority = -1 } # UM, ACTUALLY pedantic = { level = "deny", priority = -1 } # UM, ACTUALLY
nursery = { level = "deny", priority = -1 } # BETA LINTS nursery = { level = "deny", priority = -1 } # BETA LINTS
# DENY PANICS # DENY PANICS
unwrap_used = "deny" unwrap_used = "deny"
expect_used = "deny" expect_used = "deny"
-2
View File
@@ -1,5 +1,4 @@
{ {
inputs,
lib, lib,
pkgs, pkgs,
... ...
@@ -15,7 +14,6 @@
services = { services = {
snapshot_manager = { snapshot_manager = {
enable = lib.mkDefault true; enable = lib.mkDefault true;
PYTHONPATH = "${inputs.self}/";
}; };
zfs = { zfs = {
+4 -46
View File
@@ -1,5 +1,4 @@
{ {
inputs,
pkgs, pkgs,
lib, lib,
config, config,
@@ -7,23 +6,18 @@
}: }:
let let
cfg = config.services.snapshot_manager; cfg = config.services.snapshot_manager;
snapshotManagerPackages = ps: with ps; [ typer ]; snapshotManager = pkgs.callPackage ../../../rust/snapshot_manager/package.nix { };
in in
{ {
options = { options = {
services.snapshot_manager = { services.snapshot_manager = {
enable = lib.mkEnableOption "ZFS snapshot manager"; enable = lib.mkEnableOption "ZFS snapshot manager";
signalAlerts = lib.mkEnableOption "Signal failure alerts for the snapshot manager";
path = lib.mkOption { path = lib.mkOption {
type = lib.types.path; type = lib.types.path;
default = ./snapshot_config.toml; default = ./snapshot_config.toml;
description = "Path to the snapshot_manager TOML config."; description = "Path to the snapshot_manager TOML config.";
}; };
PYTHONPATH = lib.mkOption {
type = lib.types.str;
description = ''
the PYTHONPATH to use for the snapshot_manager service.
'';
};
EnvironmentFile = lib.mkOption { EnvironmentFile = lib.mkOption {
type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str); type = lib.types.nullOr (lib.types.coercedTo lib.types.path toString lib.types.str);
default = null; default = null;
@@ -36,40 +30,7 @@ in
}; };
config = lib.mkIf cfg.enable { config = lib.mkIf cfg.enable {
nixpkgs.overlays = [ system.checks = [ snapshotManager ];
(final: _prev: {
snapshot_manager_python = final.python314.withPackages snapshotManagerPackages;
snapshot_manager_test_python = final.python314.withPackages (
ps:
snapshotManagerPackages ps
++ (with ps; [
pyfakefs
pytest
pytest-asyncio
pytest-mock
pytest-xdist
])
);
snapshot_manager_tests =
final.runCommand "snapshot-manager-tests"
{
nativeBuildInputs = [ final.snapshot_manager_test_python ];
}
''
export HOME="$TMPDIR"
cd ${inputs.self}
pytest \
-o cache_dir="$TMPDIR/pytest-cache" \
tests/test_common.py \
tests/test_signal_alert.py \
tests/test_snapshot_manager.py \
tests/test_zfs.py
touch "$out"
'';
})
];
system.checks = [ pkgs.snapshot_manager_tests ];
systemd = { systemd = {
services.snapshot_manager = { services.snapshot_manager = {
@@ -77,12 +38,9 @@ in
requires = [ "zfs-import.target" ]; requires = [ "zfs-import.target" ];
after = [ "zfs-import.target" ]; after = [ "zfs-import.target" ];
path = [ pkgs.zfs ]; path = [ pkgs.zfs ];
environment = {
PYTHONPATH = cfg.PYTHONPATH;
};
serviceConfig = { serviceConfig = {
Type = "oneshot"; Type = "oneshot";
ExecStart = "${pkgs.snapshot_manager_python}/bin/python -m python.tools.snapshot_manager ${lib.escapeShellArg cfg.path}"; ExecStart = "${snapshotManager}/bin/snapshot-manager ${lib.escapeShellArg cfg.path} --signal-alerts ${lib.boolToString cfg.signalAlerts}";
} }
// lib.optionalAttrs (cfg.EnvironmentFile != null) { // lib.optionalAttrs (cfg.EnvironmentFile != null) {
EnvironmentFile = cfg.EnvironmentFile; EnvironmentFile = cfg.EnvironmentFile;
-161
View File
@@ -1,161 +0,0 @@
"""snapshot_manager."""
from __future__ import annotations
import logging
import sys
import tomllib
from functools import cache
from os import getenv
from pathlib import Path # noqa: TC003 This is required for the typer CLI
from re import compile as re_compile
from re import search
import typer
from python.common import configure_logger, utcnow
from python.signal_alert import signal_alert
from python.zfs import Dataset, get_datasets
logger = logging.getLogger(__name__)
def main(config_file: Path) -> None:
"""Main."""
configure_logger(level="DEBUG")
logger.info("Starting snapshot_manager")
from_phone = getenv("SIGNAL_ALERT_FROM_PHONE")
to_phone = getenv("SIGNAL_ALERT_TO_PHONE")
if not from_phone:
error = "SIGNAL_ALERT_FROM_PHONE environment variable is not set"
raise ValueError(error)
if not to_phone:
error = "SIGNAL_ALERT_TO_PHONE environment variable is not set"
raise ValueError(error)
try:
time_stamp = get_time_stamp()
for dataset in get_datasets():
status = dataset.create_snapshot(time_stamp)
logger.debug(f"{status=}")
if status != "snapshot created":
msg = f"{dataset.name} failed to create snapshot {time_stamp}"
logger.error(msg)
signal_alert(msg, from_phone, to_phone)
continue
count_lookup = get_count_lookup(config_file, dataset.name)
logger.info(f"using {count_lookup} for {dataset.name}")
get_snapshots_to_delete(dataset, count_lookup, from_phone, to_phone)
except Exception:
logger.exception("snapshot_manager failed")
signal_alert("snapshot_manager failed", from_phone, to_phone)
sys.exit(1)
else:
logger.info("snapshot_manager completed")
def get_count_lookup(config_file: Path, dataset_name: str) -> dict[str, int]:
"""Get the count lookup.
Args:
config_file (Path): The path to the configuration file.
dataset_name (str): The name of the dataset.
Returns:
dict[str, int]: The count lookup.
"""
config_data = load_config_data(config_file)
return config_data.get(dataset_name, get_default_config(config_data))
def get_default_config(config_data: dict[str, dict[str, int]]) -> dict[str, int]:
"""Get the default configuration.
Args:
config_data (dict[str, dict[str, int]]): The configuration data.
Returns:
dict[str, int]: The default configuration.
"""
return config_data.get(
"default",
{"15_min": 4, "hourly": 12, "daily": 0, "monthly": 0},
)
@cache
def load_config_data(config_file: Path) -> dict[str, dict[str, int]]:
"""Load a TOML configuration file.
Args:
config_file (Path): The path to the configuration file.
Returns:
dict: The configuration data.
"""
return tomllib.loads(config_file.read_text())
def get_snapshots_to_delete(
dataset: Dataset,
count_lookup: dict[str, int],
from_phone: str,
to_phone: str,
) -> None:
"""Get snapshots to delete.
Args:
dataset (Dataset): the dataset
count_lookup (dict[str, int]): the count lookup
from_phone (str): The Signal account sending alerts.
to_phone (str): The Signal account receiving alerts.
"""
snapshots = dataset.get_snapshots()
logger.info(f"calculating snapshots for {dataset.name} to be deleted")
if not snapshots:
logger.info(f"{dataset.name} has no snapshots")
return
filters = (
("15_min", re_compile(r"auto_\d{10}(?:15|30|45)")),
("hourly", re_compile(r"auto_\d{8}(?!00)\d{2}00")),
("daily", re_compile(r"auto_\d{6}(?!01)\d{2}0000")),
("monthly", re_compile(r"auto_\d{6}010000")),
)
for filter_name, snapshot_filter in filters:
logger.debug(f"{filter_name=}\n{snapshot_filter=}")
filtered_snapshots = sorted(snapshot.name for snapshot in snapshots if search(snapshot_filter, snapshot.name))
logger.debug(f"{filtered_snapshots=}")
snapshots_wanted = count_lookup[filter_name]
snapshots_being_deleted = filtered_snapshots[:-snapshots_wanted] if snapshots_wanted > 0 else filtered_snapshots
logger.info(f"{snapshots_being_deleted} are being deleted")
for snapshot in snapshots_being_deleted:
if error := dataset.delete_snapshot(snapshot):
error_message = f"{dataset.name}@{snapshot} failed to delete: {error}"
signal_alert(error_message, from_phone, to_phone)
logger.error(error_message)
def get_time_stamp() -> str:
"""Get the time stamp."""
now = utcnow()
nearest_15_min = now.replace(minute=(now.minute - (now.minute % 15)))
return nearest_15_min.strftime("auto_%Y%m%d%H%M")
def cli() -> None:
"""CLI."""
typer.run(main)
if __name__ == "__main__":
cli()
+10
View File
@@ -0,0 +1,10 @@
{ lib }:
lib.fileset.toSource {
root = ../.;
fileset = lib.fileset.unions [
../Cargo.toml
../Cargo.lock
../clippy.toml
./.
];
}
+23
View File
@@ -0,0 +1,23 @@
[package]
name = "snapshot-manager"
version = "0.1.0"
edition = "2024"
description = "Create and rotate ZFS snapshots with Signal failure alerts"
license = "MIT"
[dependencies]
anyhow.workspace = true
chrono.workspace = true
clap = { workspace = true, features = ["derive", "env"] }
env_logger.workspace = true
log.workspace = true
serde = { workspace = true, features = ["derive"] }
signal.workspace = true
toml = "0.9"
zfs.workspace = true
[dev-dependencies]
tempfile = "3"
[lints]
workspace = true
+25
View File
@@ -0,0 +1,25 @@
{ lib, rustPlatform }:
rustPlatform.buildRustPackage {
pname = "snapshot-manager";
version = "0.1.0";
src = import ../rust-source.nix { inherit lib; };
cargoLock.lockFile = ../../Cargo.lock;
cargoBuildFlags = [
"-p"
"snapshot-manager"
];
cargoTestFlags = [
"-p"
"snapshot-manager"
"-p"
"signal"
"-p"
"zfs"
];
meta = {
description = "ZFS snapshot manager with Signal failure alerts";
mainProgram = "snapshot-manager";
};
}
+383
View File
@@ -0,0 +1,383 @@
use std::{
collections::HashMap,
fs,
path::{Path, PathBuf},
process::ExitCode,
};
use anyhow::{Context, Result};
use chrono::{DateTime, Datelike, NaiveDate, NaiveTime, Timelike, Utc};
use clap::Parser;
use log::{error, info};
use serde::Deserialize;
use signal::Signal;
#[derive(Parser)]
#[command(version, about)]
struct Args {
/// TOML file containing default and per-dataset snapshot retention counts.
config_file: PathBuf,
/// Send Signal failure alerts (requires both phone numbers).
#[arg(long, env = "SIGNAL_ALERTS", default_value_t = false, action = clap::ArgAction::Set)]
signal_alerts: bool,
/// Signal account sending failure alerts.
#[arg(
long,
env = "SIGNAL_ALERT_FROM_PHONE",
hide_env_values = true,
required_if_eq("signal_alerts", "true"),
value_parser = clap::builder::NonEmptyStringValueParser::new()
)]
signal_alert_from_phone: Option<String>,
/// Signal recipient for failure alerts.
#[arg(
long,
env = "SIGNAL_ALERT_TO_PHONE",
hide_env_values = true,
required_if_eq("signal_alerts", "true"),
value_parser = clap::builder::NonEmptyStringValueParser::new()
)]
signal_alert_to_phone: Option<String>,
}
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
struct Retention {
#[serde(rename = "15_min")]
quarter_hourly: usize,
hourly: usize,
daily: usize,
monthly: usize,
}
impl Default for Retention {
fn default() -> Self {
Self {
quarter_hourly: 4,
hourly: 12,
daily: 0,
monthly: 0,
}
}
}
#[derive(Debug, Deserialize)]
#[serde(transparent)]
struct Config(HashMap<String, Retention>);
impl Config {
fn load(path: &Path) -> Result<Self> {
let contents = fs::read_to_string(path)
.with_context(|| format!("Failed to read configuration {}", path.display()))?;
toml::from_str(&contents)
.with_context(|| format!("Invalid configuration {}", path.display()))
}
fn retention_for(&self, dataset: &str) -> Retention {
self.0
.get(dataset)
.or_else(|| self.0.get("default"))
.copied()
.unwrap_or_default()
}
}
fn main() -> ExitCode {
let args = Args::parse();
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
info!("Starting snapshot_manager");
let signal = if args.signal_alerts {
args.signal_alert_from_phone
.map(|from_phone| Signal { from_phone })
} else {
None
};
let alerts = signal.as_ref().zip(args.signal_alert_to_phone.as_deref());
match run(&args.config_file, alerts) {
Ok(()) => {
info!("snapshot_manager completed");
ExitCode::SUCCESS
}
Err(error) => {
error!("{error:#}");
ExitCode::FAILURE
}
}
}
fn run(config_file: &Path, alerts: Option<(&Signal, &str)>) -> Result<()> {
let result = (|| {
// Validate all retention settings before making any changes to ZFS.
let config = Config::load(config_file)?;
let snapshot_time = timestamp(Utc::now());
for dataset_name in zfs::list_datasets()? {
if let Err(error) = zfs::create_snapshot(&dataset_name, &snapshot_time) {
let message = format!("{dataset_name} failed to create snapshot {snapshot_time}");
error!("{message}: {error:#}");
send_alert(alerts, &message)?;
continue;
}
remove_old_snapshots(&config, alerts, &dataset_name)
.context("Failed to remove old snapshots")?;
}
Ok(())
})();
if let Err(error) = &result {
error!("snapshot_manager failed: {error:#}");
if let Err(alert_error) = send_alert(alerts, "snapshot_manager failed") {
error!("Failed to send failure alert: {alert_error:#}");
}
}
result
}
fn timestamp(now: DateTime<Utc>) -> String {
let minute = now.minute().saturating_sub(now.minute() % 15);
format!("auto_{}{minute:02}", now.format("%Y%m%d%H"))
}
fn remove_old_snapshots(
config: &Config,
alerts: Option<(&Signal, &str)>,
dataset_name: &str,
) -> Result<()> {
let retention = config.retention_for(dataset_name);
info!("Using {retention:?} for {dataset_name}");
let snapshots = zfs::list_snapshots(dataset_name)?;
for snapshot in snapshots_to_delete(&snapshots, retention) {
info!("Deleting {dataset_name}@{snapshot}");
if let Some(error) = zfs::delete_snapshot(dataset_name, snapshot)? {
let message = format!("{dataset_name}@{snapshot} failed to delete: {error}");
error!("{message}");
send_alert(alerts, &message)?;
}
}
Ok(())
}
fn snapshots_to_delete(snapshots: &[String], retention: Retention) -> Vec<&str> {
let mut tiers: [Vec<&str>; 4] = Default::default();
for snapshot in snapshots {
if let Some(index) = bucket(snapshot)
&& let Some(tier) = tiers.get_mut(index)
{
tier.push(snapshot);
}
}
let counts = [
retention.quarter_hourly,
retention.hourly,
retention.daily,
retention.monthly,
];
let mut deletions = Vec::new();
for (mut tier, count) in tiers.into_iter().zip(counts) {
tier.sort_unstable();
let delete_count = tier.len().saturating_sub(count);
deletions.extend(tier.into_iter().take(delete_count));
}
deletions
}
fn bucket(name: &str) -> Option<usize> {
let digits = name.strip_prefix("auto_")?;
if digits.len() != 12 || !digits.bytes().all(|byte| byte.is_ascii_digit()) {
return None;
}
let date = NaiveDate::from_ymd_opt(
digits.get(..4)?.parse().ok()?,
digits.get(4..6)?.parse().ok()?,
digits.get(6..8)?.parse().ok()?,
)?;
let time = NaiveTime::from_hms_opt(
digits.get(8..10)?.parse().ok()?,
digits.get(10..)?.parse().ok()?,
0,
)?;
// The tiers are disjoint: midnight belongs to daily, except on the first
// of the month, when it belongs to monthly.
match (time.minute(), time.hour(), date.day()) {
(15 | 30 | 45, _, _) => Some(0),
(0, 1..=23, _) => Some(1),
(0, 0, 2..=31) => Some(2),
(0, 0, 1) => Some(3),
_ => None,
}
}
fn send_alert(alerts: Option<(&Signal, &str)>, body: &str) -> Result<()> {
if let Some((signal, to_phone)) = alerts {
signal.send(to_phone, body)?;
}
Ok(())
}
#[cfg(test)]
mod tests {
use chrono::TimeZone;
use super::*;
#[test]
fn dataset_overrides_default_and_other_datasets_fall_back() {
let config: Config = toml::from_str(
r#"
[default]
15_min = 8
hourly = 24
daily = 0
monthly = 0
["pool/home"]
15_min = 2
hourly = 3
daily = 4
monthly = 5
"#,
)
.unwrap();
assert_eq!(
config.retention_for("pool/home"),
Retention {
quarter_hourly: 2,
hourly: 3,
daily: 4,
monthly: 5,
}
);
assert_eq!(config.retention_for("pool/other").hourly, 24);
assert_eq!(config.retention_for("pool/home/child").quarter_hourly, 8);
}
#[test]
fn absent_default_uses_original_builtin_counts() {
let config: Config = toml::from_str("").unwrap();
assert_eq!(config.retention_for("pool/home"), Retention::default());
}
#[test]
fn rejects_incomplete_negative_and_unknown_retention_settings() {
for settings in [
"15_min = 1\nhourly = 2\ndaily = 3",
"15_min = -1\nhourly = 2\ndaily = 3\nmonthly = 4",
"15_min = 1\nhourly = 2\ndaily = 3\nmonthly = 4\nweekly = 5",
"15_min = 1.5\nhourly = 2\ndaily = 3\nmonthly = 4",
] {
assert!(toml::from_str::<Config>(&format!("[default]\n{settings}")).is_err());
}
}
#[test]
fn timestamps_round_down_at_quarter_hour_boundaries() {
for (minute, expected) in [
(0, "00"),
(14, "00"),
(15, "15"),
(29, "15"),
(30, "30"),
(44, "30"),
(45, "45"),
(59, "45"),
] {
let now = Utc.with_ymd_and_hms(2026, 1, 1, 0, minute, 59).unwrap();
assert_eq!(timestamp(now), format!("auto_2026010100{expected}"));
}
}
#[test]
fn tiers_are_disjoint_at_day_and_month_boundaries() {
for (name, expected) in [
("auto_202602010015", 0),
("auto_202602010030", 0),
("auto_202602010045", 0),
("auto_202602010100", 1),
("auto_202602012300", 1),
("auto_202602020000", 2),
("auto_202402290000", 2),
("auto_202602010000", 3),
("auto_202601010000", 3),
] {
assert_eq!(bucket(name), Some(expected), "{name}");
}
}
#[test]
fn only_exact_valid_manager_names_are_eligible() {
for name in [
"manual",
"manual_auto_202602010015",
"auto_202602010015_backup",
"auto_20260201001",
"auto_202602300000",
"auto_202602012400",
"auto_202602010060",
"auto_202602010005",
"auto_202602010015",
] {
assert_eq!(bucket(name), None, "{name}");
}
}
#[test]
fn keeps_newest_in_each_tier_regardless_of_input_order() {
let snapshots = [
"auto_202601010000",
"auto_202602020000",
"auto_202602020115",
"auto_202602020215",
"auto_202602020200",
"auto_202602020100",
"auto_202602030000",
"auto_202602010000",
"manual",
]
.map(String::from);
let retention = Retention {
quarter_hourly: 1,
hourly: 1,
daily: 1,
monthly: 1,
};
assert_eq!(
snapshots_to_delete(&snapshots, retention),
[
"auto_202602020115",
"auto_202602020100",
"auto_202602020000",
"auto_202601010000",
]
);
}
#[test]
fn zero_counts_delete_all_managed_snapshots_and_large_counts_keep_them() {
let snapshots = [
"auto_202601010015",
"auto_202601010100",
"auto_202601020000",
"auto_202601010000",
"manual",
]
.map(String::from);
let none = Retention {
quarter_hourly: 0,
hourly: 0,
daily: 0,
monthly: 0,
};
assert_eq!(snapshots_to_delete(&snapshots, none).len(), 4);
let all = Retention {
quarter_hourly: 100,
hourly: 100,
daily: 100,
monthly: 100,
};
assert!(snapshots_to_delete(&snapshots, all).is_empty());
assert!(snapshots_to_delete(&[], none).is_empty());
}
}
+202
View File
@@ -0,0 +1,202 @@
#![cfg(test)]
use std::{fs, os::unix::fs::symlink, path::Path, process::Command};
use tempfile::tempdir;
fn command() -> Command {
let mut command = Command::new(env!("CARGO_BIN_EXE_snapshot-manager"));
command
.env_remove("SIGNAL_ALERTS")
.env_remove("SIGNAL_ALERT_FROM_PHONE")
.env_remove("SIGNAL_ALERT_TO_PHONE")
.env_remove("SNAPSHOT_TEST_SCRIPT")
.env_remove("RUST_LOG");
command
}
#[test]
fn help_works_without_phone_numbers() {
let output = command().arg("--help").output().unwrap();
assert!(output.status.success());
assert!(
String::from_utf8(output.stdout)
.unwrap()
.contains("<CONFIG_FILE>")
);
}
#[test]
fn help_documents_environment_variables_without_exposing_phone_numbers() {
let output = command()
.env("SIGNAL_ALERT_FROM_PHONE", "+15555550101")
.env("SIGNAL_ALERT_TO_PHONE", "+15555550102")
.arg("--help")
.output()
.unwrap();
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
for name in [
"SIGNAL_ALERTS",
"SIGNAL_ALERT_FROM_PHONE",
"SIGNAL_ALERT_TO_PHONE",
] {
assert!(stdout.contains(name), "{stdout}");
}
assert!(!stdout.contains("+15555550101"), "{stdout}");
assert!(!stdout.contains("+15555550102"), "{stdout}");
}
#[test]
fn config_argument_is_required() {
let output = command().output().unwrap();
assert_eq!(output.status.code(), Some(2));
}
#[test]
fn enabled_alerts_require_phone_numbers_before_running() {
for use_env in [false, true] {
for (from, to, expected) in [
(None, None, "--signal-alert-from-phone"),
(Some(""), None, "--signal-alert-from-phone"),
(Some("+15555550101"), None, "--signal-alert-to-phone"),
(Some("+15555550101"), Some(""), "--signal-alert-to-phone"),
(None, Some("+15555550102"), "--signal-alert-from-phone"),
] {
let mut command = command();
if use_env {
command.env("SIGNAL_ALERTS", "true");
} else {
command.args(["--signal-alerts", "true"]);
}
if let Some(from) = from {
command.env("SIGNAL_ALERT_FROM_PHONE", from);
}
if let Some(to) = to {
command.env("SIGNAL_ALERT_TO_PHONE", to);
}
let output = command.arg("unused.toml").output().unwrap();
let stderr = String::from_utf8(output.stderr).unwrap();
assert_eq!(output.status.code(), Some(2), "{stderr}");
assert!(stderr.contains(expected), "{stderr}");
assert!(!stderr.contains("Starting snapshot_manager"), "{stderr}");
}
}
}
#[test]
fn enabled_alerts_accept_environment_variables_and_cli_overrides() {
let dir = tempdir().unwrap();
let config = dir.path().join("config.toml");
let scenario = dir.path().join("scenario");
fs::write(&config, "").unwrap();
// An empty dataset list completes successfully without sending an alert.
fs::write(&scenario, "").unwrap();
symlink(
Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/zfs"),
dir.path().join("zfs"),
)
.unwrap();
for use_cli in [false, true] {
let mut command = command();
if use_cli {
command
.env("SIGNAL_ALERTS", "invalid")
.env("SIGNAL_ALERT_FROM_PHONE", "")
.env("SIGNAL_ALERT_TO_PHONE", "")
.args([
"--signal-alerts",
"true",
"--signal-alert-from-phone",
"+15555550101",
"--signal-alert-to-phone",
"+15555550102",
]);
} else {
command
.env("SIGNAL_ALERTS", "true")
.env("SIGNAL_ALERT_FROM_PHONE", "+15555550101")
.env("SIGNAL_ALERT_TO_PHONE", "+15555550102");
}
let output = command
.current_dir(dir.path())
.env("PATH", dir.path())
.env("SNAPSHOT_TEST_SCRIPT", &scenario)
.arg(&config)
.output()
.unwrap();
let stderr = String::from_utf8(output.stderr).unwrap();
assert!(output.status.success(), "{stderr}");
assert!(stderr.contains("snapshot_manager completed"), "{stderr}");
}
}
#[test]
fn disabled_alerts_allow_snapshot_failures_without_phone_numbers() {
let dir = tempdir().unwrap();
let config = dir.path().join("config.toml");
fs::write(&config, "").unwrap();
symlink(
Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/zfs"),
dir.path().join("zfs"),
)
.unwrap();
for (env_value, args) in [
(None, vec![]),
(Some("false"), vec![]),
(None, vec!["--signal-alerts", "false"]),
(Some("true"), vec!["--signal-alerts", "false"]),
] {
let mut command = command();
if let Some(value) = env_value {
command.env("SIGNAL_ALERTS", value);
}
let output = command
.env("PATH", dir.path())
.args(args)
.arg(&config)
.output()
.unwrap();
let stderr = String::from_utf8(output.stderr).unwrap();
assert!(output.status.success(), "{stderr}");
assert!(
stderr.contains("pool/home failed to create snapshot"),
"{stderr}"
);
assert!(stderr.contains("snapshot_manager completed"), "{stderr}");
assert!(!stderr.contains("Signal"), "{stderr}");
}
}
#[test]
fn disabled_alerts_preserve_fatal_errors_without_phone_numbers() {
let dir = tempdir().unwrap();
for args in [vec![], vec!["--signal-alerts", "false"]] {
let output = command()
.args(args)
.arg(dir.path().join("missing.toml"))
.output()
.unwrap();
let stderr = String::from_utf8(output.stderr).unwrap();
assert_eq!(output.status.code(), Some(1), "{stderr}");
assert!(stderr.contains("Failed to read configuration"), "{stderr}");
assert!(!stderr.contains("SIGNAL_ALERT_"), "{stderr}");
assert!(!stderr.contains("Failed to send failure alert"), "{stderr}");
}
}
#[test]
fn signal_alerts_requires_a_boolean_value() {
for use_env in [false, true] {
let mut command = command();
if use_env {
command.env("SIGNAL_ALERTS", "invalid");
} else {
command.args(["--signal-alerts", "invalid"]);
}
let output = command.arg("unused.toml").output().unwrap();
assert_eq!(output.status.code(), Some(2));
}
}
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
if [ -n "${SNAPSHOT_TEST_SCRIPT:-}" ]; then
printf '%s\n' "$*" >> calls
. "$SNAPSHOT_TEST_SCRIPT"
exit
fi
case "$*" in
'list -Hp -t filesystem -o name') printf '%s\n' pool/home ;;
'snapshot pool/home@'*) printf '%s\n' 'permission denied' >&2; exit 1 ;;
*) exit 99 ;;
esac
+290
View File
@@ -0,0 +1,290 @@
#![cfg(test)]
use std::{
fs,
os::unix::fs::symlink,
path::{Path, PathBuf},
process::{Command, Output},
};
use tempfile::{TempDir, tempdir};
struct Fixture {
dir: TempDir,
config: PathBuf,
}
impl Fixture {
fn new(script: &str) -> Self {
let dir = tempdir().unwrap();
// Only the child process sees this PATH. The real zfs cannot be found.
symlink(
Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/zfs"),
dir.path().join("zfs"),
)
.unwrap();
// The executable is a checked-in fixture; scenario files are read by
// its shell, so parallel tests never execute a file being written.
fs::write(dir.path().join("scenario"), script).unwrap();
let config = dir.path().join("config.toml");
fs::write(
&config,
"[default]\n15_min = 1\nhourly = 0\ndaily = 0\nmonthly = 0\n",
)
.unwrap();
Self { dir, config }
}
fn calls(&self) -> Vec<String> {
fs::read_to_string(self.dir.path().join("calls"))
.unwrap_or_default()
.lines()
.map(String::from)
.collect()
}
fn normalized_calls(&self) -> Vec<String> {
self.calls()
.into_iter()
.map(|call| {
if call.starts_with("snapshot ") {
let (dataset, name) = call.split_once('@').unwrap();
assert!(name.starts_with("auto_"), "{call}");
format!("{dataset}@<timestamp>")
} else {
call
}
})
.collect()
}
fn run(&self) -> Output {
Command::new(env!("CARGO_BIN_EXE_snapshot-manager"))
.current_dir(self.dir.path())
.env("PATH", self.dir.path())
.env("SNAPSHOT_TEST_SCRIPT", self.dir.path().join("scenario"))
.env_remove("SIGNAL_ALERTS")
.env_remove("SIGNAL_ALERT_FROM_PHONE")
.env_remove("SIGNAL_ALERT_TO_PHONE")
.env_remove("RUST_LOG")
.arg(&self.config)
.output()
.unwrap()
}
}
fn stderr(output: &Output) -> &str {
std::str::from_utf8(&output.stderr).unwrap()
}
#[test]
fn creates_then_prunes_only_this_datasets_managed_snapshots() {
let fixture = Fixture::new(
r#"
case "$*" in
'list -Hp -t filesystem -o name')
printf '%s\n' pool pool/home
printf '%s\n' 'warning on stderr' >&2 ;;
'snapshot pool/home@auto_'*) ;;
'list -Hp -t snapshot -o name -d 1 pool/home')
printf '%s\n' pool/home@auto_200002020145 pool/home@auto_200002020115 pool/home@manual pool/home@manual_auto_200001010015 pool/home/child@auto_200001010015 pool/other@auto_200001010015 ;;
'destroy pool/home@auto_200002020115') ;;
*) exit 99 ;;
esac
"#,
);
let output = fixture.run();
assert!(output.status.success(), "{}", stderr(&output));
assert_eq!(
fixture.normalized_calls(),
[
"list -Hp -t filesystem -o name",
"snapshot pool/home@<timestamp>",
"list -Hp -t snapshot -o name -d 1 pool/home",
"destroy pool/home@auto_200002020115",
]
);
}
#[test]
fn failed_or_duplicate_creation_skips_pruning_but_continues() {
for error in ["snapshot already exists", "permission denied"] {
let fixture = Fixture::new(&format!(
r#"
case "$*" in
'list -Hp -t filesystem -o name') printf '%s\n' pool/home pool/other ;;
'snapshot pool/home@auto_'*) printf '%s\n' '{error}' >&2; exit 1 ;;
'snapshot pool/other@auto_'*) ;;
'list -Hp -t snapshot -o name -d 1 pool/other') ;;
*) exit 99 ;;
esac
"#
));
let output = fixture.run();
assert!(output.status.success(), "{}", stderr(&output));
assert!(stderr(&output).contains("pool/home failed to create snapshot auto_"));
assert!(stderr(&output).contains(error));
assert_eq!(
fixture.normalized_calls(),
[
"list -Hp -t filesystem -o name",
"snapshot pool/home@<timestamp>",
"snapshot pool/other@<timestamp>",
"list -Hp -t snapshot -o name -d 1 pool/other",
]
);
let calls = fixture.calls();
assert_eq!(
calls[1].split_once('@').unwrap().1,
calls[2].split_once('@').unwrap().1
);
}
}
#[test]
fn dependent_clone_failure_is_reported_and_remaining_deletions_continue() {
let fixture = Fixture::new(
r#"
case "$*" in
'list -Hp -t filesystem -o name') printf '%s\n' pool/home ;;
'snapshot pool/home@auto_'*) ;;
'list -Hp -t snapshot -o name -d 1 pool/home')
printf '%s\n' pool/home@auto_200002020115 pool/home@auto_200002020130 pool/home@auto_200002020145 ;;
'destroy pool/home@auto_200002020115')
printf '%s\n' "cannot destroy 'pool/home@auto_200002020115': snapshot has dependent clones" >&2; exit 1 ;;
'destroy pool/home@auto_200002020130') ;;
*) exit 99 ;;
esac
"#,
);
let output = fixture.run();
assert!(output.status.success(), "{}", stderr(&output));
assert!(
stderr(&output).contains(
"pool/home@auto_200002020115 failed to delete: snapshot has dependent clones"
)
);
assert_eq!(
fixture.calls().last().unwrap(),
"destroy pool/home@auto_200002020130"
);
}
#[test]
fn unexpected_destroy_failure_aborts_and_reports_the_error() {
let fixture = Fixture::new(
r#"
case "$1" in
list)
if [ "$4" = filesystem ]; then
printf '%s\n' pool/home
else
printf '%s\n' pool/home@auto_200002020115 pool/home@auto_200002020130 pool/home@auto_200002020145
fi ;;
snapshot) ;;
destroy) printf '%s\n' 'permission denied' >&2; exit 1 ;;
esac
"#,
);
let output = fixture.run();
assert_eq!(output.status.code(), Some(1), "{}", stderr(&output));
assert!(stderr(&output).contains("permission denied"));
assert!(stderr(&output).contains("snapshot_manager failed"));
assert_eq!(
fixture.calls().last().unwrap(),
"destroy pool/home@auto_200002020115"
);
}
#[test]
fn failed_list_never_uses_partial_output() {
for stream in ["", " >&2"] {
let fixture = Fixture::new(&format!(
"printf '%s\\n' pool/home\nprintf '%s\\n' 'cannot list datasets'{stream}\nexit 1"
));
let output = fixture.run();
assert_eq!(output.status.code(), Some(1), "{}", stderr(&output));
assert!(stderr(&output).contains("cannot list datasets"));
assert!(stderr(&output).contains("snapshot_manager failed"));
assert_eq!(fixture.calls(), ["list -Hp -t filesystem -o name"]);
}
}
#[test]
fn failed_snapshot_listing_aborts_without_destroying_anything() {
let fixture = Fixture::new(
r#"
case "$*" in
'list -Hp -t filesystem -o name') printf '%s\n' pool/home ;;
'snapshot pool/home@auto_'*) ;;
*) printf '%s\n' pool/home@auto_200002020115; exit 1 ;;
esac
"#,
);
let output = fixture.run();
assert_eq!(output.status.code(), Some(1), "{}", stderr(&output));
assert!(stderr(&output).contains("snapshot_manager failed"));
assert_eq!(fixture.calls().len(), 3);
}
#[test]
fn invalid_config_stops_before_any_zfs_commands() {
let fixture = Fixture::new("exit 99");
fs::write(
&fixture.config,
"[default]\n15_min = -1\nhourly = 1\ndaily = 0\nmonthly = 0",
)
.unwrap();
let output = fixture.run();
assert_eq!(output.status.code(), Some(1), "{}", stderr(&output));
assert!(stderr(&output).contains("Invalid configuration"));
assert!(fixture.calls().is_empty());
}
#[test]
fn missing_zfs_executable_fails_and_reports_the_error() {
let fixture = Fixture::new("exit 99");
fs::remove_file(fixture.dir.path().join("zfs")).unwrap();
let output = fixture.run();
assert_eq!(output.status.code(), Some(1), "{}", stderr(&output));
assert!(stderr(&output).contains("Failed to execute zfs"));
assert!(stderr(&output).contains("snapshot_manager failed"));
assert!(fixture.calls().is_empty());
}
#[test]
fn dataset_names_are_passed_as_one_argument_without_a_shell() {
let fixture = Fixture::new(
r#"
case "$1" in
list)
[ "$*" = 'list -Hp -t filesystem -o name' ] || exit 98
printf '%s\n' 'pool/two words;touch injected' ;;
snapshot)
[ "$#" -eq 2 ] || exit 97
case "$2" in
'pool/two words;touch injected@auto_'*) printf '%s\n' 'argument preserved' >&2; exit 1 ;;
*) exit 96 ;;
esac ;;
*) exit 99 ;;
esac
"#,
);
let output = fixture.run();
assert!(output.status.success(), "{}", stderr(&output));
assert!(stderr(&output).contains("argument preserved"));
assert_eq!(
fixture.normalized_calls(),
[
"list -Hp -t filesystem -o name",
"snapshot pool/two words;touch injected@<timestamp>"
]
);
assert!(fs::read_dir(fixture.dir.path()).unwrap().all(|entry| {
!entry
.unwrap()
.file_name()
.to_string_lossy()
.starts_with("injected")
}));
}
+1 -9
View File
@@ -2,15 +2,7 @@
rustPlatform.buildRustPackage { rustPlatform.buildRustPackage {
pname = "van-weather"; pname = "van-weather";
version = "0.1.0"; version = "0.1.0";
src = lib.fileset.toSource { src = import ../rust-source.nix { inherit lib; };
root = ../..;
fileset = lib.fileset.unions [
../../Cargo.toml
../../Cargo.lock
../../clippy.toml
../.
];
};
cargoLock.lockFile = ../../Cargo.lock; cargoLock.lockFile = ../../Cargo.lock;
cargoBuildFlags = [ cargoBuildFlags = [
+1
View File
@@ -107,6 +107,7 @@ let
nixos-rebuild nixos-rebuild
nodejs nodejs
ruff ruff
rustfmt
treefmt treefmt
wget wget
]; ];
-204
View File
@@ -1,204 +0,0 @@
"""test_snapshot_manager."""
from __future__ import annotations
from datetime import UTC, datetime
from pathlib import Path
from typing import TYPE_CHECKING
import pytest
from python.tools.snapshot_manager import get_snapshots_to_delete, get_time_stamp, load_config_data, main
from python.zfs.dataset import Dataset, Snapshot
if TYPE_CHECKING:
from pyfakefs.fake_filesystem import FakeFilesystem
from pytest_mock import MockerFixture
SNAPSHOT_MANAGER = "python.tools.snapshot_manager"
FROM_PHONE = "1234567890"
TO_PHONE = "0987654321"
def patch_phone_numbers(mocker: MockerFixture) -> None:
"""Patch the Signal phone number environment variables."""
phone_numbers = {
"SIGNAL_ALERT_FROM_PHONE": FROM_PHONE,
"SIGNAL_ALERT_TO_PHONE": TO_PHONE,
}
mocker.patch(f"{SNAPSHOT_MANAGER}.getenv", side_effect=phone_numbers.get)
def patch_utcnow(mocker: MockerFixture, datetime_value: datetime) -> None:
"""patch_utcnow."""
mocker.patch("python.tools.snapshot_manager.utcnow", return_value=datetime_value)
def create_mock_snapshot(mocker: MockerFixture, name: str) -> Snapshot:
"""create_mock_snapshot."""
mock_snapshot = mocker.MagicMock(spec=Snapshot)
mock_snapshot.name = name
return mock_snapshot
def test_main(mocker: MockerFixture, fs: FakeFilesystem) -> None:
"""Test main."""
load_config_data.cache_clear()
patch_phone_numbers(mocker)
mocker.patch(f"{SNAPSHOT_MANAGER}.get_time_stamp", return_value="2023-01-01T00:00:00")
mock_dataset = mocker.MagicMock(spec=Dataset)
mock_dataset.name = "test_dataset"
mock_dataset.create_snapshot.return_value = "snapshot created"
mock_get_datasets = mocker.patch(f"{SNAPSHOT_MANAGER}.get_datasets", return_value=(mock_dataset,))
mock_get_snapshots_to_delete = mocker.patch(f"{SNAPSHOT_MANAGER}.get_snapshots_to_delete")
mock_signal_alert = mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
mock_snapshot_config_toml = '["default"]\n15_min = 8\nhourly = 24\ndaily = 0\nmonthly = 0\n'
fs.create_file("/mock_snapshot_config.toml", contents=mock_snapshot_config_toml)
main(Path("/mock_snapshot_config.toml"))
mock_signal_alert.assert_not_called()
mock_get_datasets.assert_called_once()
mock_get_snapshots_to_delete.assert_called_once_with(
mock_dataset,
{
"15_min": 8,
"hourly": 24,
"daily": 0,
"monthly": 0,
},
FROM_PHONE,
TO_PHONE,
)
def test_main_create_snapshot_failure(mocker: MockerFixture, fs: FakeFilesystem) -> None:
"""Test main."""
load_config_data.cache_clear()
patch_phone_numbers(mocker)
mocker.patch(f"{SNAPSHOT_MANAGER}.get_time_stamp", return_value="2023-01-01T00:00:00")
mock_dataset = mocker.MagicMock(spec=Dataset)
mock_dataset.name = "test_dataset"
mock_dataset.create_snapshot.return_value = "snapshot not created"
mock_get_datasets = mocker.patch(f"{SNAPSHOT_MANAGER}.get_datasets", return_value=(mock_dataset,))
mock_get_snapshots_to_delete = mocker.patch(f"{SNAPSHOT_MANAGER}.get_snapshots_to_delete")
mock_signal_alert = mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
mock_snapshot_config_toml = '["default"]\n15_min = 8\nhourly = 24\ndaily = 0\nmonthly = 0\n'
fs.create_file("/mock_snapshot_config.toml", contents=mock_snapshot_config_toml)
main(Path("/mock_snapshot_config.toml"))
mock_signal_alert.assert_called_once_with(
"test_dataset failed to create snapshot 2023-01-01T00:00:00",
FROM_PHONE,
TO_PHONE,
)
mock_get_datasets.assert_called_once()
mock_get_snapshots_to_delete.assert_not_called()
def test_main_exception(mocker: MockerFixture, fs: FakeFilesystem) -> None:
"""Test main."""
load_config_data.cache_clear()
patch_phone_numbers(mocker)
mocker.patch(f"{SNAPSHOT_MANAGER}.get_time_stamp", return_value="2023-01-01T00:00:00")
mock_dataset = mocker.MagicMock(spec=Dataset)
mock_dataset.name = "test_dataset"
mock_dataset.create_snapshot.return_value = "snapshot created"
mock_get_datasets = mocker.patch(f"{SNAPSHOT_MANAGER}.get_datasets", side_effect=Exception("test"))
mock_get_snapshots_to_delete = mocker.patch(f"{SNAPSHOT_MANAGER}.get_snapshots_to_delete")
mock_signal_alert = mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
mock_snapshot_config_toml = '["default"]\n15_min = 8\nhourly = 24\ndaily = 0\nmonthly = 0\n'
fs.create_file("/mock_snapshot_config.toml", contents=mock_snapshot_config_toml)
with pytest.raises(SystemExit) as pytest_wrapped_e:
main(Path("/mock_snapshot_config.toml"))
assert isinstance(pytest_wrapped_e.value, SystemExit)
assert pytest_wrapped_e.value.code == 1
mock_signal_alert.assert_called_once_with("snapshot_manager failed", FROM_PHONE, TO_PHONE)
mock_get_datasets.assert_called_once()
mock_get_snapshots_to_delete.assert_not_called()
def test_get_snapshots_to_delete(mocker: MockerFixture) -> None:
"""test_get_snapshots_to_delete."""
mock_snapshot_0 = create_mock_snapshot(mocker, "auto_202509150415")
mock_snapshot_1 = create_mock_snapshot(mocker, "auto_202509150415")
mock_dataset = mocker.MagicMock(spec=Dataset)
mock_dataset.name = "test_dataset"
mock_dataset.get_snapshots.return_value = (mock_snapshot_0, mock_snapshot_1)
mock_dataset.delete_snapshot.return_value = None
mock_signal_alert = mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
get_snapshots_to_delete(
mock_dataset,
{"15_min": 1, "hourly": 0, "daily": 0, "monthly": 0},
FROM_PHONE,
TO_PHONE,
)
mock_signal_alert.assert_not_called()
mock_dataset.delete_snapshot.assert_called_once_with("auto_202509150415")
def test_get_snapshots_to_delete_no_snapshot(mocker: MockerFixture) -> None:
"""test_get_snapshots_to_delete_no_snapshot."""
mock_dataset = mocker.MagicMock(spec=Dataset)
mock_dataset.name = "test_dataset"
mock_dataset.get_snapshots.return_value = ()
mock_dataset.delete_snapshot.return_value = None
mock_signal_alert = mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
get_snapshots_to_delete(
mock_dataset,
{"15_min": 1, "hourly": 0, "daily": 0, "monthly": 0},
FROM_PHONE,
TO_PHONE,
)
mock_signal_alert.assert_not_called()
mock_dataset.delete_snapshot.assert_not_called()
def test_get_snapshots_to_delete_errored(mocker: MockerFixture) -> None:
"""test_get_snapshots_to_delete_errored."""
mock_snapshot_0 = create_mock_snapshot(mocker, "auto_202509150415")
mock_snapshot_1 = create_mock_snapshot(mocker, "auto_202509150415")
mock_dataset = mocker.MagicMock(spec=Dataset)
mock_dataset.name = "test_dataset"
mock_dataset.get_snapshots.return_value = (mock_snapshot_0, mock_snapshot_1)
mock_dataset.delete_snapshot.return_value = "snapshot has dependent clones"
mock_signal_alert = mocker.patch(f"{SNAPSHOT_MANAGER}.signal_alert")
get_snapshots_to_delete(
mock_dataset,
{"15_min": 1, "hourly": 0, "daily": 0, "monthly": 0},
FROM_PHONE,
TO_PHONE,
)
mock_signal_alert.assert_called_once_with(
"test_dataset@auto_202509150415 failed to delete: snapshot has dependent clones",
FROM_PHONE,
TO_PHONE,
)
mock_dataset.delete_snapshot.assert_called_once_with("auto_202509150415")
def test_get_time_stamp(mocker: MockerFixture) -> None:
"""Test get_time_stamp."""
patch_utcnow(mocker, datetime(2023, 1, 1, 0, 0, 0, tzinfo=UTC))
assert get_time_stamp() == "auto_202301010000"
+5
View File
@@ -13,6 +13,11 @@ command = "nixfmt"
# Glob pattern of files to include # Glob pattern of files to include
includes = ["*.nix"] includes = ["*.nix"]
[formatter.rustfmt]
command = "rustfmt"
options = ["--edition", "2024"]
includes = ["*.rs"]
[formatter.ruff-format] [formatter.ruff-format]
command = "ruff" command = "ruff"
options = ["format"] options = ["format"]