Implement ZFS dataset management and snapshot configuration
- Introduced a new `datasets.nix` file to define ZFS datasets and their properties, allowing for centralized management of dataset configurations. - Updated `default.nix` to import and enable the `zfs_manager` service, integrating it into the system configuration. - Refactored `zfs.sh` to remove dataset creation commands, delegating dataset management to the `zfs_manager` service. - Removed the legacy `snapshot_config.toml` file, as snapshot configurations are now handled within `datasets.nix`. - Modified `vars.nix` to derive mountpoint paths from `datasets.nix`, ensuring consistency across the configuration. - Created a new `zfs.nix` file to define the `zfs_manager` service and its dependencies. - Added comprehensive tests for the `zfs_manager` functionality, covering dataset creation, property management, and error handling.
This commit is contained in:
@@ -0,0 +1,362 @@
|
||||
# Dataset declarations for jeeves, kept as plain data rather than inside
|
||||
# zfs.nix so the dataset tree stays separate from the NixOS service wiring.
|
||||
# Datasets are nested the way zfs nests them: a pool holds datasets, which can
|
||||
# hold datasets of their own. The tree is flattened into "pool/parent/child"
|
||||
# names below, which is what zfs and services.zfs_manager work in.
|
||||
#
|
||||
# Consumed by ./zfs.nix, which feeds it to services.zfs_manager.
|
||||
let
|
||||
# Every pool on jeeves was created with the same -O options.
|
||||
poolDefaults = mountpoint: {
|
||||
inherit mountpoint;
|
||||
acltype = "posix"; # zfs reports posixacl back as posix
|
||||
atime = "off";
|
||||
compression = "zstd";
|
||||
dnodesize = "auto";
|
||||
xattr = "sa";
|
||||
};
|
||||
|
||||
zfsKey = "file:///root/zfs.key";
|
||||
|
||||
# What a dataset gets when it is not called out below, kept identical to the
|
||||
# "default" table so the datasets that used to fall through are unchanged.
|
||||
standard = {
|
||||
"15_min" = 8;
|
||||
hourly = 24;
|
||||
};
|
||||
|
||||
disabledSnapshots = {
|
||||
"15_min" = 0;
|
||||
hourly = 0;
|
||||
daily = 0;
|
||||
monthly = 0;
|
||||
};
|
||||
|
||||
pools = {
|
||||
# root_pool: retention only, its properties are not managed yet.
|
||||
root_pool = {
|
||||
manageProperties = false;
|
||||
datasets = {
|
||||
home = {
|
||||
manageProperties = false;
|
||||
snapshots = {
|
||||
"15_min" = 8;
|
||||
hourly = 24;
|
||||
daily = 14;
|
||||
};
|
||||
};
|
||||
root = {
|
||||
manageProperties = false;
|
||||
snapshots = standard;
|
||||
};
|
||||
nix = {
|
||||
manageProperties = false;
|
||||
snapshots."15_min" = 4;
|
||||
};
|
||||
var = {
|
||||
manageProperties = false;
|
||||
snapshots = {
|
||||
"15_min" = 8;
|
||||
hourly = 24;
|
||||
daily = 30;
|
||||
monthly = 6;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
media = {
|
||||
properties = poolDefaults "/zfs/media";
|
||||
datasets = {
|
||||
temp = {
|
||||
properties = {
|
||||
redundant_metadata = "none";
|
||||
sync = "disabled";
|
||||
};
|
||||
snapshots."15_min" = 2;
|
||||
};
|
||||
secure = {
|
||||
# An encryption root provisioned by scripts/zfs.sh. Its create-only
|
||||
# properties are declared for verification, but zfs_manager must
|
||||
# never create it automatically.
|
||||
createIfMissing = true;
|
||||
properties = {
|
||||
encryption = "aes-256-gcm";
|
||||
keyformat = "hex";
|
||||
keylocation = zfsKey;
|
||||
};
|
||||
snapshots = disabledSnapshots;
|
||||
datasets = {
|
||||
docker = {
|
||||
properties = {
|
||||
mountpoint = "/zfs/media/docker";
|
||||
compression = "zstd-9";
|
||||
};
|
||||
snapshots = {
|
||||
"15_min" = 3;
|
||||
hourly = 12;
|
||||
daily = 14;
|
||||
monthly = 2;
|
||||
};
|
||||
};
|
||||
"github-runners" = {
|
||||
properties = {
|
||||
mountpoint = "/zfs/media/github-runners";
|
||||
compression = "zstd-9";
|
||||
sync = "disabled";
|
||||
};
|
||||
snapshots = {
|
||||
"15_min" = 6;
|
||||
hourly = 2;
|
||||
daily = 1;
|
||||
};
|
||||
};
|
||||
home_assistant = {
|
||||
properties = {
|
||||
mountpoint = "/zfs/media/home_assistant";
|
||||
compression = "zstd-19";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
important = {
|
||||
properties = {
|
||||
compression = "zstd-9";
|
||||
copies = "2";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
notes = {
|
||||
properties = {
|
||||
mountpoint = "/zfs/media/notes";
|
||||
copies = "2";
|
||||
};
|
||||
snapshots = {
|
||||
"15_min" = 8;
|
||||
hourly = 24;
|
||||
daily = 30;
|
||||
monthly = 12;
|
||||
};
|
||||
};
|
||||
postgres = {
|
||||
properties = {
|
||||
mountpoint = "/zfs/media/database/postgres";
|
||||
primarycache = "metadata";
|
||||
recordsize = "16K";
|
||||
};
|
||||
snapshots = {
|
||||
"15_min" = 8;
|
||||
hourly = 24;
|
||||
daily = 7;
|
||||
};
|
||||
};
|
||||
"postgres-wal" = {
|
||||
properties = {
|
||||
compression = "lz4";
|
||||
logbias = "latency";
|
||||
mountpoint = "/zfs/media/database/postgres-wal";
|
||||
primarycache = "metadata";
|
||||
recordsize = "32K";
|
||||
secondarycache = "none";
|
||||
special_small_blocks = "32K";
|
||||
};
|
||||
snapshots = {
|
||||
"15_min" = 4;
|
||||
hourly = 2;
|
||||
};
|
||||
};
|
||||
prometheus = {
|
||||
properties = {
|
||||
mountpoint = "/zfs/media/database/prometheus";
|
||||
compression = "lz4";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
services = {
|
||||
properties = {
|
||||
mountpoint = "/zfs/media/services";
|
||||
compression = "zstd-9";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
share = {
|
||||
properties = {
|
||||
mountpoint = "/zfs/media/share";
|
||||
exec = "off";
|
||||
};
|
||||
snapshots."15_min" = 4;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
storage = {
|
||||
properties = poolDefaults "/zfs/storage";
|
||||
datasets = {
|
||||
nomad = {
|
||||
properties = {
|
||||
mountpoint = "/zfs/storage/nomad";
|
||||
compression = "zstd-9";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
ollama = {
|
||||
properties = {
|
||||
compression = "zstd-19";
|
||||
recordsize = "1M";
|
||||
sync = "disabled";
|
||||
};
|
||||
snapshots."15_min" = 2;
|
||||
};
|
||||
secure = {
|
||||
# An encryption root provisioned by scripts/zfs.sh. Its create-only
|
||||
# properties are declared for verification, but zfs_manager must
|
||||
# never create it automatically.
|
||||
createIfMissing = false;
|
||||
properties = {
|
||||
encryption = "aes-256-gcm";
|
||||
keyformat = "hex";
|
||||
keylocation = zfsKey;
|
||||
};
|
||||
snapshots = disabledSnapshots;
|
||||
datasets = {
|
||||
archive = {
|
||||
properties = {
|
||||
compression = "zstd-19";
|
||||
mountpoint = "/zfs/storage/archive";
|
||||
recordsize = "1M";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
important = {
|
||||
properties = {
|
||||
compression = "zstd-19";
|
||||
copies = "2";
|
||||
mountpoint = "/zfs/storage/important";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
library = {
|
||||
properties = {
|
||||
compression = "zstd-19";
|
||||
mountpoint = "/zfs/storage/library";
|
||||
recordsize = "1M";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
main = {
|
||||
properties = {
|
||||
compression = "zstd-19";
|
||||
mountpoint = "/zfs/storage/main";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
photos = {
|
||||
properties = {
|
||||
compression = "zstd-19";
|
||||
copies = "2";
|
||||
mountpoint = "/zfs/storage/photos";
|
||||
recordsize = "16K";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
plex = {
|
||||
properties = {
|
||||
compression = "zstd-19";
|
||||
mountpoint = "/zfs/storage/plex";
|
||||
recordsize = "1M";
|
||||
};
|
||||
snapshots = {
|
||||
"15_min" = 6;
|
||||
hourly = 2;
|
||||
daily = 1;
|
||||
};
|
||||
};
|
||||
secrets = {
|
||||
properties = {
|
||||
compression = "zstd-19";
|
||||
copies = "3";
|
||||
mountpoint = "/zfs/storage/secrets";
|
||||
};
|
||||
snapshots = {
|
||||
"15_min" = 8;
|
||||
hourly = 24;
|
||||
daily = 30;
|
||||
monthly = 12;
|
||||
};
|
||||
};
|
||||
syncthing = {
|
||||
properties = {
|
||||
compression = "zstd-19";
|
||||
mountpoint = "/zfs/storage/syncthing";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
transmission = {
|
||||
properties = {
|
||||
compression = "zstd-9";
|
||||
exec = "off";
|
||||
mountpoint = "/zfs/storage/transmission";
|
||||
recordsize = "1M";
|
||||
sync = "disabled";
|
||||
};
|
||||
snapshots."15_min" = 4;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
scratch = {
|
||||
properties = poolDefaults "/zfs/scratch" // {
|
||||
encryption = "aes-256-gcm";
|
||||
keyformat = "hex";
|
||||
keylocation = zfsKey;
|
||||
};
|
||||
datasets = {
|
||||
kafka = {
|
||||
properties = {
|
||||
mountpoint = "/zfs/scratch/kafka";
|
||||
recordsize = "1M";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
kestra = {
|
||||
properties = {
|
||||
mountpoint = "/zfs/scratch/kestra";
|
||||
sync = "disabled";
|
||||
};
|
||||
snapshots = standard;
|
||||
};
|
||||
transmission = {
|
||||
properties = {
|
||||
mountpoint = "/zfs/scratch/transmission";
|
||||
recordsize = "16K";
|
||||
sync = "disabled";
|
||||
};
|
||||
snapshots."15_min" = 2;
|
||||
};
|
||||
uv_cache = {
|
||||
properties.mountpoint = "/zfs/scratch/uv_cache";
|
||||
snapshots."15_min" = 2;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# Collapse the tree into the flat "pool/parent/child" names zfs uses. Each
|
||||
# node keeps everything except its children.
|
||||
flatten =
|
||||
name: node:
|
||||
builtins.foldl' (result: child: result // flatten "${name}/${child}" node.datasets.${child}) {
|
||||
${name} = builtins.removeAttrs node [ "datasets" ];
|
||||
} (builtins.attrNames (node.datasets or { }));
|
||||
|
||||
datasets = builtins.foldl' (result: pool: result // flatten pool pools.${pool}) { } (
|
||||
builtins.attrNames pools
|
||||
);
|
||||
in
|
||||
{
|
||||
inherit datasets;
|
||||
defaultSnapshots = standard;
|
||||
}
|
||||
@@ -1,7 +1,4 @@
|
||||
{ inputs, ... }:
|
||||
let
|
||||
vars = import ./vars.nix;
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
"${inputs.self}/users/dov"
|
||||
@@ -15,6 +12,7 @@ in
|
||||
"${inputs.self}/common/optional/syncthing_base.nix"
|
||||
"${inputs.self}/common/optional/update.nix"
|
||||
"${inputs.self}/common/optional/zerotier.nix"
|
||||
"${inputs.self}/common/optional/zfs_manager.nix"
|
||||
./monitoring
|
||||
./docker
|
||||
./services
|
||||
@@ -24,6 +22,7 @@ in
|
||||
./programs.nix
|
||||
./runners
|
||||
./syncthing.nix
|
||||
./zfs.nix
|
||||
];
|
||||
|
||||
services = {
|
||||
@@ -31,11 +30,6 @@ in
|
||||
|
||||
smartd.enable = true;
|
||||
|
||||
snapshot_manager = {
|
||||
path = ./snapshot_config.toml;
|
||||
EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
|
||||
};
|
||||
|
||||
zerotierone.joinNetworks = [ "a09acf02330d37b9" ];
|
||||
};
|
||||
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Pool and vdev creation only. This is run by hand once per pool.
|
||||
#
|
||||
# Datasets and their properties are declared in systems/jeeves/zfs.nix and
|
||||
# reconciled by the zfs_manager service. Do not add zfs create lines here.
|
||||
|
||||
# zpools
|
||||
|
||||
# media
|
||||
@@ -12,34 +17,10 @@ sudo zpool add storage -o ashift=12 special mirror
|
||||
sudo zpool add storage -o ashift=12 logs mirror
|
||||
|
||||
# scratch
|
||||
sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///key -m /zfs/scratch
|
||||
sudo zpool create scratch -o ashift=12 -O acltype=posixacl -O atime=off -O dnodesize=auto -O xattr=sa -O compression=zstd -O encryption=aes-256-gcm -O keyformat=hex -O keylocation=file:///root/zfs.key -m /zfs/scratch
|
||||
|
||||
# media datasets
|
||||
sudo zfs create media/temp -o sync=disabled -o redundant_metadata=none
|
||||
# The two encrypted parent datasets have to exist before zfs_manager can create
|
||||
# anything under them, since encryption cannot be set after creation.
|
||||
# These will be removed if/when the media and storage pools are encrypted in the future.
|
||||
sudo zfs create media/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
|
||||
sudo zfs create media/secure/docker -o compression=zstd-9
|
||||
sudo zfs create media/secure/github-runners -o compression=zstd-9 -o sync=disabled
|
||||
sudo zfs create media/secure/notes -o copies=2
|
||||
sudo zfs create media/secure/postgres -o mountpoint=/zfs/media/database/postgres -o recordsize=16k -o primarycache=metadata
|
||||
sudo zfs create media/secure/postgres-wal -o mountpoint=/zfs/media/database/postgres-wal -o recordsize=32k -o primarycache=metadata -o special_small_blocks=32K -o compression=lz4 -o secondarycache=none -o logbias=latency
|
||||
sudo zfs create media/secure/prometheus -o mountpoint=/zfs/media/database/prometheus -o compression=lz4
|
||||
sudo zfs create media/secure/services -o compression=zstd-9
|
||||
sudo zfs create media/secure/share -o mountpoint=/zfs/media/share -o exec=off
|
||||
|
||||
# scratch datasets
|
||||
sudo zfs create scratch/kafka -o mountpoint=/zfs/scratch/kafka -o recordsize=1M
|
||||
sudo zfs create scratch/transmission -o mountpoint=/zfs/scratch/transmission -o recordsize=16k -o sync=disabled -o redundant_metadata=none
|
||||
sudo zfs create scratch/uv_cache -o mountpoint=/zfs/scratch/uv_cache
|
||||
|
||||
# storage datasets
|
||||
sudo zfs create storage/ollama -o recordsize=1M -o compression=zstd-19 -o sync=disabled
|
||||
sudo zfs create storage/secure -o encryption=aes-256-gcm -o keyformat=hex -o keylocation=file:///root/zfs.key
|
||||
sudo zfs create storage/secure/archive -o recordsize=1M -o compression=zstd-19
|
||||
sudo zfs create storage/secure/library -o recordsize=1M -o compression=zstd-19
|
||||
sudo zfs create storage/secure/main -o compression=zstd-19
|
||||
sudo zfs create storage/secure/photos -o recordsize=16K -o compression=zstd-19 -o copies=2
|
||||
sudo zfs create storage/secure/plex -o recordsize=1M -o compression=zstd-19
|
||||
sudo zfs create storage/secure/secrets -o compression=zstd-19 -o copies=3
|
||||
sudo zfs create storage/secure/syncthing -o compression=zstd-19
|
||||
sudo zfs create storage/secure/transmission -o recordsize=1M -o compression=zstd-9 -o exec=off -o sync=disabled
|
||||
sudo zfs create storage/secure/important -o compression=zstd-19 -o copies=2 -o mountpoint=/zfs/storage/important
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
["default"]
|
||||
15_min = 8
|
||||
hourly = 24
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
# root_pool
|
||||
["root_pool/home"]
|
||||
15_min = 8
|
||||
hourly = 24
|
||||
daily = 14
|
||||
monthly = 0
|
||||
|
||||
["root_pool/root"]
|
||||
15_min = 8
|
||||
hourly = 24
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["root_pool/nix"]
|
||||
15_min = 4
|
||||
hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["root_pool/var"]
|
||||
15_min = 8
|
||||
hourly = 24
|
||||
daily = 30
|
||||
monthly = 6
|
||||
# storage
|
||||
["storage/ollama"]
|
||||
15_min = 2
|
||||
hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["storage/secure"]
|
||||
15_min = 0
|
||||
hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["storage/secure/plex"]
|
||||
15_min = 6
|
||||
hourly = 2
|
||||
daily = 1
|
||||
monthly = 0
|
||||
|
||||
["storage/secure/transmission"]
|
||||
15_min = 4
|
||||
hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["storage/secure/secrets"]
|
||||
15_min = 8
|
||||
hourly = 24
|
||||
daily = 30
|
||||
monthly = 12
|
||||
|
||||
# media
|
||||
["media/temp"]
|
||||
15_min = 2
|
||||
hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["media/secure"]
|
||||
15_min = 0
|
||||
hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["media/secure/plex"]
|
||||
15_min = 6
|
||||
hourly = 2
|
||||
daily = 1
|
||||
monthly = 0
|
||||
|
||||
["media/secure/postgres-wal"]
|
||||
15_min = 4
|
||||
hourly = 2
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
|
||||
["media/secure/postgres"]
|
||||
15_min = 8
|
||||
hourly = 24
|
||||
daily = 7
|
||||
monthly = 0
|
||||
|
||||
["media/secure/share"]
|
||||
15_min = 4
|
||||
hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["media/secure/github-runners"]
|
||||
15_min = 6
|
||||
hourly = 2
|
||||
daily = 1
|
||||
monthly = 0
|
||||
|
||||
["media/secure/notes"]
|
||||
15_min = 8
|
||||
hourly = 24
|
||||
daily = 30
|
||||
monthly = 12
|
||||
|
||||
["media/secure/docker"]
|
||||
15_min = 3
|
||||
hourly = 12
|
||||
daily = 14
|
||||
monthly = 2
|
||||
|
||||
# scratch
|
||||
["scratch/transmission"]
|
||||
15_min = 2
|
||||
hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
|
||||
["scratch/uv_cache"]
|
||||
15_min = 2
|
||||
hourly = 0
|
||||
daily = 0
|
||||
monthly = 0
|
||||
@@ -0,0 +1,20 @@
|
||||
{ inputs, ... }:
|
||||
let
|
||||
vars = import ./vars.nix;
|
||||
jeeves_zfs = import ./datasets.nix;
|
||||
in
|
||||
{
|
||||
services = {
|
||||
zfs_manager = {
|
||||
enable = true;
|
||||
PYTHONPATH = "${inputs.self}/";
|
||||
EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
|
||||
|
||||
inherit (jeeves_zfs) datasets defaultSnapshots;
|
||||
};
|
||||
|
||||
# Its retention config is generated from ./datasets.nix by
|
||||
# common/optional/zfs_manager.nix, so only the credentials are set here.
|
||||
snapshot_manager.EnvironmentFile = "${vars.secrets}/services/snapshot_manager";
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user