Richie
aa552ecca1
treefmt / nix fmt (pull_request) Failing after 5s
pytest / pytest (pull_request) Successful in 30s
test ebook search / test-ebook-search (pull_request) Successful in 36s
build_systems / build-bob (pull_request) Successful in 49s
build_systems / build-brain (pull_request) Successful in 49s
build_systems / build-rhapsody-in-green (pull_request) Successful in 1m1s
build_systems / build-jeeves (pull_request) Successful in 2m18s
- Introduced a new `datasets.nix` file to define ZFS datasets and their properties, allowing for centralized management of dataset configurations. - Updated `default.nix` to import and enable the `zfs_manager` service, integrating it into the system configuration. - Refactored `zfs.sh` to remove dataset creation commands, delegating dataset management to the `zfs_manager` service. - Removed the legacy `snapshot_config.toml` file, as snapshot configurations are now handled within `datasets.nix`. - Modified `vars.nix` to derive mountpoint paths from `datasets.nix`, ensuring consistency across the configuration. - Created a new `zfs.nix` file to define the `zfs_manager` service and its dependencies. - Added comprehensive tests for the `zfs_manager` functionality, covering dataset creation, property management, and error handling.
365 lines
11 KiB
Nix
365 lines
11 KiB
Nix
# Dataset declarations for jeeves, kept as plain data rather than inside
|
|
# zfs.nix so that vars.nix can derive its paths from the same source.
|
|
#
|
|
# Datasets are nested the way zfs nests them: a pool holds datasets, which can
|
|
# hold datasets of their own. The tree is flattened into "pool/parent/child"
|
|
# names below, which is what zfs and services.zfs_manager work in.
|
|
#
|
|
# Consumed by ./zfs.nix (which feeds it to services.zfs_manager) and by
|
|
# ./vars.nix (which resolves the mountpoints).
|
|
let
|
|
# Every pool on jeeves was created with the same -O options.
|
|
poolDefaults = mountpoint: {
|
|
inherit mountpoint;
|
|
acltype = "posix"; # zfs reports posixacl back as posix
|
|
atime = "off";
|
|
compression = "zstd";
|
|
dnodesize = "auto";
|
|
xattr = "sa";
|
|
};
|
|
|
|
zfsKey = "file:///root/zfs.key";
|
|
|
|
# What a dataset gets when it is not called out below, kept identical to the
|
|
# "default" table so the datasets that used to fall through are unchanged.
|
|
standard = {
|
|
"15_min" = 8;
|
|
hourly = 24;
|
|
};
|
|
|
|
pools = {
|
|
# root_pool: retention only, its properties are not managed yet.
|
|
root_pool = {
|
|
manageProperties = false;
|
|
datasets = {
|
|
home = {
|
|
manageProperties = false;
|
|
snapshots = {
|
|
"15_min" = 8;
|
|
hourly = 24;
|
|
daily = 14;
|
|
};
|
|
};
|
|
root = {
|
|
manageProperties = false;
|
|
snapshots = standard;
|
|
};
|
|
nix = {
|
|
manageProperties = false;
|
|
snapshots."15_min" = 4;
|
|
};
|
|
var = {
|
|
manageProperties = false;
|
|
snapshots = {
|
|
"15_min" = 8;
|
|
hourly = 24;
|
|
daily = 30;
|
|
monthly = 6;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
media = {
|
|
properties = poolDefaults "/zfs/media";
|
|
datasets = {
|
|
temp = {
|
|
properties = {
|
|
redundant_metadata = "none";
|
|
sync = "disabled";
|
|
};
|
|
snapshots."15_min" = 2;
|
|
};
|
|
secure = {
|
|
# An encryption root. zfs_manager must never create this: encryption
|
|
# is fixed at creation and is not declared here, so an automatic
|
|
# create would produce an unencrypted dataset. See scripts/zfs.sh.
|
|
createIfMissing = false;
|
|
properties.keylocation = zfsKey;
|
|
snapshots = { };
|
|
datasets = {
|
|
docker = {
|
|
properties = {
|
|
mountpoint = "/zfs/media/docker";
|
|
compression = "zstd-9";
|
|
};
|
|
snapshots = {
|
|
"15_min" = 3;
|
|
hourly = 12;
|
|
daily = 14;
|
|
monthly = 2;
|
|
};
|
|
};
|
|
"github-runners" = {
|
|
properties = {
|
|
mountpoint = "/zfs/media/github-runners";
|
|
compression = "zstd-9";
|
|
sync = "disabled";
|
|
};
|
|
snapshots = {
|
|
"15_min" = 6;
|
|
hourly = 2;
|
|
daily = 1;
|
|
};
|
|
};
|
|
home_assistant = {
|
|
properties = {
|
|
mountpoint = "/zfs/media/home_assistant";
|
|
compression = "zstd-19";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
important = {
|
|
properties = {
|
|
compression = "zstd-9";
|
|
copies = "2";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
notes = {
|
|
properties = {
|
|
mountpoint = "/zfs/media/notes";
|
|
copies = "2";
|
|
};
|
|
snapshots = {
|
|
"15_min" = 8;
|
|
hourly = 24;
|
|
daily = 30;
|
|
monthly = 12;
|
|
};
|
|
};
|
|
postgres = {
|
|
properties = {
|
|
mountpoint = "/zfs/media/database/postgres";
|
|
primarycache = "metadata";
|
|
recordsize = "16K";
|
|
};
|
|
snapshots = {
|
|
"15_min" = 8;
|
|
hourly = 24;
|
|
daily = 7;
|
|
};
|
|
};
|
|
"postgres-wal" = {
|
|
properties = {
|
|
compression = "lz4";
|
|
logbias = "latency";
|
|
mountpoint = "/zfs/media/database/postgres-wal";
|
|
primarycache = "metadata";
|
|
recordsize = "32K";
|
|
secondarycache = "none";
|
|
special_small_blocks = "32K";
|
|
};
|
|
snapshots = {
|
|
"15_min" = 4;
|
|
hourly = 2;
|
|
};
|
|
};
|
|
prometheus = {
|
|
properties = {
|
|
mountpoint = "/zfs/media/database/prometheus";
|
|
compression = "lz4";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
services = {
|
|
properties = {
|
|
mountpoint = "/zfs/media/services";
|
|
compression = "zstd-9";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
share = {
|
|
properties = {
|
|
mountpoint = "/zfs/media/share";
|
|
exec = "off";
|
|
};
|
|
snapshots."15_min" = 4;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
storage = {
|
|
properties = poolDefaults "/zfs/storage";
|
|
datasets = {
|
|
nomad = {
|
|
properties = {
|
|
mountpoint = "/zfs/storage/nomad";
|
|
compression = "zstd-9";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
ollama = {
|
|
properties = {
|
|
compression = "zstd-19";
|
|
recordsize = "1M";
|
|
sync = "disabled";
|
|
};
|
|
snapshots."15_min" = 2;
|
|
};
|
|
secure = {
|
|
# An encryption root. zfs_manager must never create this: encryption
|
|
# is fixed at creation and is not declared here, so an automatic
|
|
# create would produce an unencrypted dataset. See scripts/zfs.sh.
|
|
createIfMissing = false;
|
|
properties.keylocation = zfsKey;
|
|
snapshots = { };
|
|
datasets = {
|
|
archive = {
|
|
properties = {
|
|
compression = "zstd-19";
|
|
mountpoint = "/zfs/storage/archive";
|
|
recordsize = "1M";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
important = {
|
|
properties = {
|
|
compression = "zstd-19";
|
|
copies = "2";
|
|
mountpoint = "/zfs/storage/important";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
library = {
|
|
properties = {
|
|
compression = "zstd-19";
|
|
mountpoint = "/zfs/storage/library";
|
|
recordsize = "1M";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
main = {
|
|
properties = {
|
|
compression = "zstd-19";
|
|
mountpoint = "/zfs/storage/main";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
photos = {
|
|
properties = {
|
|
compression = "zstd-19";
|
|
copies = "2";
|
|
mountpoint = "/zfs/storage/photos";
|
|
recordsize = "16K";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
plex = {
|
|
properties = {
|
|
compression = "zstd-19";
|
|
mountpoint = "/zfs/storage/plex";
|
|
recordsize = "1M";
|
|
};
|
|
snapshots = {
|
|
"15_min" = 6;
|
|
hourly = 2;
|
|
daily = 1;
|
|
};
|
|
};
|
|
secrets = {
|
|
properties = {
|
|
compression = "zstd-19";
|
|
copies = "3";
|
|
mountpoint = "/zfs/storage/secrets";
|
|
};
|
|
snapshots = {
|
|
"15_min" = 8;
|
|
hourly = 24;
|
|
daily = 30;
|
|
monthly = 12;
|
|
};
|
|
};
|
|
syncthing = {
|
|
properties = {
|
|
compression = "zstd-19";
|
|
mountpoint = "/zfs/storage/syncthing";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
transmission = {
|
|
properties = {
|
|
compression = "zstd-9";
|
|
exec = "off";
|
|
mountpoint = "/zfs/storage/transmission";
|
|
recordsize = "1M";
|
|
sync = "disabled";
|
|
};
|
|
snapshots."15_min" = 4;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
scratch = {
|
|
properties = poolDefaults "/zfs/scratch" // {
|
|
keylocation = zfsKey;
|
|
};
|
|
datasets = {
|
|
kafka = {
|
|
properties = {
|
|
mountpoint = "/zfs/scratch/kafka";
|
|
recordsize = "1M";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
kestra = {
|
|
properties = {
|
|
mountpoint = "/zfs/scratch/kestra";
|
|
sync = "disabled";
|
|
};
|
|
snapshots = standard;
|
|
};
|
|
transmission = {
|
|
properties = {
|
|
mountpoint = "/zfs/scratch/transmission";
|
|
recordsize = "16K";
|
|
sync = "disabled";
|
|
};
|
|
snapshots."15_min" = 2;
|
|
};
|
|
uv_cache = {
|
|
properties.mountpoint = "/zfs/scratch/uv_cache";
|
|
snapshots."15_min" = 2;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
# Collapse the tree into the flat "pool/parent/child" names zfs uses. Each
|
|
# node keeps everything except its children.
|
|
flatten =
|
|
name: node:
|
|
builtins.foldl' (result: child: result // flatten "${name}/${child}" node.datasets.${child}) {
|
|
${name} = builtins.removeAttrs node [ "datasets" ];
|
|
} (builtins.attrNames (node.datasets or { }));
|
|
|
|
datasets = builtins.foldl' (result: pool: result // flatten pool pools.${pool}) { } (
|
|
builtins.attrNames pools
|
|
);
|
|
|
|
# zfs gives a dataset with no mountpoint of its own its parent's mountpoint
|
|
# plus its final name component, so resolve it the same way. Pool roots all
|
|
# declare a mountpoint, which terminates the recursion.
|
|
mountpointOf =
|
|
name:
|
|
let
|
|
properties = datasets.${name}.properties or { };
|
|
in
|
|
if properties ? mountpoint then
|
|
properties.mountpoint
|
|
else if builtins.match ".*/.*" name != null then
|
|
"${mountpointOf (builtins.dirOf name)}/${builtins.baseNameOf name}"
|
|
else
|
|
throw "jeeves: ${name} has no mountpoint and no parent to inherit one from";
|
|
|
|
mountpoints = builtins.mapAttrs (name: _: mountpointOf name) datasets;
|
|
in
|
|
{
|
|
inherit datasets mountpoints;
|
|
defaultSnapshots = standard;
|
|
}
|