Compare commits

..

37 Commits

Author SHA1 Message Date
Richie 2c366e581d opning ports for testing
pytest / pytest (pull_request) Failing after 27s
build_systems / build-bob (pull_request) Successful in 47s
treefmt / nix fmt (pull_request) Successful in 5s
build_systems / build-brain (pull_request) Successful in 46s
build_systems / build-leviathan (pull_request) Successful in 56s
build_systems / build-rhapsody-in-green (pull_request) Successful in 58s
build_systems / build-jeeves (pull_request) Successful in 2m42s
2026-06-13 22:41:09 -04:00
Richie 5e2252641d added a index for the VEctor DB 2026-06-13 22:41:09 -04:00
Richie bb3c433b9d improved BM25 write 2026-06-13 22:41:09 -04:00
Richie 66ea18af82 added ZstdMiddleware to ebook_search 2026-06-13 22:41:09 -04:00
Richie 51855725a1 added vector_engine to fix name postgres name space issue 2026-06-13 22:41:09 -04:00
Richie ed45051eb5 reworked ebook_search routers 2026-06-13 22:41:09 -04:00
Richie 479191050e made fastapi tools 2026-06-13 22:41:09 -04:00
Richie c5418b50fd added proper cache invalidation to load_bm25_corpus 2026-06-13 22:41:09 -04:00
Richie 70f24cdbc6 updated tests 2026-06-13 22:41:09 -04:00
Richie 2f1affa2e5 improved reranking weights 2026-06-13 22:41:09 -04:00
Richie 3d582243fc fixed duplicat enrichment 2026-06-13 22:41:09 -04:00
Richie 2efc9e30a8 improved queary for vector search 2026-06-13 22:41:09 -04:00
Richie a38ce4505f add .ebook_search_bm25 to gitignore 2026-06-13 22:41:09 -04:00
Richie 1efa7b047a updated python 2026-06-13 22:41:09 -04:00
Richie cad3f6f79e setup tests 2026-06-13 22:41:09 -04:00
Richie 07dd1922b1 build api and frountend 2026-06-13 22:41:09 -04:00
Richie 93ff2200fe added answer.py and config 2026-06-13 22:41:09 -04:00
Richie 3a5b278c15 added __init__ 2026-06-13 22:41:09 -04:00
Richie 4bd61bc170 made llm_interface.py 2026-06-13 22:41:09 -04:00
Richie f7b72c4053 added rerank 2026-06-13 22:41:09 -04:00
Richie d1b59955d0 built ingest 2026-06-13 22:41:09 -04:00
Richie ff7b2ab2fa built rag search setup 2026-06-13 22:41:09 -04:00
Richie 5308ff8be6 set up embedding system 2026-06-13 22:41:09 -04:00
Richie 5087fbb4c0 built BM25 search foundation 2026-06-13 22:41:09 -04:00
Richie aa135a3af2 clean up 2026-06-13 22:41:09 -04:00
Richie 6289000766 added ebook embedding to orm 2026-06-13 22:41:09 -04:00
Richie 3ee884f6b4 removed hedgedoc 2026-06-13 22:41:09 -04:00
Richie 2e8c0570e4 adding embedding Models to jeeves 2026-06-13 22:41:09 -04:00
Richie 9290cb46ee updated series_index to float and added UniqueConstraint to audiobook and audiobook_author
treefmt / nix fmt (push) Successful in 5s
build_systems / build-bob (push) Successful in 32s
build_systems / build-leviathan (push) Successful in 41s
build_systems / build-rhapsody-in-green (push) Successful in 44s
pytest / pytest (push) Successful in 27s
build_systems / build-brain (push) Successful in 31s
build_systems / build-jeeves (push) Successful in 2m28s
pytest / pytest (pull_request) Successful in 26s
build_systems / build-bob (pull_request) Successful in 47s
treefmt / nix fmt (pull_request) Successful in 5s
build_systems / build-brain (pull_request) Successful in 44s
build_systems / build-leviathan (pull_request) Successful in 52s
build_systems / build-rhapsody-in-green (pull_request) Successful in 58s
build_systems / build-jeeves (pull_request) Successful in 2m27s
2026-06-13 22:29:56 -04:00
Richie acd3f2d3ac fixed omnibus for audio books 2026-06-13 22:29:56 -04:00
Richie 08e716f66a deleted frontend dir 2026-06-13 22:29:56 -04:00
Richie d197731af4 added llm_tool_calling.py 2026-06-13 22:29:56 -04:00
Richie 1ffc48bb02 built workflow 2026-06-13 22:29:56 -04:00
Richie b6395ef18f Add catalog.py for manually adding authors and series to the database. 2026-06-13 22:29:56 -04:00
Richie aff6f4e1bd adding audiobook data to DB 2026-06-13 22:29:56 -04:00
Richie a9a96db944 cleaned up old_installer.py
treefmt / nix fmt (pull_request) Successful in 5s
pytest / pytest (pull_request) Successful in 25s
pytest / pytest (push) Successful in 24s
build_systems / build-rhapsody-in-green (pull_request) Successful in 1m5s
treefmt / nix fmt (push) Successful in 4s
build_systems / build-brain (pull_request) Successful in 49s
build_systems / build-bob (pull_request) Successful in 50s
build_systems / build-brain (push) Successful in 44s
build_systems / build-bob (push) Successful in 45s
build_systems / build-leviathan (pull_request) Successful in 57s
build_systems / build-leviathan (push) Successful in 53s
build_systems / build-rhapsody-in-green (push) Successful in 55s
build_systems / build-jeeves (pull_request) Successful in 2m41s
build_systems / build-jeeves (push) Successful in 2m37s
2026-06-13 22:27:11 -04:00
Richie d34154541d moved installer.py to old_installer.py 2026-06-13 22:20:58 -04:00
@@ -1,4 +1,3 @@
# ruff: noqa: LOG015, E501, D102, D103, D107 These need the be fixed
"""Install NixOS on a ZFS pool.""" """Install NixOS on a ZFS pool."""
from __future__ import annotations from __future__ import annotations
@@ -17,6 +16,9 @@ from typing import TYPE_CHECKING
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Sequence from collections.abc import Sequence
logger = logging.getLogger(__name__)
ESCAPE_KEY = 27
def configure_logger(level: str = "INFO") -> None: def configure_logger(level: str = "INFO") -> None:
"""Configure the logger. """Configure the logger.
@@ -42,7 +44,7 @@ def bash_wrapper(command: str) -> str:
Tuple[str, int]: A tuple containing the output of the command (stdout) as a string, Tuple[str, int]: A tuple containing the output of the command (stdout) as a string,
the error output (stderr) as a string (optional), and the return code as an integer. the error output (stderr) as a string (optional), and the return code as an integer.
""" """
logging.debug(f"running {command=}") logger.debug(f"running {command=}")
# This is a acceptable risk # This is a acceptable risk
process = Popen(command.split(), stdout=PIPE, stderr=PIPE) process = Popen(command.split(), stdout=PIPE, stderr=PIPE)
output, _ = process.communicate() output, _ = process.communicate()
@@ -63,7 +65,7 @@ def partition_disk(disk: str, swap_size: int, reserve: int = 0) -> None:
reserve (int, optional): The size of the reserve partition in GB. Defaults to 0. reserve (int, optional): The size of the reserve partition in GB. Defaults to 0.
minimum value is 0. minimum value is 0.
""" """
logging.info(f"partitioning {disk=}") logger.info(f"partitioning {disk=}")
swap_size = max(swap_size, 1) swap_size = max(swap_size, 1)
reserve = max(reserve, 0) reserve = max(reserve, 0)
@@ -71,16 +73,16 @@ def partition_disk(disk: str, swap_size: int, reserve: int = 0) -> None:
if reserve > 0: if reserve > 0:
msg = f"Creating swap partition on {disk=} with size {swap_size=}GiB and reserve {reserve=}GiB" msg = f"Creating swap partition on {disk=} with size {swap_size=}GiB and reserve {reserve=}GiB"
logging.info(msg) logger.info(msg)
swap_start = swap_size + reserve swap_start = swap_size + reserve
swap_partition = f"mkpart swap -{swap_start}GiB -{reserve}GiB " swap_partition = f"mkpart swap -{swap_start}GiB -{reserve}GiB "
else: else:
logging.info(f"Creating swap partition on {disk=} with size {swap_size=}GiB") logger.info(f"Creating swap partition on {disk=} with size {swap_size=}GiB")
swap_start = swap_size swap_start = swap_size
swap_partition = f"mkpart swap -{swap_start}GiB 100% " swap_partition = f"mkpart swap -{swap_start}GiB 100% "
logging.debug(f"{swap_partition=}") logger.debug(f"{swap_partition=}")
create_partitions = ( create_partitions = (
f"parted --script --align=optimal {disk} -- " f"parted --script --align=optimal {disk} -- "
@@ -92,7 +94,7 @@ def partition_disk(disk: str, swap_size: int, reserve: int = 0) -> None:
) )
bash_wrapper(create_partitions) bash_wrapper(create_partitions)
logging.info(f"{disk=} successfully partitioned") logger.info(f"{disk=} successfully partitioned")
def create_zfs_pool(pool_disks: Sequence[str], mnt_dir: str) -> None: def create_zfs_pool(pool_disks: Sequence[str], mnt_dir: str) -> None:
@@ -131,7 +133,7 @@ def create_zfs_pool(pool_disks: Sequence[str], mnt_dir: str) -> None:
bash_wrapper(zpool_create) bash_wrapper(zpool_create)
zpools = bash_wrapper("zpool list -o name") zpools = bash_wrapper("zpool list -o name")
if "root_pool" not in zpools.splitlines(): if "root_pool" not in zpools.splitlines():
logging.critical("Failed to create root_pool") logger.critical("Failed to create root_pool")
sys.exit(1) sys.exit(1)
@@ -151,7 +153,7 @@ def create_zfs_datasets() -> None:
} }
missing_datasets = expected_datasets.difference(datasets.splitlines()) missing_datasets = expected_datasets.difference(datasets.splitlines())
if missing_datasets: if missing_datasets:
logging.critical(f"Failed to create pools {missing_datasets}") logger.critical(f"Failed to create pools {missing_datasets}")
sys.exit(1) sys.exit(1)
@@ -164,8 +166,7 @@ def get_cpu_manufacturer() -> str:
for line in output.splitlines(): for line in output.splitlines():
if "vendor_id" in line: if "vendor_id" in line:
return id_vendor[line.split(": ")[1].strip()] return id_vendor[line.split(": ")[1].strip()]
error = "Failed to get CPU manufacturer"
error = "CPU manufacturer not found"
raise RuntimeError(error) raise RuntimeError(error)
@@ -200,7 +201,15 @@ def create_nix_hardware_file(mnt_dir: str, disks: Sequence[str], *, encrypt: boo
' imports = [ (modulesPath + "/installer/scan/not-detected.nix") ];\n\n' ' imports = [ (modulesPath + "/installer/scan/not-detected.nix") ];\n\n'
" boot = {\n" " boot = {\n"
" initrd = {\n" " initrd = {\n"
' availableKernelModules = [ \n "ahci"\n "ehci_pci"\n "nvme"\n "sd_mod"\n "usb_storage"\n "usbhid"\n "xhci_pci"\n ];\n' " availableKernelModules = [ \n"
' "ahci"\n'
' "ehci_pci"\n'
' "nvme"\n'
' "sd_mod"\n'
' "usb_storage"\n'
' "usbhid"\n'
' "xhci_pci"\n'
" ];\n"
" kernelModules = [ ];\n" " kernelModules = [ ];\n"
f" {devices}" f" {devices}"
" };\n" " };\n"
@@ -214,11 +223,18 @@ def create_nix_hardware_file(mnt_dir: str, disks: Sequence[str], *, encrypt: boo
' "/nix" = {\n device = "root_pool/nix";\n fsType = "zfs";\n };\n\n' ' "/nix" = {\n device = "root_pool/nix";\n fsType = "zfs";\n };\n\n'
' "/boot" = {\n' ' "/boot" = {\n'
f' device = "/dev/disk/by-uuid/{get_boot_drive_id(disks[0])}";\n' f' device = "/dev/disk/by-uuid/{get_boot_drive_id(disks[0])}";\n'
' fsType = "vfat";\n options = [\n "fmask=0077"\n "dmask=0077"\n ];\n };\n };\n\n' ' fsType = "vfat";\n'
" options = [\n"
' "fmask=0077"\n'
' "dmask=0077"\n'
" ];\n"
" };\n"
" };\n\n"
" swapDevices = [ ];\n\n" " swapDevices = [ ];\n\n"
" networking.useDHCP = lib.mkDefault true;\n\n" " networking.useDHCP = lib.mkDefault true;\n\n"
' nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";\n' ' nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";\n'
f" hardware.cpu.{cpu_manufacturer}.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;\n" f" hardware.cpu.{cpu_manufacturer}.updateMicrocode = lib.mkDefault "
"config.hardware.enableRedistributableFirmware;\n"
f' networking.hostId = "{host_id}";\n' f' networking.hostId = "{host_id}";\n'
"}\n" "}\n"
) )
@@ -256,18 +272,30 @@ def installer(
encrypt_key: str | None, encrypt_key: str | None,
) -> None: ) -> None:
"""Main.""" """Main."""
logging.info("Starting installation") logger.info("Starting installation")
for disk in disks: for disk in disks:
partition_disk(disk, swap_size, reserve) partition_disk(disk, swap_size, reserve)
if encrypt_key: if encrypt_key:
sleep(1) sleep(1)
for command in ( key_input = encrypt_key.encode()
f'printf "{encrypt_key}" | cryptsetup luksFormat --type luks2 {disk}-part2 -', run(
f'printf "{encrypt_key}" | cryptsetup luksOpen {disk}-part2 luks-root-pool-{disk.split("/")[-1]}-part2 -', ("cryptsetup", "luksFormat", "--type", "luks2", f"{disk}-part2", "-"),
): input=key_input,
run(command, shell=True, check=True) # noqa: S602 check=True,
)
run(
(
"cryptsetup",
"luksOpen",
f"{disk}-part2",
f"luks-root-pool-{disk.split('/')[-1]}-part2",
"-",
),
input=key_input,
check=True,
)
mnt_dir = "/tmp/nix_install" # noqa: S108 mnt_dir = "/tmp/nix_install" # noqa: S108
@@ -282,59 +310,73 @@ def installer(
create_zfs_datasets() create_zfs_datasets()
install_nixos(mnt_dir, disks, encrypt=encrypt_key) install_nixos(mnt_dir, disks, encrypt=bool(encrypt_key))
logging.info("Installation complete") logger.info("Installation complete")
class Cursor: class Cursor:
"""Cursor class to store the cursor position.""" """Track cursor position and constrain movement to screen bounds."""
def __init__(self) -> None: def __init__(self) -> None:
"""Initialize cursor position and screen dimensions."""
self.x_position = 0 self.x_position = 0
self.y_position = 0 self.y_position = 0
self.height = 0 self.height = 0
self.width = 0 self.width = 0
def set_height(self, height: int) -> None: def set_height(self, height: int) -> None:
"""Set the maximum screen height."""
self.height = height self.height = height
def set_width(self, width: int) -> None: def set_width(self, width: int) -> None:
"""Set the maximum screen width."""
self.width = width self.width = width
def x_bounce_check(self, cursor: int) -> int: def x_bounce_check(self, cursor: int) -> int:
"""Clamp an x position to the screen width."""
cursor = max(0, cursor) cursor = max(0, cursor)
return min(self.width - 1, cursor) return min(self.width - 1, cursor)
def y_bounce_check(self, cursor: int) -> int: def y_bounce_check(self, cursor: int) -> int:
"""Clamp a y position to the screen height."""
cursor = max(0, cursor) cursor = max(0, cursor)
return min(self.height - 1, cursor) return min(self.height - 1, cursor)
def set_x(self, x: int) -> None: def set_x(self, x: int) -> None:
"""Set the cursor x position."""
self.x_position = self.x_bounce_check(x) self.x_position = self.x_bounce_check(x)
def set_y(self, y: int) -> None: def set_y(self, y: int) -> None:
"""Set the cursor y position."""
self.y_position = self.y_bounce_check(y) self.y_position = self.y_bounce_check(y)
def get_x(self) -> int: def get_x(self) -> int:
"""Get the cursor x position."""
return self.x_position return self.x_position
def get_y(self) -> int: def get_y(self) -> int:
"""Get the cursor y position."""
return self.y_position return self.y_position
def move_up(self) -> None: def move_up(self) -> None:
"""Move the cursor up one row."""
self.set_y(self.y_position - 1) self.set_y(self.y_position - 1)
def move_down(self) -> None: def move_down(self) -> None:
"""Move the cursor down one row."""
self.set_y(self.y_position + 1) self.set_y(self.y_position + 1)
def move_left(self) -> None: def move_left(self) -> None:
"""Move the cursor left one column."""
self.set_x(self.x_position - 1) self.set_x(self.x_position - 1)
def move_right(self) -> None: def move_right(self) -> None:
"""Move the cursor right one column."""
self.set_x(self.x_position + 1) self.set_x(self.x_position + 1)
def navigation(self, key: int) -> None: def navigation(self, key: int) -> None:
"""Move the cursor for a curses navigation key."""
action = { action = {
curses.KEY_DOWN: self.move_down, curses.KEY_DOWN: self.move_down,
curses.KEY_UP: self.move_up, curses.KEY_UP: self.move_up,
@@ -349,6 +391,7 @@ class State:
"""State class to store the state of the program.""" """State class to store the state of the program."""
def __init__(self) -> None: def __init__(self) -> None:
"""Initialize installer menu state."""
self.key = 0 self.key = 0
self.cursor = Cursor() self.cursor = Cursor()
@@ -366,6 +409,7 @@ class State:
def get_device(raw_device: str) -> dict[str, str]: def get_device(raw_device: str) -> dict[str, str]:
"""Parse an lsblk key-value device row."""
raw_device_components = raw_device.split(" ") raw_device_components = raw_device.split(" ")
return {thing.split("=")[0].lower(): thing.split("=")[1].strip('"') for thing in raw_device_components} return {thing.split("=")[0].lower(): thing.split("=")[1].strip('"') for thing in raw_device_components}
@@ -395,6 +439,7 @@ def get_device_id_mapping() -> dict[str, set[str]]:
def calculate_device_menu_padding(devices: list[dict[str, str]], column: str, padding: int = 0) -> int: def calculate_device_menu_padding(devices: list[dict[str, str]], column: str, padding: int = 0) -> int:
"""Calculate the width needed for a device menu column."""
return max(len(device[column]) for device in devices) + padding return max(len(device[column]) for device in devices) + padding
@@ -406,6 +451,7 @@ def draw_device_ids(
menu_width: list[int], menu_width: list[int],
device_ids: set[str], device_ids: set[str],
) -> tuple[State, int]: ) -> tuple[State, int]:
"""Draw selectable device IDs for a device row."""
for device_id in sorted(device_ids): for device_id in sorted(device_ids):
row_number = row_number + 1 row_number = row_number + 1
if row_number == state.cursor.get_y() and state.cursor.get_x() in menu_width: if row_number == state.cursor.get_y() and state.cursor.get_x() in menu_width:
@@ -434,7 +480,7 @@ def draw_device_menu(
state: State, state: State,
menu_start_y: int = 0, menu_start_y: int = 0,
menu_start_x: int = 0, menu_start_x: int = 0,
) -> State: ) -> tuple[State, int]:
"""Draw the device menu and handle user input. """Draw the device menu and handle user input.
Args: Args:
@@ -490,6 +536,7 @@ def draw_device_menu(
def debug_menu(std_screen: curses.window, key: int) -> None: def debug_menu(std_screen: curses.window, key: int) -> None:
"""Draw debug information for the current curses screen."""
height, width = std_screen.getmaxyx() height, width = std_screen.getmaxyx()
width_height = f"Width: {width}, Height: {height}" width_height = f"Width: {width}, Height: {height}"
std_screen.addstr(height - 4, 0, width_height, curses.color_pair(5)) std_screen.addstr(height - 4, 0, width_height, curses.color_pair(5))
@@ -509,6 +556,7 @@ def status_bar(
width: int, width: int,
height: int, height: int,
) -> None: ) -> None:
"""Draw the footer status bar."""
std_screen.attron(curses.A_REVERSE) std_screen.attron(curses.A_REVERSE)
std_screen.attron(curses.color_pair(3)) std_screen.attron(curses.color_pair(3))
@@ -521,6 +569,7 @@ def status_bar(
def set_color() -> None: def set_color() -> None:
"""Initialize curses color pairs."""
curses.start_color() curses.start_color()
curses.use_default_colors() curses.use_default_colors()
for i in range(curses.COLORS): for i in range(curses.COLORS):
@@ -528,6 +577,7 @@ def set_color() -> None:
def get_text_input(std_screen: curses.window, prompt: str, y: int, x: int) -> str: def get_text_input(std_screen: curses.window, prompt: str, y: int, x: int) -> str:
"""Read text input from a curses screen."""
curses.echo() curses.echo()
std_screen.addstr(y, x, prompt) std_screen.addstr(y, x, prompt)
input_str = "" input_str = ""
@@ -535,7 +585,7 @@ def get_text_input(std_screen: curses.window, prompt: str, y: int, x: int) -> st
key = std_screen.getch() key = std_screen.getch()
if key == ord("\n"): if key == ord("\n"):
break break
if key == 27: # ESC key # noqa: PLR2004 if key == ESCAPE_KEY:
input_str = "" input_str = ""
break break
if key in (curses.KEY_BACKSPACE, ord("\b"), 127): if key in (curses.KEY_BACKSPACE, ord("\b"), 127):
@@ -553,6 +603,7 @@ def swap_size_input(
state: State, state: State,
swap_offset: int, swap_offset: int,
) -> State: ) -> State:
"""Handle swap size input."""
swap_size_text = "Swap size (GB): " swap_size_text = "Swap size (GB): "
std_screen.addstr(swap_offset, 0, f"{swap_size_text}{state.swap_size}") std_screen.addstr(swap_offset, 0, f"{swap_size_text}{state.swap_size}")
if state.key == ord("\n") and state.cursor.get_y() == swap_offset: if state.key == ord("\n") and state.cursor.get_y() == swap_offset:
@@ -576,6 +627,7 @@ def reserve_size_input(
state: State, state: State,
reserve_offset: int, reserve_offset: int,
) -> State: ) -> State:
"""Handle reserve size input."""
reserve_size_text = "reserve size (GB): " reserve_size_text = "reserve size (GB): "
std_screen.addstr(reserve_offset, 0, f"{reserve_size_text}{state.reserve_size}") std_screen.addstr(reserve_offset, 0, f"{reserve_size_text}{state.reserve_size}")
if state.key == ord("\n") and state.cursor.get_y() == reserve_offset: if state.key == ord("\n") and state.cursor.get_y() == reserve_offset:
@@ -599,6 +651,7 @@ def draw_menu(std_screen: curses.window) -> State:
Args: Args:
std_screen (curses.window): the curses window to draw on std_screen (curses.window): the curses window to draw on
Returns: Returns:
State: the state object State: the state object
""" """
@@ -658,17 +711,18 @@ def draw_menu(std_screen: curses.window) -> State:
def main() -> None: def main() -> None:
"""Run the installer menu and start installation."""
configure_logger("DEBUG") configure_logger("DEBUG")
state = curses.wrapper(draw_menu) state = curses.wrapper(draw_menu)
encrypt_key = getenv("ENCRYPT_KEY") encrypt_key = getenv("ENCRYPT_KEY")
logging.info("installing_nixos") logger.info("installing_nixos")
logging.info(f"disks: {state.selected_device_ids}") logger.info(f"disks: {state.selected_device_ids}")
logging.info(f"swap_size: {state.swap_size}") logger.info(f"swap_size: {state.swap_size}")
logging.info(f"reserve: {state.reserve_size}") logger.info(f"reserve: {state.reserve_size}")
logging.info(f"encrypted: {bool(encrypt_key)}") logger.info(f"encrypted: {bool(encrypt_key)}")
sleep(3) sleep(3)