feat(ebook-search): add containerized test environment
treefmt / nix fmt (pull_request) Successful in 5s
pytest / pytest (pull_request) Successful in 26s
build_systems / build-brain (pull_request) Successful in 45s
build_systems / build-bob (pull_request) Successful in 45s
build_systems / build-rhapsody-in-green (pull_request) Successful in 57s
build_systems / build-jeeves (pull_request) Successful in 2m17s
test ebook search / test-ebook-search (pull_request) Failing after 4s
treefmt / nix fmt (pull_request) Successful in 5s
pytest / pytest (pull_request) Successful in 26s
build_systems / build-brain (pull_request) Successful in 45s
build_systems / build-bob (pull_request) Successful in 45s
build_systems / build-rhapsody-in-green (pull_request) Successful in 57s
build_systems / build-jeeves (pull_request) Successful in 2m17s
test ebook search / test-ebook-search (pull_request) Failing after 4s
This commit is contained in:
@@ -3,6 +3,18 @@
|
|||||||
.mypy_cache
|
.mypy_cache
|
||||||
.pytest_cache
|
.pytest_cache
|
||||||
.ruff_cache
|
.ruff_cache
|
||||||
|
.venv
|
||||||
|
.env
|
||||||
|
.cache
|
||||||
|
.claude
|
||||||
|
.coverage
|
||||||
|
.vscode
|
||||||
|
.stfolder
|
||||||
|
.literotica_data
|
||||||
|
esphome
|
||||||
|
htmlcov
|
||||||
|
data
|
||||||
|
ebooks
|
||||||
__pycache__
|
__pycache__
|
||||||
**/__pycache__
|
**/__pycache__
|
||||||
*.pyc
|
*.pyc
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
name: test ebook search
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test-ebook-search:
|
||||||
|
runs-on: self-hosted
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Build test image
|
||||||
|
run: docker build --file python/ebook_search/docker/Dockerfile --target test --tag ebook-search:test .
|
||||||
|
- name: Run ebook search tests
|
||||||
|
run: docker run --rm ebook-search:test
|
||||||
@@ -17,9 +17,7 @@
|
|||||||
|
|
||||||
python-env = final: _prev: {
|
python-env = final: _prev: {
|
||||||
my_python = final.python314.withPackages (
|
my_python = final.python314.withPackages (
|
||||||
ps:
|
ps: with ps; [
|
||||||
with ps;
|
|
||||||
[
|
|
||||||
alembic
|
alembic
|
||||||
apprise
|
apprise
|
||||||
apscheduler
|
apscheduler
|
||||||
|
|||||||
+1
-1
@@ -117,6 +117,6 @@ exclude_lines = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[tool.pytest.ini_options]
|
[tool.pytest.ini_options]
|
||||||
addopts = "-n auto -ra"
|
addopts = "-n auto -ra --ignore=tests/ebook_search"
|
||||||
testpaths = ["tests"]
|
testpaths = ["tests"]
|
||||||
# --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch
|
# --cov=system_tools --cov-report=term-missing --cov-report=xml --cov-report=html --cov-branch
|
||||||
|
|||||||
@@ -1,12 +1,17 @@
|
|||||||
FROM python:3.14-slim
|
FROM python:3.14-slim AS base
|
||||||
|
|
||||||
|
COPY --from=ghcr.io/astral-sh/uv:0.11.26 /uv /uvx /bin/
|
||||||
|
|
||||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||||
PYTHONUNBUFFERED=1 \
|
PYTHONUNBUFFERED=1 \
|
||||||
PIP_NO_CACHE_DIR=1 \
|
|
||||||
APP_DIR=/home/richie/dotfiles \
|
APP_DIR=/home/richie/dotfiles \
|
||||||
EBOOK_SEARCH_HOST=0.0.0.0 \
|
UV_PROJECT_ENVIRONMENT=/opt/venv \
|
||||||
EBOOK_SEARCH_PORT=8070 \
|
UV_PYTHON_DOWNLOADS=never \
|
||||||
EBOOK_SEARCH_BM25_INDEX_DIR=/data/bm25
|
UV_NO_CACHE=1
|
||||||
|
|
||||||
|
# Separate ENV instruction so ${APP_DIR} and ${PATH} from above resolve.
|
||||||
|
ENV PYTHONPATH=${APP_DIR} \
|
||||||
|
PATH=/opt/venv/bin:${PATH}
|
||||||
|
|
||||||
WORKDIR ${APP_DIR}
|
WORKDIR ${APP_DIR}
|
||||||
|
|
||||||
@@ -14,29 +19,29 @@ RUN apt-get update \
|
|||||||
&& apt-get install -y --no-install-recommends build-essential curl \
|
&& apt-get install -y --no-install-recommends build-essential curl \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
COPY pyproject.toml README.md LICENSE ./
|
COPY python/ebook_search/docker/pyproject.toml python/ebook_search/docker/uv.lock ./
|
||||||
COPY python ./python
|
|
||||||
|
|
||||||
RUN python -m pip install --upgrade pip \
|
RUN uv sync --locked --no-dev
|
||||||
&& python -m pip install \
|
|
||||||
"alembic" \
|
|
||||||
"beautifulsoup4" \
|
FROM base AS test
|
||||||
"bm25s" \
|
|
||||||
"ebooklib" \
|
RUN uv sync --locked
|
||||||
"fastapi" \
|
|
||||||
"httpx" \
|
COPY python ./python
|
||||||
"jinja2" \
|
COPY tests/__init__.py ./tests/__init__.py
|
||||||
"pgvector" \
|
COPY tests/ebook_search ./tests/ebook_search
|
||||||
"psycopg[binary]" \
|
|
||||||
"pydantic" \
|
CMD ["pytest"]
|
||||||
"pydantic-settings" \
|
|
||||||
"python-multipart" \
|
|
||||||
"sqlalchemy" \
|
FROM base AS runtime
|
||||||
"tiktoken" \
|
|
||||||
"typer" \
|
ENV EBOOK_SEARCH_HOST=0.0.0.0 \
|
||||||
"uvicorn[standard]" \
|
EBOOK_SEARCH_PORT=8070 \
|
||||||
"yake" \
|
EBOOK_SEARCH_BM25_INDEX_DIR=/data/bm25
|
||||||
&& python -m pip install --no-deps --editable "${APP_DIR}"
|
|
||||||
|
COPY python ./python
|
||||||
|
|
||||||
RUN useradd --create-home --uid 10001 app \
|
RUN useradd --create-home --uid 10001 app \
|
||||||
&& mkdir -p /data \
|
&& mkdir -p /data \
|
||||||
|
|||||||
@@ -3,26 +3,27 @@
|
|||||||
Run the EPUB search app against the existing Postgres database on `jeeves`:
|
Run the EPUB search app against the existing Postgres database on `jeeves`:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
ebook-search-containers start --library-path /path/to/epubs --build
|
python -m python.ebook_search.docker.containers start --library-path /path/to/epubs --build
|
||||||
```
|
```
|
||||||
|
|
||||||
All ebook-search Docker files live in this directory:
|
All ebook-search Docker files live in this directory:
|
||||||
|
|
||||||
- `Dockerfile`
|
- `Dockerfile` — multi-stage: `test` (runs pytest) and `runtime` (default target, the app image)
|
||||||
- `docker-compose.yml`
|
- `docker-compose.yml`
|
||||||
- `containers.py`
|
- `containers.py` — Typer lifecycle CLI
|
||||||
- `container.py`
|
- `pyproject.toml` / `uv.lock` — the container's uv-locked dependencies
|
||||||
|
|
||||||
The app listens on `http://localhost:8070`.
|
The app listens on `http://localhost:8070`.
|
||||||
|
|
||||||
Useful lifecycle commands:
|
Useful lifecycle commands:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
ebook-search-containers build
|
python -m python.ebook_search.docker.containers build
|
||||||
ebook-search-containers start --library-path /path/to/epubs
|
python -m python.ebook_search.docker.containers start --library-path /path/to/epubs
|
||||||
ebook-search-containers logs
|
python -m python.ebook_search.docker.containers test
|
||||||
ebook-search-containers ps
|
python -m python.ebook_search.docker.containers logs
|
||||||
ebook-search-containers stop
|
python -m python.ebook_search.docker.containers ps
|
||||||
|
python -m python.ebook_search.docker.containers stop
|
||||||
```
|
```
|
||||||
|
|
||||||
Direct compose usage from the repo root:
|
Direct compose usage from the repo root:
|
||||||
@@ -31,10 +32,40 @@ Direct compose usage from the repo root:
|
|||||||
docker compose -f python/ebook_search/docker/docker-compose.yml ps
|
docker compose -f python/ebook_search/docker/docker-compose.yml ps
|
||||||
```
|
```
|
||||||
|
|
||||||
The compose service also loads the repo root `.env` into the container via `env_file`.
|
## Dependencies
|
||||||
|
|
||||||
|
The image builds its environment with uv from `pyproject.toml` + `uv.lock` in this
|
||||||
|
directory — this is the source of truth for the container's dependencies. To add or
|
||||||
|
update a dependency, edit `pyproject.toml` here and regenerate the lock (requires uv
|
||||||
|
on the host; it is not in the nix dev shell):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cd python/ebook_search/docker && uv lock
|
||||||
|
```
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
The main pytest suite excludes `tests/ebook_search` (its dependencies are no longer
|
||||||
|
in the nix dev shell). The ebook tests run inside the Docker `test` image instead:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python -m python.ebook_search.docker.containers test
|
||||||
|
```
|
||||||
|
|
||||||
|
or the raw docker equivalent (what the `test ebook search` CI workflow runs):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker build --file python/ebook_search/docker/Dockerfile --target test --tag ebook-search:test .
|
||||||
|
docker run --rm ebook-search:test
|
||||||
|
```
|
||||||
|
|
||||||
|
To run them on the host instead, use a venv with the ebook deps installed and
|
||||||
|
override the ignore: `pytest tests/ebook_search -o addopts="-ra"`.
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
The compose service loads the repo root `.env` into the container via `env_file`.
|
||||||
|
|
||||||
Mount your EPUB directory by setting `EBOOK_LIBRARY_HOST_PATH` in an env file or on the command line. The container sees it as `/library`, and `EBOOK_SEARCH_LIBRARY_PATHS` is set to `/library` inside the container.
|
Mount your EPUB directory by setting `EBOOK_LIBRARY_HOST_PATH` in an env file or on the command line. The container sees it as `/library`, and `EBOOK_SEARCH_LIBRARY_PATHS` is set to `/library` inside the container.
|
||||||
|
|
||||||
Database connection settings are controlled by `RICHIE_DB`, `RICHIE_HOST`, `RICHIE_PORT`, `RICHIE_USER`, and `RICHIE_PASSWORD`. The default host is `jeeves`.
|
Database connection settings are controlled by `RICHIE_DB`, `RICHIE_HOST`, `RICHIE_PORT`, `RICHIE_USER`, and `RICHIE_PASSWORD`. The default host is `jeeves`.
|
||||||
|
|
||||||
Startup runs the Richie Alembic migrations automatically after creating the `main` schema and `vector` extension.
|
|
||||||
|
|||||||
@@ -73,6 +73,23 @@ def build_image() -> None:
|
|||||||
raise RuntimeError(msg)
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def build_test_image() -> None:
|
||||||
|
"""Build the ebook search test Docker image."""
|
||||||
|
dockerfile = Path(__file__).resolve().with_name("Dockerfile")
|
||||||
|
result = docker_run(["build", "--file", str(dockerfile), "--target", "test", "--tag", "ebook-search:test", "."])
|
||||||
|
if result.returncode != 0:
|
||||||
|
msg = "Failed to build ebook search test image"
|
||||||
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def run_test_image() -> None:
|
||||||
|
"""Run the ebook search test suite inside Docker."""
|
||||||
|
result = docker_run(["run", "--rm", "ebook-search:test"])
|
||||||
|
if result.returncode != 0:
|
||||||
|
msg = f"Ebook search tests failed with code {result.returncode}"
|
||||||
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
|
|
||||||
def start_stack(
|
def start_stack(
|
||||||
*,
|
*,
|
||||||
library_path: Path | None = None,
|
library_path: Path | None = None,
|
||||||
@@ -210,6 +227,19 @@ def logs(
|
|||||||
typer.echo(output)
|
typer.echo(output)
|
||||||
|
|
||||||
|
|
||||||
|
@app.command("test")
|
||||||
|
def run_tests(
|
||||||
|
*,
|
||||||
|
build: Annotated[bool, typer.Option("--build/--no-build", help="Build the test image before running.")] = True,
|
||||||
|
log_level: Annotated[str, typer.Option(help="Log level.")] = "INFO",
|
||||||
|
) -> None:
|
||||||
|
"""Run ebook search tests inside the Docker test image."""
|
||||||
|
configure_logger(log_level)
|
||||||
|
if build:
|
||||||
|
build_test_image()
|
||||||
|
run_test_image()
|
||||||
|
|
||||||
|
|
||||||
@app.command("ps")
|
@app.command("ps")
|
||||||
def ps() -> None:
|
def ps() -> None:
|
||||||
"""Show ebook search container status."""
|
"""Show ebook search container status."""
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
[project]
|
||||||
|
name = "ebook-search"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "Locked runtime environment for the ebook search container."
|
||||||
|
requires-python = "~=3.14.0"
|
||||||
|
dependencies = [
|
||||||
|
"alembic",
|
||||||
|
"beautifulsoup4",
|
||||||
|
"bm25s",
|
||||||
|
"ebooklib",
|
||||||
|
"fastapi",
|
||||||
|
"httpx",
|
||||||
|
"jinja2",
|
||||||
|
"pgvector",
|
||||||
|
"psycopg[binary]",
|
||||||
|
"pydantic",
|
||||||
|
"pydantic-settings",
|
||||||
|
"python-multipart",
|
||||||
|
"sqlalchemy",
|
||||||
|
"tiktoken",
|
||||||
|
"typer",
|
||||||
|
"uvicorn[standard]",
|
||||||
|
"yake",
|
||||||
|
]
|
||||||
|
|
||||||
|
[dependency-groups]
|
||||||
|
dev = [
|
||||||
|
"pytest",
|
||||||
|
"pytest-mock",
|
||||||
|
"pytest-xdist",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.uv]
|
||||||
|
package = false
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
addopts = "-n auto -ra"
|
||||||
|
testpaths = ["tests/ebook_search"]
|
||||||
Generated
+1109
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user