From 2e3e0ef6ad734b2457a128da4a2a7bc002e0019e Mon Sep 17 00:00:00 2001 From: Zachary Levonian Date: Fri, 12 Jun 2026 22:32:51 -0700 Subject: [PATCH] docs: prune patchwork Sphinx config and add class inheritance diagrams The conf.py was largely lifted from the matplotlib/pydata doc templates and carried a lot of unused machinery. Remove what isn't used and wire up the one piece worth keeping. Removed (all verified unused; build is clean under -W without them): - sphinx.ext.linkcode + the 55-line linkcode_resolve(), which was verbatim matplotlib boilerplate that always returned None here. viewcode already provides the working "[source]" links. - The no-op docutils_conf dict (Sphinx never reads a module-level var by that name). - numpydoc: the codebase is 100% Google-style (napoleon handles that), and its class-member tables were already disabled, so it did nothing. Dropping it also lets the numpydoc_show_class_members workaround go. - The three matplotlib.sphinxext.* extensions and sphinx_design / sphinx_tags, none of which are used by any directive. Their packages are dropped from the docs dependency group too. - Now-unused conf.py imports (inspect, Path, parse_version, flexeval). Added: - Two sphinx.ext.inheritance_diagram diagrams on the API page (the DataSource hierarchy and the peewee model hierarchy). These need Graphviz, so both CI workflows now install it and DEVELOPMENT.md notes the local prerequisite. - DEVELOPMENT.md notes that CI builds with -W (warnings are errors) and how to reproduce locally, plus a `make linkcheck` pointer. Addresses items 1, 2, 5, and 7 of #117. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/github-pages.yml | 5 ++ .github/workflows/validate.yaml | 5 ++ DEVELOPMENT.md | 38 ++++++++++--- docs/api.rst | 33 +++++++++++- docs/conf.py | 87 ++---------------------------- pyproject.toml | 3 -- uv.lock | 75 -------------------------- 7 files changed, 75 insertions(+), 171 deletions(-) diff --git a/.github/workflows/github-pages.yml b/.github/workflows/github-pages.yml index e515ea5..eeefbfe 100644 --- a/.github/workflows/github-pages.yml +++ b/.github/workflows/github-pages.yml @@ -43,6 +43,11 @@ jobs: with: python-version-file: ".python-version" + # Graphviz (`dot`) is required by sphinx.ext.inheritance_diagram for the + # class diagrams on the API page. + - name: Install Graphviz + run: sudo apt-get update && sudo apt-get install -y graphviz + - name: Install docs dependencies run: | uv sync --group docs diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 9ddd814..5efca9f 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -51,6 +51,11 @@ jobs: with: python-version-file: ".python-version" + # Graphviz (`dot`) is required by sphinx.ext.inheritance_diagram for the + # class diagrams on the API page. + - name: Install Graphviz + run: sudo apt-get update && sudo apt-get install -y graphviz + - name: Install docs dependencies run: | uv sync --group docs diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index edf89f0..dc7c12f 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -168,6 +168,15 @@ dependency group; `uv sync --all-groups` installs them too): uv sync --group docs ``` +The class diagrams on the API page are rendered by +[`sphinx.ext.inheritance_diagram`](https://www.sphinx-doc.org/en/master/usage/extensions/inheritance.html), +which needs [Graphviz](https://graphviz.org/) (the `dot` binary) on your `PATH`: + +```bash +brew install graphviz # macOS +sudo apt-get install graphviz # Debian/Ubuntu +``` + Build the site once: ```bash @@ -200,17 +209,32 @@ make docclean docautobuild ``` Sphinx prints `WARNING:` lines during the build and ends with a summary -(e.g. `build succeeded, N warnings.`). Warnings are worth scanning — they flag broken -cross-references, malformed docstrings, and members that failed to render. +(e.g. `build succeeded, N warnings.`). The CI docs build runs with +`SPHINXOPTS="-W --keep-going"`, which turns warnings into errors (and reports all of +them before failing), so the build must be warning-free to pass. Reproduce that +locally with: + +```bash +make docclean +make html SPHINXOPTS="-W --keep-going" +``` + +You can also check for broken links and cross-references (this makes network requests, +so it's not part of CI): + +```bash +make linkcheck +``` A note on the API reference: FlexEval's database models (`src/flexeval/classes/`) are [peewee](https://docs.peewee-orm.com/) models, and peewee's metaclass rewrites field definitions into descriptors and adds generated members (a per-model -`DoesNotExist` exception and a `_id` alias for every foreign key). The -`skip_peewee_internals` hook in `docs/conf.py` hides that generated noise while -keeping the real fields, and `inherited-members: False` keeps inherited peewee/pydantic -machinery out of the reference. If model attributes stop appearing on a generated page, -that hook (and the `autodoc_default_options`) is the place to look. +`DoesNotExist` exception, a `_id` alias for every foreign key, and a +back-reference accessor for every incoming relation). The `skip_peewee_internals` hook +in `docs/conf.py` hides that generated noise while keeping the real fields, and +`inherited-members: False` keeps inherited peewee/pydantic machinery out of the +reference. If model attributes stop appearing on a generated page, that hook (and the +`autodoc_default_options`) is the place to look. ## Releasing a new version diff --git a/docs/api.rst b/docs/api.rst index 5d9686a..f2118b4 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -5,10 +5,39 @@ The API for FlexEval is still largely undocumented, although the package is not A good place to start is with the Pydantic class :class:`~flexeval.schema.evalrun_schema.EvalRun`, which defines the inputs expected by FlexEval. -Packages: +Data sources +------------ + +FlexEval accepts several kinds of data source, all sharing a common +:class:`~flexeval.schema.evalrun_schema.DataSource` base: + +.. inheritance-diagram:: flexeval.schema.evalrun_schema.FileDataSource + flexeval.schema.evalrun_schema.NamedDataSource + flexeval.schema.evalrun_schema.IterableDataSource + :parts: 1 + :top-classes: flexeval.schema.evalrun_schema.DataSource + +Database models +--------------- + +Loaded data is stored in a hierarchy of `peewee `_ +models, all descending from a common base: + +.. inheritance-diagram:: flexeval.classes.dataset.Dataset + flexeval.classes.thread.Thread + flexeval.classes.turn.Turn + flexeval.classes.message.Message + flexeval.classes.tool_call.ToolCall + flexeval.classes.metric.Metric + flexeval.classes.eval_set_run.EvalSetRun + :parts: 1 + :top-classes: flexeval.classes.base.BaseModel + +Packages +-------- .. autosummary:: :toctree: generated :recursive: - flexeval \ No newline at end of file + flexeval diff --git a/docs/conf.py b/docs/conf.py index d3b7e35..ae79ceb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,14 +4,9 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html import sys import os -import inspect -from pathlib import Path -from packaging.version import parse as parse_version import peewee as pw -import flexeval - sys.path.append(os.path.abspath(".")) sys.path.append(".") @@ -30,24 +25,15 @@ extensions = [ "sphinx.ext.autodoc", "sphinx.ext.autosummary", - "sphinx.ext.inheritance_diagram", + "sphinx.ext.inheritance_diagram", # class diagrams on the API page (needs Graphviz) "sphinx.ext.intersphinx", - # "IPython.sphinxext.ipython_console_highlighting", - # "IPython.sphinxext.ipython_directive", - "numpydoc", # Needs to be loaded *after* autodoc. - "sphinx.ext.napoleon", - "matplotlib.sphinxext.plot_directive", - "matplotlib.sphinxext.roles", - "matplotlib.sphinxext.figmpl_directive", + "sphinx.ext.napoleon", # Google-style docstrings "sphinxext.github", "sphinx_copybutton", - "sphinx_design", - "sphinx_tags", - "sphinx.ext.linkcode", # "myst_parser", # don't use myst_parser with myst_nb; it is automatically loaded by myst_parser "myst_nb", "sphinxcontrib.autodoc_pydantic", # because sphinx plays badly with pydantic - "sphinx.ext.viewcode", # should add a view code link + "sphinx.ext.viewcode", # adds a "[source]" link next to documented objects "sphinxcontrib.programoutput", # for inline bash execution ] source_suffix = { @@ -91,67 +77,6 @@ "pydantic": ("https://docs.pydantic.dev/latest", None), } -docutils_conf = { - "line-length-limit": None, # Disable docutils line-length limit -} - - -def linkcode_resolve(domain, info): - """ - Determine the URL corresponding to Python object - """ - if domain != "py": - return None - - modname = info["module"] - fullname = info["fullname"] - - submod = sys.modules.get(modname) - if submod is None: - return None - - obj = submod - for part in fullname.split("."): - try: - obj = getattr(obj, part) - except AttributeError: - return None - - if inspect.isfunction(obj): - obj = inspect.unwrap(obj) - try: - fn = inspect.getsourcefile(obj) - except TypeError: - fn = None - if not fn or fn.endswith("__init__.py"): - try: - fn = inspect.getsourcefile(sys.modules[obj.__module__]) - except (TypeError, AttributeError, KeyError): - fn = None - if not fn: - return None - - try: - source, lineno = inspect.getsourcelines(obj) - except (OSError, TypeError): - lineno = None - - linespec = f"#L{lineno:d}-L{lineno + len(source) - 1:d}" if lineno else "" - - startdir = Path(flexeval.__file__).parent.parent - try: - fn = os.path.relpath(fn, start=startdir).replace(os.path.sep, "/") - except ValueError: - return None - - if not fn.startswith(("matplotlib/", "mpl_toolkits/")): - return None - - version = parse_version(flexeval.__version__) - tag = "main" if version.is_devrelease else f"v{version.public}" - return f"https://github.com/matplotlib/matplotlib/blob/{tag}/lib/{fn}{linespec}" - - # myst-parser # https://myst-parser.readthedocs.io/en/latest/configuration.html myst_gfm_only = False @@ -170,12 +95,6 @@ def linkcode_resolve(domain, info): nb_merge_streams = True autosummary_generate = True -# Don't let numpydoc inject its own per-class "Methods"/"Attributes" summary -# tables. They duplicate the member documentation autodoc already renders below, -# and for our peewee models they list every inherited peewee.Model method -# (save, select, bulk_create, ...) as noise. Disabling this also avoids the -# "stub file not found" warnings those tables' :toctree: would otherwise emit. -numpydoc_show_class_members = False autodoc_typehints = "signature" # Some models hold fields that have no JSON-schema representation (e.g. # FunctionsCollection.functions is a list[Callable]). Coerce rather than warn so diff --git a/pyproject.toml b/pyproject.toml index 691c690..1f7ae15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,13 +79,10 @@ docs = [ "linkify-it-py>=2.0.3", "myst-nb>=1.2.0", "myst-parser>=4.0.1", - "numpydoc>=1.9.0", "pydata-sphinx-theme>=0.16.1", "sphinx>=8.1.3", "sphinx-autobuild>=2024.10.3", "sphinx-copybutton>=0.5.2", - "sphinx-design>=0.6.1", - "sphinx-tags>=0.4", "sphinxcontrib-programoutput>=0.18", ] diff --git a/uv.lock b/uv.lock index 1febf2a..f5c1667 100644 --- a/uv.lock +++ b/uv.lock @@ -3498,21 +3498,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, ] -[[package]] -name = "numpydoc" -version = "1.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/3c/dfccc9e7dee357fb2aa13c3890d952a370dd0ed071e0f7ed62ed0df567c1/numpydoc-1.10.0.tar.gz", hash = "sha256:3f7970f6eee30912260a6b31ac72bba2432830cd6722569ec17ee8d3ef5ffa01", size = 94027, upload-time = "2025-12-02T16:39:12.937Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/5e/3a6a3e90f35cea3853c45e5d5fb9b7192ce4384616f932cf7591298ab6e1/numpydoc-1.10.0-py3-none-any.whl", hash = "sha256:3149da9874af890bcc2a82ef7aae5484e5aa81cb2778f08e3c307ba6d963721b", size = 69255, upload-time = "2025-12-02T16:39:11.561Z" }, -] - [[package]] name = "openai" version = "2.41.1" @@ -4496,7 +4481,6 @@ docs = [ { name = "myst-nb" }, { name = "myst-parser", version = "4.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "myst-parser", version = "5.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "numpydoc" }, { name = "pydata-sphinx-theme" }, { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, @@ -4504,9 +4488,6 @@ docs = [ { name = "sphinx-autobuild", version = "2024.10.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "sphinx-autobuild", version = "2025.8.25", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "sphinx-copybutton" }, - { name = "sphinx-design", version = "0.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx-design", version = "0.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "sphinx-tags" }, { name = "sphinxcontrib-programoutput" }, ] @@ -4550,13 +4531,10 @@ docs = [ { name = "linkify-it-py", specifier = ">=2.0.3" }, { name = "myst-nb", specifier = ">=1.2.0" }, { name = "myst-parser", specifier = ">=4.0.1" }, - { name = "numpydoc", specifier = ">=1.9.0" }, { name = "pydata-sphinx-theme", specifier = ">=0.16.1" }, { name = "sphinx", specifier = ">=8.1.3" }, { name = "sphinx-autobuild", specifier = ">=2024.10.3" }, { name = "sphinx-copybutton", specifier = ">=0.5.2" }, - { name = "sphinx-design", specifier = ">=0.6.1" }, - { name = "sphinx-tags", specifier = ">=0.4" }, { name = "sphinxcontrib-programoutput", specifier = ">=0.18" }, ] @@ -5502,59 +5480,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/48/1ea60e74949eecb12cdd6ac43987f9fd331156388dcc2319b45e2ebb81bf/sphinx_copybutton-0.5.2-py3-none-any.whl", hash = "sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e", size = 13343, upload-time = "2023-04-14T08:10:20.844Z" }, ] -[[package]] -name = "sphinx-design" -version = "0.6.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -dependencies = [ - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2b/69/b34e0cb5336f09c6866d53b4a19d76c227cdec1bbc7ac4de63ca7d58c9c7/sphinx_design-0.6.1.tar.gz", hash = "sha256:b44eea3719386d04d765c1a8257caca2b3e6f8421d7b3a5e742c0fd45f84e632", size = 2193689, upload-time = "2024-08-02T13:48:44.277Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/43/65c0acbd8cc6f50195a3a1fc195c404988b15c67090e73c7a41a9f57d6bd/sphinx_design-0.6.1-py3-none-any.whl", hash = "sha256:b11f37db1a802a183d61b159d9a202314d4d2fe29c163437001324fe2f19549c", size = 2215338, upload-time = "2024-08-02T13:48:42.106Z" }, -] - -[[package]] -name = "sphinx-design" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and sys_platform == 'win32'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", - "python_full_version == '3.11.*' and sys_platform == 'emscripten'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", -] -dependencies = [ - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/13/7b/804f311da4663a4aecc6cf7abd83443f3d4ded970826d0c958edc77d4527/sphinx_design-0.7.0.tar.gz", hash = "sha256:d2a3f5b19c24b916adb52f97c5f00efab4009ca337812001109084a740ec9b7a", size = 2203582, upload-time = "2026-01-19T13:12:53.297Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/30/cf/45dd359f6ca0c3762ce0490f681da242f0530c49c81050c035c016bfdd3a/sphinx_design-0.7.0-py3-none-any.whl", hash = "sha256:f82bf179951d58f55dca78ab3706aeafa496b741a91b1911d371441127d64282", size = 2220350, upload-time = "2026-01-19T13:12:51.077Z" }, -] - -[[package]] -name = "sphinx-tags" -version = "0.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/10/de/cd7669ce4619bf03418d428b4e68f20bc90f9de0d9ed36b1d43147d6e96d/sphinx_tags-0.4.tar.gz", hash = "sha256:3065219bacf47567c706f2237d5665b22f3a5167367b4c4568bcda4371a5359d", size = 15292, upload-time = "2024-07-08T15:43:24.068Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/70/71/1fb5aa2013e04d36e6940b308e65608a48e2d5f59f2a21c98f01f2ce0b85/sphinx_tags-0.4-py2.py3-none-any.whl", hash = "sha256:af31203fe4e0ebd39cb95d4fef38720185def6af9fdb921b70275ff659a5dac5", size = 7231, upload-time = "2024-07-08T15:43:23.077Z" }, -] - [[package]] name = "sphinxcontrib-applehelp" version = "2.0.0"