From 48a686ad7a26eb4e238d904de3b990a5ada133da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Moreira=20Souza?= Date: Fri, 31 Jul 2026 10:24:02 -0300 Subject: [PATCH 1/3] feat: add accelerated(), a supported way to ask whether numba is in use 0.7.0's acceleration is invisible when it works, and there was no supported way to tell whether it had taken effect. A draft of the how-to reached for `from python_som._accelerate import bmu_kernel`, which is a private module and not something to publish. A function rather than a constant. `NUMBA_AVAILABLE` would have to resolve at import time, forcing the numba import on every `import python_som`; that costs about 80 ms and 0.7.0 deferred it deliberately. A function defers it to whoever asked for the answer. Named for the capability rather than the backend, so replacing numba later is not a rename. The docs still say numba where they tell you what to install. Calling it imports numba but does not compile the kernel: njit is lazy, so the roughly 500 ms compile happens on the first call that trains. Both figures are measured and in the docstring. Tested in both directions, in the two files that can each see only one of them: test_numba_kernel.py asserts True where numba is installed, test_core_boundary.py asserts False where it is not and skips if numba is unexpectedly present. A predicate checked one way round would pass while always returning the same answer. Two boundary assertions come with it. Calling accelerated() must not import numba when numba is absent, and `import python_som` must still not import it when present, which is exactly the invariant a new public name could break. The pinned public surface in test_core_boundary.py grows by one entry, so the addition is a decision rather than an accident. --- src/python_som/__init__.py | 2 ++ src/python_som/_accelerate.py | 17 ++++++++++++- tests/test_core_boundary.py | 45 ++++++++++++++++++++++++++++++----- tests/test_numba_kernel.py | 24 +++++++++++++++++++ 4 files changed, 81 insertions(+), 7 deletions(-) diff --git a/src/python_som/__init__.py b/src/python_som/__init__.py index 7e0c852..7cdbd19 100644 --- a/src/python_som/__init__.py +++ b/src/python_som/__init__.py @@ -19,6 +19,7 @@ from __future__ import annotations +from ._accelerate import accelerated from ._artifact import ArtifactError, SOMConfig, TrainingReport from ._core._decay import ( asymptotic_decay, @@ -75,6 +76,7 @@ "TrainingReport", "WeightInit", "WeightInitStr", + "accelerated", "asymptotic_decay", "bubble", "euclidean_distance", diff --git a/src/python_som/_accelerate.py b/src/python_som/_accelerate.py index 1a7ca55..cc49730 100644 --- a/src/python_som/_accelerate.py +++ b/src/python_som/_accelerate.py @@ -30,7 +30,7 @@ from ._core._protocols import BmuKernel -__all__ = ["bmu_kernel"] +__all__ = ["accelerated", "bmu_kernel"] @functools.cache @@ -86,3 +86,18 @@ def fused_bmu( # pragma: no cover kernel: BmuKernel = fused_bmu # pragma: no cover only when numba is installed return kernel # pragma: no cover + + +def accelerated() -> bool: + """Whether training will use the compiled kernel. + + True once numba is installed; nothing else is needed to enable it, and results are identical + either way. + + **Calling this imports numba**, which takes about 80 ms the first time. It does not compile the + kernel: ``njit`` is lazy, so the roughly 500 ms compile happens on the first call that actually + trains. Both are paid once per process. + + :return: True if the compiled kernel is available. + """ + return bmu_kernel() is not None diff --git a/tests/test_core_boundary.py b/tests/test_core_boundary.py index 4d7f1b5..c62bcdc 100644 --- a/tests/test_core_boundary.py +++ b/tests/test_core_boundary.py @@ -7,9 +7,11 @@ from __future__ import annotations import ast -import importlib +import importlib.util import pathlib import pkgutil +import subprocess +import sys import numpy as np import pytest @@ -259,11 +261,9 @@ def test_nothing_that_0_3_0_exported_has_been_removed() -> None: def test_the_public_surface_is_exactly_this() -> None: """Pin the whole surface, so growing it is a decision rather than an accident. - 0.4.0 adds the enums and their ``Literal`` counterparts, the strategy protocols, and the - artifact types. All are additive: every existing call keeps working, and the enum members are - ``str`` subclasses that compare equal to the strings they replace. ``__version__`` is - deliberately absent -- it is re-exported with the ``as`` idiom, since ``__all__`` is the public - API and a dunder is not part of it. + Everything here is additive across 0.4.0 to 0.8.0 and every existing call keeps working. + ``__version__`` is deliberately absent: it is re-exported with the ``as`` idiom, and ``__all__`` + is the public API where a dunder is not. """ assert sorted(python_som.__all__) == [ "ArtifactError", @@ -284,6 +284,7 @@ def test_the_public_surface_is_exactly_this() -> None: "TrainingReport", "WeightInit", "WeightInitStr", + "accelerated", "asymptotic_decay", "bubble", "euclidean_distance", @@ -431,3 +432,35 @@ def test_error_messages_never_leak_a_private_name(kwargs: dict[str, object], exp message = str(excinfo.value) assert "_init_" not in message, message assert "_som" not in message, message + + +def test_accelerated_is_false_without_numba() -> None: + """The default install has no numba, so the predicate must say so. + + The True direction is in ``tests/test_numba_kernel.py``, which only runs where numba is present. + Skipped rather than failed if numba happens to be installed here, so the two files cannot both + be wrong in the same direction. + """ + if importlib.util.find_spec("numba") is not None: + pytest.skip("numba is installed; the True direction is asserted elsewhere") + assert python_som.accelerated() is False + + +def test_asking_whether_it_is_accelerated_does_not_import_numba() -> None: + """``accelerated()`` is public now, so it is a new way to accidentally import numba eagerly. + + A subprocess, because this one may already have imported it. + """ + code = ( + "import sys, python_som; " + "assert python_som.accelerated() in (True, False); " + "import importlib.util; " + "expected = importlib.util.find_spec('numba') is not None; " + "assert ('numba' in sys.modules) == expected, sys.modules.keys(); " + "print('clean')" + ) + result = subprocess.run( # noqa: S603 + [sys.executable, "-c", code], capture_output=True, text=True, check=False + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "clean" in result.stdout diff --git a/tests/test_numba_kernel.py b/tests/test_numba_kernel.py index 812abaf..faac633 100644 --- a/tests/test_numba_kernel.py +++ b/tests/test_numba_kernel.py @@ -177,3 +177,27 @@ def test_importing_the_package_does_not_import_numba() -> None: ) assert result.returncode == 0, result.stdout + result.stderr assert "clean" in result.stdout + + +def test_accelerated_reports_true() -> None: + """The public predicate must agree with the kernel actually being there. + + The False direction is in ``tests/test_core_boundary.py``, which runs in the environment without + numba. A predicate checked in only one direction would pass while always returning the same + answer. + """ + assert python_som.accelerated() is True + + +def test_asking_twice_does_not_resolve_twice() -> None: + """``bmu_kernel`` is cached, so repeated calls cost nothing after the first. + + Without the cache, every ``accelerated()`` call would redefine the jitted function. + """ + bmu_kernel() + before = bmu_kernel.cache_info() + python_som.accelerated() + python_som.accelerated() + after = bmu_kernel.cache_info() + assert after.misses == before.misses + assert after.hits >= before.hits + 2 From 47ef4e2f40312f963f876cdd7ff97c9fdf3cfe31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Moreira=20Souza?= Date: Fri, 31 Jul 2026 10:24:16 -0300 Subject: [PATCH 2/3] docs: make the numba acceleration findable 0.7.0's Install section listed four pip commands and numba was not one of them. Its only user-facing mention was a bullet in the feature list, and the documentation site mentioned numba exactly once, in a caveat about MiniSom's development branch. Nothing told a reader the option existed. The Install section now carries the command, the NumPy consequence of running it, and a link to a new how-to. A packaging test asserts that section rather than the description as a whole, which is the distinction that matters here: the string was already present in 0.7.0, just nowhere a reader deciding what to install would look. The new page is a how-to and stays one. An earlier draft explained why numba is not an extra, quoted Kohonen on batch convergence, and described which phase of training dominates; all three are rationale, and a how-to that teaches is the failure the mode exists to avoid. Each now links to the explanation page that already covers it. What remains is four steps in order of effect and what each costs the reader. Three claims in that draft were wrong and are corrected. The confirmation snippet imported a private module. "A one-off compile of about a second" is 80 ms to import numba and 500 ms to compile, measured separately. "A 100x100 map is roughly 25 times the work of a 20x20 one" was invented and wrong in shape, since the update grows faster than node count; it now points at the measured table. The Kohonen rule of thumb was checked against Section 3.6, p. 56, which reads "about 50 items per node on the average". "About", not "at least". --- README.md | 12 +++++- docs/how-to/speed-up-training.md | 64 ++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + tests/test_packaging.py | 36 ++++++++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 docs/how-to/speed-up-training.md diff --git a/README.md b/README.md index 739d207..f516430 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,16 @@ pip install "python-som[sklearn]" # adds the scikit-learn estimator adapte pip install "python-som[examples]" # adds matplotlib and seaborn, for the plots ``` +Training can also use a compiled kernel, worth up to 2.4x on batch training. It is a separate +package rather than an extra, and it is picked up automatically once present: + +```bash +pip install numba # note: numba currently requires numpy<2.5 +``` + +Confirm it is active with `python_som.accelerated()`. See +[Speed up training](https://andremsouza.github.io/python-som/how-to/speed-up-training/). + ## Quick start ```python @@ -57,7 +67,7 @@ A full worked example with plots is in [examples/iris.py](https://github.com/and * NumPy is the only runtime dependency; a fresh install is 69 MB across one package * Batch training 23x to 31x faster than MiniSom and 26x to 94x faster than SOMPY, measured -* Optional numba acceleration: `pip install numba` and it is used automatically +* Optional compiled kernel via numba, used automatically when installed; `accelerated()` reports it * Stepwise and batch training * Random, random-sampling and linear (PCA) weight initialization * Automatic selection of the map size ratio, from PCA diff --git a/docs/how-to/speed-up-training.md b/docs/how-to/speed-up-training.md new file mode 100644 index 0000000..435ab89 --- /dev/null +++ b/docs/how-to/speed-up-training.md @@ -0,0 +1,64 @@ +# Speed up training + +Four changes, ordered from largest effect to smallest. The first two cost nothing. + +## 1. Use batch training + +```python +som.train(data, n_iteration=100, mode="batch") +``` + +It is the default, it is the mode the optimization work targets, and it needs far fewer iterations: +10 per sample against 1000 for the stepwise modes. If you are passing `mode="random"` or +`mode="sequential"` for no particular reason, change it. + +See [Batch vs stepwise](../explanation/batch-vs-stepwise.md) for what the modes do differently. + +## 2. Keep the default distance function + +The winner search has a fast path that applies only to the Euclidean distance. Supplying your own +`distance_function` falls back to one evaluation per sample, several times slower. If you need a +custom distance, see [Use a custom strategy](use-a-custom-strategy.md); if you do not, leave it +alone. + +## 3. Install numba + +```bash +pip install numba +``` + +Nothing else to do. The compiled kernel is detected and used automatically, and results are +identical to the NumPy path. Check it took effect: + +```python +import python_som + +python_som.accelerated() # True once numba is installed +``` + +Three things to expect: + +- **NumPy is downgraded in that environment.** numba 0.66 requires `numpy<2.5`. Install it into a + virtual environment you are willing to hold below that. +- **The gain is uneven.** Measured on batch training: 2.4x on a 60x60 map, about 1.0x on a 20x20 + or a 100x100 one. Measure your own case before accepting the NumPy constraint. +- **The first run is slower.** About 80 ms to import numba and 500 ms to compile, once per process. + +## 4. Size the map for the data + +Cost grows with the node count, faster than linearly. Kohonen's rule of thumb is about 50 samples +per node on average; well past that, you are paying for resolution the data does not support. + +Let the constructor pick the ratio from the data rather than choosing both sides yourself: + +```python +som = python_som.SOM(x=20, y=None, input_len=4, data=data) +``` + +The [measured timings](../explanation/comparison-with-som-libraries.md#against-minisom) show what +map size costs across a range of shapes. + +## Why these and not others + +[How batch training is computed](../explanation/how-batch-training-is-computed.md) covers what the +library does internally, including the approaches that were measured and rejected. diff --git a/mkdocs.yml b/mkdocs.yml index d52e3c9..42b3197 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -34,6 +34,7 @@ nav: - How-to: - Save and load a map: how-to/save-and-load-a-map.md - Reproduce a result: how-to/reproduce-a-result.md + - Speed up training: how-to/speed-up-training.md - Use a custom strategy: how-to/use-a-custom-strategy.md - Use the estimator interface: how-to/use-with-scikit-learn.md - Reference: diff --git a/tests/test_packaging.py b/tests/test_packaging.py index a93066b..b71c15d 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -101,3 +101,39 @@ def test_dev_dependencies_are_pinned_exactly(pyproject: dict[str, Any]) -> None: for spec in pyproject["project"]["optional-dependencies"][group]: requirement = spec.split(";")[0].strip() assert "==" in requirement, f"{group} dependency is not pinned: {spec}" + + +def _readme_section(name: str) -> str: + """Return one ``## `` section of the README, which is also the PyPI description. + + :param name: Heading text, without the ``## ``. + :return: The section body. + """ + lines = (PROJECT_ROOT / "README.md").read_text(encoding="utf-8").splitlines() + start = next(n for n, line in enumerate(lines) if line.strip() == f"## {name}") + end = next((n for n in range(start + 1, len(lines)) if lines[n].startswith("## ")), len(lines)) + return "\n".join(lines[start:end]) + + +def test_the_install_section_mentions_every_installable_option() -> None: + """The README is the PyPI description, and a published description cannot be edited. + + 0.7.0 shipped the numba acceleration with its only user-facing mention in the feature list, so + someone reading Install to decide what to install never saw it. Correcting that took a release. + Asserting on the Install section rather than the whole description is the point: the string was + present in 0.7.0 and still in the wrong place. + """ + section = _readme_section("Install") + for option in ( + "pip install python-som", + "[cli]", + "[sklearn]", + "[examples]", + "pip install numba", + ): + assert option in section, f"the Install section does not mention {option!r}" + + +def test_the_install_section_warns_that_numba_constrains_numpy() -> None: + """Installing numba downgrades NumPy, which a reader has to know before running the command.""" + assert "numpy<2.5" in _readme_section("Install") From 4543df523054681b1b160d7e4e0aff6d80cd5bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Moreira=20Souza?= Date: Fri, 31 Jul 2026 10:24:26 -0300 Subject: [PATCH 3/3] chore: release 0.8.0 Minor rather than patch: accelerated() is new public API, and semver puts new functionality in a minor release. Nothing else about it is minor-shaped, so the entry says plainly that no behaviour and no numerical results change, and there is no reproducibility note for the first time in three releases. The release exists because a PyPI description cannot be edited after upload, which is the same reason 0.6.1 existed. 0.7.0's description mentions numba only in its feature list, and the Install section a reader consults is what needed correcting. --- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- src/python_som/_version.py | 2 +- uv.lock | 2 +- 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f4fdc8..f7a8e3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,43 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.0] - 2026-07-31 + +0.7.0 shipped optional numba acceleration that a user had no way to find or to check. This makes it +discoverable and adds one public function to inspect it. No behaviour changes and no numerical +changes. + +### Added + +- **`python_som.accelerated()`**, which reports whether training will use the compiled kernel: + + ```python + import python_som + + python_som.accelerated() # True once numba is installed + ``` + + A function rather than a constant because a constant would have to resolve at import time, which + forces the numba import on every `import python_som`. That import costs about 80 ms, and 0.7.0 + deferred it deliberately; a function defers it to the caller who asked. Named for the capability + rather than the backend, so a future change of backend is not a rename. + + Calling it imports numba but does not compile the kernel: `njit` is lazy, so the roughly 500 ms + compile happens on the first call that trains. + +- **[Speed up training](https://andremsouza.github.io/python-som/how-to/speed-up-training/)**, a + how-to covering the four levers in order of effect: batch mode, the default distance function, + numba, and map size. + +### Fixed + +- **The numba acceleration is now findable.** 0.7.0's Install section listed four `pip install` + lines and numba was not among them; the only user-facing mention was one bullet in the feature + list, and the documentation site mentioned numba just once, in a caveat about MiniSom. A published + PyPI description cannot be edited, so correcting it takes a release. The README's Install section + now carries the command, its NumPy consequence, and a link to the how-to, and a packaging test + asserts that section of the built metadata rather than the description as a whole. + ## [0.7.0] - 2026-07-31 Batch training is 20x to 40x faster and now beats both comparable libraries by a wide margin. The diff --git a/pyproject.toml b/pyproject.toml index a8cd067..73ac473 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "python-som" -version = "0.7.0" +version = "0.8.0" authors = [{ name = "André Moreira Souza", email = "msouza.andre@hotmail.com" }] description = "Python implementation of the Self-Organizing Map" readme = "README.md" diff --git a/src/python_som/_version.py b/src/python_som/_version.py index d4ec072..6ead870 100644 --- a/src/python_som/_version.py +++ b/src/python_som/_version.py @@ -9,4 +9,4 @@ __all__ = ["__version__"] -__version__ = "0.7.0" +__version__ = "0.8.0" diff --git a/uv.lock b/uv.lock index 1f9be63..5ee710e 100644 --- a/uv.lock +++ b/uv.lock @@ -2650,7 +2650,7 @@ wheels = [ [[package]] name = "python-som" -version = "0.7.0" +version = "0.8.0" source = { editable = "." } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },