diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7f5afd7d..f7060095 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -83,51 +83,3 @@ jobs: path: dist/ - name: publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 - - # ------------------------------------ - - build-oeq-extjax: - name: Build extjax dist - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-tags: true - - name: set up Python - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 - with: - python-version: '3.10' - - name: replace symlinks with copies - run: python3 .github/release.py resolve-symlinks - - name: set version from tag - run: | - python3 -m pip install -r .github/workflows/requirements_release_ci.txt - python3 .github/release.py set-version openequivariance_extjax/pyproject.toml - - name: install dependencies, then build source tarball - run: | - cd openequivariance_extjax - python3 -m pip install build - python3 -m build --sdist - - name: store the distribution packages - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: oeqjax-package-distributions - path: openequivariance_extjax/dist/ - - pypi-publish-extjax: - name: Upload extjax to PyPI - runs-on: ubuntu-latest - needs: build-oeq-extjax - environment: - name: pypi - url: https://pypi.org/p/openequivariance_extjax - permissions: - id-token: write - steps: - - name: download the distributions - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: oeqjax-package-distributions - path: dist/ - - name: publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 diff --git a/.github/workflows/verify_extension_build.yml b/.github/workflows/verify_extension_build.yml index c33a4da2..5b5d70aa 100644 --- a/.github/workflows/verify_extension_build.yml +++ b/.github/workflows/verify_extension_build.yml @@ -29,16 +29,12 @@ jobs: sudo apt-get update sudo apt install nvidia-cuda-toolkit pip install -r .github/workflows/requirements_cuda_ci.txt - pip install -e "./openequivariance[jax]" + pip install -e "./openequivariance" - name: Test CUDA extension build via import run: | pytest tests/import_test.py - - name: Test JAX extension build - run: | - XLA_DIRECT_DOWNLOAD=1 pip install -e "./openequivariance_extjax" --no-build-isolation - verify_hip_extension: if: ${{ github.event.label.name == 'ci-ready' || github.event_name != 'pull_request' }} runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 99f3887c..634d73e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ ## Latest Changes +### v0.8.2 (2026-11-17) + +This release eliminates the need +for `openequivariance_extjax` by moving +the JAX extension to a precompiled library +inside a wheeldist. Also introduces +quality-of-life changes to JIT branch. + + +**Deprecated**: +- JAX users no longer need to +install `openequivariance_extjax` in addition +to the regular `openequivariance` package. + +### v0.8.1 (2026-11-10) + +This release is a preliminary test of precompiled library wheel distribution. + **Added**: - Route symmetric contraction's grouped matrix multiplications through PyTorch on CUDA, HIP, and SYCL, removing direct cuBLAS and rocBLAS dependencies. diff --git a/README.md b/README.md index 24fa7a0f..c710f645 100644 --- a/README.md +++ b/README.md @@ -34,21 +34,7 @@ OpenEquivariance now supports Intel XPUs on PyTorch by compiling SYCL kernels. See our installation guide for more details. -⭐️ **JAX**: We support JAX for NVIDIA and AMD GPUs. -For NVIDIA GPUs, install it (after installing JAX) -with the following two commands strictly in order: - -``` bash -pip install openequivariance[jax] -pip install openequivariance_extjax --no-build-isolation -``` - -For AMD GPUs: -``` bash -pip install openequivariance[jax] -JAX_HIP=1 pip install openequivariance_extjax --no-build-isolation -``` - +⭐️ **JAX**: We support JAX for NVIDIA and AMD GPUs. See the section below for example usage and our [API page](https://passionlab.github.io/OpenEquivariance/api/) for more details. diff --git a/docs/conf.py b/docs/conf.py index 70dd285e..29499457 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -37,7 +37,6 @@ "jax", "openequivariance._torch.extlib", "openequivariance.jax.extlib", - "openequivariance_extjax", "openequivariance.jax.jvp.tp_prim", "openequivariance.jax.jvp.conv_prim", "jinja2", diff --git a/docs/installation.rst b/docs/installation.rst index 8405f77f..5b96f407 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -1,6 +1,15 @@ Installation ============================== +.. admonition:: Deprecation of ``openequivariance_extjax`` + :class: tip + + Installing ``openequivariance_extjax`` is no longer + required or supported, since the regular + ``openequivariance`` package contains the + relevant binaries. + + .. toctree:: :maxdepth: 1 :caption: Contents: @@ -8,103 +17,93 @@ Installation You need the following to install OpenEquivariance: - A Linux system equipped with an NVIDIA / AMD / Intel graphics card. -- Either PyTorch >= 2.10, or - JAX>0.5.0 with CUDA, RocM, or XPU support. -- To build from source: GCC 9+ and the CUDA / HIP toolkit, or the - oneAPI DPC++ compiler (``icpx``) for Intel GPUs. The command - ``c++ --version`` should return >= 9.0; see below for details on - setting an alternate compiler. - -.. note:: - - On Intel GPUs the kernels are generated as SYCL and compiled at runtime - through the oneAPI kernel compiler, so ``icpx`` must be on your ``PATH`` - (PyTorch locates the SYCL toolchain from it). Tensors passed to the - kernels live on the ``xpu`` device rather than ``cuda``. The precompiled - extension and the JAX frontend both remain CUDA/HIP only. +- Either PyTorch >= 2.10, or JAX >= 0.8.1 with CUDA, ROCm, or XPU support. -.. tab:: PyTorch +.. tab:: CUDA or HIP Wheels - Installation is one easy command, followed by import verification: + Installation is one easy command for all PyTorch, JAX, CUDA, and + HIP platforms: .. code-block:: bash pip install openequivariance - python -c "import openequivariance" OpenEquivariance bundles a precompiled library files for two recent CUDA versions and a HIP version. If you're building for an architecture - we don't support or need a different CUDA version, you'll need to build - the package from source: + we don't support or need a different CUDA version, see the source + distribution installation instructions. - .. code-block:: bash - pip install --no-binary openequivariance openequivariance - - Set the environment variables ``CC`` and ``CXX`` before the pip install - to build with a different compiler: + If you're using JAX, set the environment variable + ``OEQ_NOTORCH=1`` to avoid a PyTorch import: .. code-block:: bash - export CC=/path/to/your/gcc - export CXX=/path/to/your/g++ + export OEQ_NOTORCH=1 + python -c "import openequivariance.jax" - On Intel GPUs, the extension is still compiled on first import, - which can take a couple of minutes; subsequent imports use the - cached build. Install the extra build dependencies with: +.. tab:: CUDA or HIP Source Distribution + + Run .. code-block:: bash - pip install openequivariance[sycl] + pip install --no-binary openequivariance openequivariance + Optionally, set the environment variables ``CC`` and ``CXX`` before the pip install + to build with a different compiler: -.. tab:: JAX NVIDIA GPUs + .. code-block:: bash - First ensure the appropriate JAX Python - package is installed in your environment. Then - run the following two commands stricly in order: + export CC=/path/to/your/gcc + export CXX=/path/to/your/g++ + pip install --no-binary openequivariance openequivariance - .. code-block:: bash - pip install openequivariance[jax] - pip install openequivariance_extjax --no-build-isolation +.. tab:: SYCL -.. tab:: JAX AMD GPUs + You'll need the oneAPI DPC++ compiler (``icpx``) for Intel GPUs. + Kernels are generated as SYCL and compiled at runtime + through the oneAPI kernel compiler, so ``icpx`` must be on your ``PATH`` + (PyTorch locates the SYCL toolchain from it). Tensors passed to the + kernels are resident on the ``xpu`` device rather than ``cuda``. - Ensure that JAX is installed correctly with RocM support - before running, in order, + Our SYCL extension is still compiled on first import, + which can take a couple of minutes; subsequent imports use the + cached build. Install via - .. code-block:: bash + .. code-block:: bash - pip install openequivariance[jax] - JAX_HIP=1 pip install openequivariance_extjax --no-build-isolation + pip install "openequivariance[sycl]" + export CC=/path/to/your/cc + export CXX=/path/to/your/c++ + python -c "import openequivariance" -.. tab:: Nightly (PT) +.. tab:: Developer Install - .. code-block:: bash + This mode enables fast recompilation for + incremental builds. Build isolation is disabled + in this mode: - pip install "git+https://github.com/PASSIONLab/OpenEquivariance#subdirectory=openequivariance" + .. code-block:: bash + git clone https://github.com/PASSIONLab/OpenEquivariance + cd OpenEquivariance + pip install $(python -c "import tomllib; print(*tomllib.load(open('openequivariance/pyproject.toml', 'rb'))['build-system']['requires'])") + pip install --no-build-isolation -Ceditable.rebuild=true -e ./openequivariance -.. tab:: Nightly (JAX) +.. tab:: Nightly .. code-block:: bash - pip install "openequivariance[jax] @ git+https://github.com/PASSIONLab/OpenEquivariance#subdirectory=openequivariance" - pip install "git+https://github.com/PASSIONLab/OpenEquivariance#subdirectory=openequivariance_extjax" --no-build-isolation + pip install --force-reinstall "git+https://github.com/PASSIONLab/OpenEquivariance#subdirectory=openequivariance" - # Use the command below for JAX+AMD - # JAX_HIP=1 pip install "git+https://github.com/PASSIONLab/OpenEquivariance#subdirectory=openequivariance_extjax" --no-build-isolation + To install from a specific branch on our repo: + .. code-block:: bash -If you're using JAX, set the environment variable -``OEQ_NOTORCH=1`` to avoid a PyTorch import: - -.. code-block:: bash - - export OEQ_NOTORCH=1 - python -c "import openequivariance.jax" + pip install --force-reinstall "git+https://github.com/PASSIONLab/OpenEquivariance@my-test-branch#subdirectory=openequivariance" Configurations on Major Platforms diff --git a/docs/tests_and_benchmarks.rst b/docs/tests_and_benchmarks.rst index e0be94b6..97e031a4 100644 --- a/docs/tests_and_benchmarks.rst +++ b/docs/tests_and_benchmarks.rst @@ -35,9 +35,8 @@ To set up an editable install and run our tests, use the following code: git clone https://github.com/PASSIONLab/OpenEquivariance cd OpenEquivariance - pip install "./openequivariance[jax]" + pip install "./openequivariance" pip install "./openequivariance[dev]" - pip install "./openequivariance_extjax" --no-build-isolation pytest --jax tests/example_test.py pytest --jax tests/batch_test.py diff --git a/openequivariance/CMakeLists.txt b/openequivariance/CMakeLists.txt index 1ceeadbd..4e6c084f 100644 --- a/openequivariance/CMakeLists.txt +++ b/openequivariance/CMakeLists.txt @@ -40,7 +40,6 @@ message(STATUS "LibTorch Lib: ${LIBTORCH_LIB_DIR}") message(STATUS "Torch CPU Library: ${TORCH_CPU_LIB}") message(STATUS "Torch C10 Library: ${C10_LIB}") -# Setup Nanobind execute_process( COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_ROOT @@ -61,38 +60,53 @@ set(OEQ_SOURCES set(OEQ_INSTALL_DIR "openequivariance/_torch/extlib") -function(add_stable_extension target_name backend_define link_libraries) - nanobind_add_module(${target_name} BACKEND_MODULE nanobind_backend ${OEQ_SOURCES}) +if(NOT XLA_FFI_URL OR NOT XLA_FFI_C_API_SHA256 OR NOT XLA_FFI_API_SHA256 OR NOT XLA_FFI_FFI_SHA256) + message(FATAL_ERROR "XLA_FFI_URL / XLA_FFI_*_SHA256 not set") +endif() + +set(XLA_FFI_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/xla_ffi") +foreach(header c_api api ffi) + string(TOUPPER "${header}" header_var) + file(DOWNLOAD + "${XLA_FFI_URL}/${header}.h" + "${XLA_FFI_INCLUDE_DIR}/xla/ffi/api/${header}.h" + EXPECTED_HASH SHA256=${XLA_FFI_${header_var}_SHA256} + TLS_VERIFY ON + ) +endforeach() + +set(OEQ_JAX_SOURCES + ${EXT_DIR}/jax/ffi_handlers.cpp + ${EXT_DIR}/jax/ffi_provider.cpp + ${EXT_JSON_DIR}/json11.cpp +) + +set(OEQ_JAX_INSTALL_DIR "openequivariance/jax/extlib") + +function(add_jax_extension target_name backend_define link_libraries) + add_library(${target_name} SHARED ${OEQ_JAX_SOURCES}) set_target_properties(${target_name} PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON POSITION_INDEPENDENT_CODE ON ) - - # Enforce CXX11 ABI to match LibTorch - target_compile_definitions(${target_name} PRIVATE - ${backend_define}=1 - _GLIBCXX_USE_CXX11_ABI=1 - INCLUDE_NB_EXTENSION - EXTENSION_NAME=${target_name} - ) - + target_compile_options(${target_name} PRIVATE -Wno-attributes -Wno-return-type) + target_compile_definitions(${target_name} PRIVATE ${backend_define}=1) target_include_directories(${target_name} PRIVATE ${EXT_DIR} + ${EXT_DIR}/jax ${EXT_BACKEND_DIR} ${EXT_JSON_DIR} - ${LIBTORCH_INCLUDE_DIR} - ) - target_link_libraries(${target_name} PRIVATE - ${TORCH_CPU_LIB} - ${C10_LIB} - ${link_libraries} + ${XLA_FFI_INCLUDE_DIR} ) - - install(TARGETS ${target_name} LIBRARY DESTINATION "${OEQ_INSTALL_DIR}") + target_link_libraries(${target_name} PRIVATE ${link_libraries}) + target_link_options(${target_name} PRIVATE -Wl,--no-undefined) - # AOTI C++ library (identical except without nanobind and without INCLUDE_NB_EXTENSION) + install(TARGETS ${target_name} LIBRARY DESTINATION "${OEQ_JAX_INSTALL_DIR}") +endfunction() + +function(add_torch_extension target_name backend_define link_libraries) set(aoti_target_name ${target_name}_aoti) add_library(${aoti_target_name} SHARED ${OEQ_SOURCES}) @@ -101,12 +115,12 @@ function(add_stable_extension target_name backend_define link_libraries) CXX_STANDARD_REQUIRED ON POSITION_INDEPENDENT_CODE ON ) - - target_compile_definitions(${aoti_target_name} PRIVATE + + target_compile_definitions(${aoti_target_name} PRIVATE ${backend_define}=1 _GLIBCXX_USE_CXX11_ABI=1 ) - + target_include_directories(${aoti_target_name} PRIVATE ${EXT_DIR} ${EXT_BACKEND_DIR} @@ -118,10 +132,32 @@ function(add_stable_extension target_name backend_define link_libraries) ${C10_LIB} ${link_libraries} ) - + install(TARGETS ${aoti_target_name} LIBRARY DESTINATION "${OEQ_INSTALL_DIR}") endfunction() +set(OEQ_UTILITIES_INSTALL_DIR "openequivariance/core") + +function(add_utilities_library backend backend_define link_libraries) + set(target_name oeq_utilities_${backend}) + nanobind_add_module(${target_name} BACKEND_MODULE nanobind_backend ${EXT_DIR}/utilities.cpp) + + set_target_properties(${target_name} PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + ) + target_compile_definitions(${target_name} PRIVATE + ${backend_define}=1 + OEQ_UTILITIES_MODULE_NAME=${target_name} + ) + target_include_directories(${target_name} PRIVATE ${EXT_DIR} ${EXT_BACKEND_DIR}) + target_link_libraries(${target_name} PRIVATE ${link_libraries}) + target_link_options(${target_name} PRIVATE -Wl,--as-needed) + + install(TARGETS ${target_name} LIBRARY DESTINATION "${OEQ_UTILITIES_INSTALL_DIR}") +endfunction() + find_package(hip QUIET) find_package(hiprtc QUIET) @@ -176,7 +212,15 @@ if(OEQ_CUDA_FOUND AND NOT OEQ_CUDA_ROOTS) ${CUDA_nvrtc_LIBRARY} cuda_stub_lib ) - add_stable_extension(oeq_stable_cuda${CUDAToolkit_VERSION_MAJOR} CUDA_BACKEND "${CUDA_LINK_LIBS}") + add_torch_extension(oeq_stable_cuda${CUDAToolkit_VERSION_MAJOR} CUDA_BACKEND "${CUDA_LINK_LIBS}") + + set(CUDA_JAX_LINK_LIBS + CUDA::cudart + CUDA::cuda_driver + ${CUDA_nvrtc_LIBRARY} + ) + add_jax_extension(oeq_jax_cuda${CUDAToolkit_VERSION_MAJOR} CUDA_BACKEND "${CUDA_JAX_LINK_LIBS}") + add_utilities_library(cuda${CUDAToolkit_VERSION_MAJOR} CUDA_BACKEND "${CUDA_JAX_LINK_LIBS}") endif() foreach(cuda_root ${OEQ_CUDA_ROOTS}) @@ -199,7 +243,9 @@ foreach(cuda_root ${OEQ_CUDA_ROOTS}) ) set(CUDA_LINK_LIBS oeq_cuda${cuda_major}_toolkit cuda_stub_lib) - add_stable_extension(oeq_stable_cuda${cuda_major} CUDA_BACKEND "${CUDA_LINK_LIBS}") + add_torch_extension(oeq_stable_cuda${cuda_major} CUDA_BACKEND "${CUDA_LINK_LIBS}") + add_jax_extension(oeq_jax_cuda${cuda_major} CUDA_BACKEND oeq_cuda${cuda_major}_toolkit) + add_utilities_library(cuda${cuda_major} CUDA_BACKEND oeq_cuda${cuda_major}_toolkit) endforeach() if(hip_FOUND) @@ -222,7 +268,9 @@ if(hip_FOUND) hip::host hiprtc::hiprtc ) - add_stable_extension(oeq_stable_hip HIP_BACKEND "${HIP_LINK_LIBS}") + add_torch_extension(oeq_stable_hip HIP_BACKEND "${HIP_LINK_LIBS}") + add_jax_extension(oeq_jax_hip HIP_BACKEND "hip::host;hiprtc::hiprtc") + add_utilities_library(hip HIP_BACKEND "hip::host;hiprtc::hiprtc") endif() set(OEQ_SYCL_FOUND FALSE) @@ -251,12 +299,10 @@ if(OEQ_SYCL_FOUND) sycl_stub_lib sycl ) - add_stable_extension(oeq_stable_sycl SYCL_BACKEND "${SYCL_LINK_LIBS}") + add_torch_extension(oeq_stable_sycl SYCL_BACKEND "${SYCL_LINK_LIBS}") - foreach(tgt oeq_stable_sycl oeq_stable_sycl_aoti) - target_compile_options(${tgt} PRIVATE -fsycl) - target_link_options(${tgt} PRIVATE -fsycl) - endforeach() + target_compile_options(oeq_stable_sycl_aoti PRIVATE -fsycl) + target_link_options(oeq_stable_sycl_aoti PRIVATE -fsycl) endif() if(NOT OEQ_CUDA_FOUND AND NOT hip_FOUND AND NOT OEQ_SYCL_FOUND) diff --git a/openequivariance/openequivariance/__init__.py b/openequivariance/openequivariance/__init__.py index b8e02d74..41547f01 100644 --- a/openequivariance/openequivariance/__init__.py +++ b/openequivariance/openequivariance/__init__.py @@ -22,16 +22,50 @@ print(f"Warning: Could not determine oeq version: {e}", file=sys.stderr) -def _check_package_editable(): +def check_package_editable(): import json from importlib.metadata import Distribution - direct_url = Distribution.from_name("openequivariance").read_text("direct_url.json") + direct_url = ( + Distribution.from_name("openequivariance").read_text("direct_url.json") or "{}" + ) return json.loads(direct_url).get("dir_info", {}).get("editable", False) _editable_install_output_path = Path(__file__).parent.parent.parent / "outputs" + +def skip_rebuild_without_flock(): + import errno + import fcntl + + for finder in sys.meta_path: + build_dir = getattr(finder, "path", None) + if ( + not getattr(finder, "rebuild_flag", False) + or "openequivariance" not in getattr(finder, "known_source_files", {}) + or not build_dir + or not os.path.isdir(build_dir) + ): + continue + fd = os.open( + os.path.join(build_dir, "editable_rebuild.lock"), os.O_RDWR | os.O_CREAT + ) + try: + fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB) + fcntl.flock(fd, fcntl.LOCK_UN) + except OSError as e: + if e.errno not in (errno.EAGAIN, errno.EACCES): + skip = os.environ.get("SKBUILD_EDITABLE_SKIP", "") + os.environ["SKBUILD_EDITABLE_SKIP"] = os.pathsep.join((skip, build_dir)) + warnings.warn(f"flock unsupported in {build_dir}, skipping rebuild.") + finally: + os.close(fd) + + +if check_package_editable(): + skip_rebuild_without_flock() + if "OEQ_NOTORCH" not in os.environ or os.environ["OEQ_NOTORCH"] != "1": import torch @@ -74,22 +108,13 @@ def torch_ext_so_path(): return None -jax = None -try: - import openequivariance_extjax - import openequivariance.jax as jax - -except Exception as e: - error = e - - class JAX_ERR: - def TensorProduct(*args, **kwargs): - raise error +def __getattr__(name): + if name == "jax": + import openequivariance.jax as jax_module - def TensorProductConv(*args, **kwargs): - raise error + return jax_module + raise AttributeError(f"module 'openequivariance' has no attribute {name!r}") - jax = JAX_ERR() __all__ = [ "TPProblem", @@ -97,8 +122,7 @@ def TensorProductConv(*args, **kwargs): "TensorProduct", "TensorProductConv", "torch_to_oeq_dtype", - "_check_package_editable", + "check_package_editable", "torch_ext_so_path", - "jax", "transpose_irreps", ] diff --git a/openequivariance/openequivariance/_torch/extlib/__init__.py b/openequivariance/openequivariance/_torch/extlib/__init__.py index 43adfd64..f602d510 100644 --- a/openequivariance/openequivariance/_torch/extlib/__init__.py +++ b/openequivariance/openequivariance/_torch/extlib/__init__.py @@ -30,6 +30,7 @@ def __init__(self): self.built_extension_error = None self.ext_module_python = None self.aoti_module_path = None + self.ext_module_utilities = None class JITExtension(Extension): @@ -152,6 +153,7 @@ def __init__(self): torch.ops.load_library(self.ext_module_python.__file__) self.built_extension = True self.aoti_module_path = self.ext_module_python.__file__ + self.ext_module_utilities = self.ext_module_python except Exception as e: # If compiling torch fails (e.g. low gcc version), we should fall back to the @@ -167,31 +169,30 @@ class PrecompiledExtension(Extension): def __init__(self, backend): super().__init__() if backend == "cuda": - ext_module_py_name = f"oeq_stable_cuda{torch.version.cuda.split('.')[0]}" - else: - ext_module_py_name = f"oeq_stable_{backend}" - - full_name = f"openequivariance._torch.extlib.{ext_module_py_name}" - spec = importlib.util.find_spec(full_name) - aoti_module_path = spec and os.path.join( - os.path.dirname(spec.origin), f"lib{ext_module_py_name}_aoti.so" + backend = f"cuda{torch.version.cuda.split('.')[0]}" + + library_name = f"liboeq_stable_{backend}_aoti" + spec = importlib.util.find_spec( + f"openequivariance._torch.extlib.{library_name}" ) - if spec is None or not os.path.exists(aoti_module_path): + if spec is None: self.built_extension_error = ( - f"No precompiled OpenEquivariance extension {ext_module_py_name} " + f"No precompiled OpenEquivariance extension {library_name}.so " f"for PyTorch {torch.__version__}. {SOURCE_BUILD_HINT}" ) return try: - self.ext_module_python = importlib.import_module(full_name) - torch.ops.load_library(self.ext_module_python.__file__) + torch.ops.load_library(spec.origin) + self.ext_module_utilities = importlib.import_module( + f"openequivariance.core.oeq_utilities_{backend}" + ) self.built_extension = True - self.aoti_module_path = aoti_module_path + self.aoti_module_path = spec.origin except Exception as e: self.built_extension_error = ( f"Error loading precompiled OpenEquivariance extension " - f"{ext_module_py_name}: {e}\n{SOURCE_BUILD_HINT}" + f"{library_name}.so: {e}\n{SOURCE_BUILD_HINT}" ) @@ -224,11 +225,8 @@ def _detect_backend(): if BUILT_EXTENSION: - sys.modules["oeq_utilities"] = extension.ext_module_python - from oeq_utilities import ( - DeviceProp, - GPUTimer, - ) + DeviceProp = extension.ext_module_utilities.DeviceProp + GPUTimer = extension.ext_module_utilities.GPUTimer def torch_ext_so_path(): return extension.aoti_module_path diff --git a/openequivariance/openequivariance/benchmark/ConvBenchmarkSuite.py b/openequivariance/openequivariance/benchmark/ConvBenchmarkSuite.py index 8e2ac98a..f35ae547 100644 --- a/openequivariance/openequivariance/benchmark/ConvBenchmarkSuite.py +++ b/openequivariance/openequivariance/benchmark/ConvBenchmarkSuite.py @@ -65,7 +65,7 @@ def run( high_precision_ref=False, ): if output_folder is None: - if oeq._check_package_editable(): + if oeq.check_package_editable(): output_folder = ( oeq._editable_install_output_path / f"{self.millis_since_epoch}" ) diff --git a/openequivariance/openequivariance/benchmark/TestBenchmarkSuite.py b/openequivariance/openequivariance/benchmark/TestBenchmarkSuite.py index 72ada84d..68ad5874 100644 --- a/openequivariance/openequivariance/benchmark/TestBenchmarkSuite.py +++ b/openequivariance/openequivariance/benchmark/TestBenchmarkSuite.py @@ -112,7 +112,7 @@ def run( self.results = [] millis_since_epoch = round(time.time() * 1000) if output_folder is None: - if oeq._check_package_editable(): + if oeq.check_package_editable(): output_folder = ( oeq._editable_install_output_path / f"{millis_since_epoch}" ) diff --git a/openequivariance/openequivariance/extension/jax/ffi_handler_table.h b/openequivariance/openequivariance/extension/jax/ffi_handler_table.h new file mode 100644 index 00000000..15287ef4 --- /dev/null +++ b/openequivariance/openequivariance/extension/jax/ffi_handler_table.h @@ -0,0 +1,42 @@ +#pragma once + +#include + +#if defined(_WIN32) +#define OEQ_FFI_EXPORT __declspec(dllexport) +#elif defined(__GNUC__) || defined(__clang__) +#define OEQ_FFI_EXPORT __attribute__((visibility("default"))) +#else +#define OEQ_FFI_EXPORT +#endif + +// This is a stable, Python-free view of XLA FFI handlers. Consumers must +// reject a table with an unknown ABI version or entry layout. +#ifdef __cplusplus +extern "C" { +#endif + +#define OEQ_FFI_ABI_VERSION 2 + +#define OEQ_FFI_TRAIT_COMMAND_BUFFER_COMPATIBLE (1u << 0) + +typedef struct OeqFfiHandler { + const char* name; + void* instantiate; + void* prepare; + void* initialize; + void* execute; + uint32_t traits; +} OeqFfiHandler; + +typedef struct OeqFfiHandlerTable { + uint32_t abi_version; + uint32_t handler_count; + const OeqFfiHandler* handlers; +} OeqFfiHandlerTable; + +OEQ_FFI_EXPORT const OeqFfiHandlerTable* oeq_ffi_handler_table(); + +#ifdef __cplusplus +} +#endif diff --git a/openequivariance/openequivariance/extension/jax/ffi_handlers.cpp b/openequivariance/openequivariance/extension/jax/ffi_handlers.cpp new file mode 100644 index 00000000..fb3a702e --- /dev/null +++ b/openequivariance/openequivariance/extension/jax/ffi_handlers.cpp @@ -0,0 +1,771 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "xla/ffi/api/ffi.h" +#include "json11/json11.hpp" +#include "ffi_handler_table.h" + +namespace ffi = xla::ffi; +using json = json11::Json; + +#ifdef CUDA_BACKEND + #include + #include + + #include "backend/backend_cuda.hpp" + using JITKernel = CUJITKernel; + using GPU_Allocator = CUDA_Allocator; + using stream_t = cudaStream_t; +#endif + +#ifdef HIP_BACKEND + #include "backend/backend_hip.hpp" + using JITKernel = HIPJITKernel; + using GPU_Allocator = HIP_Allocator; + using stream_t = hipStream_t; +#endif + +#include "tensorproducts.hpp" +#include "convolution.hpp" + +xla::ffi::DataType enum_to_xla_dtype(int64_t i){ + switch(i) { + case 1: + return xla::ffi::DataType::F32; + case 2: + return xla::ffi::DataType::F64; + case 3: + return xla::ffi::DataType::S32; + case 4: + return xla::ffi::DataType::S64; + case 5: + return xla::ffi::DataType::U8; + } + throw logic_error("Unsupported tensor datatype!"); +} + +std::string xla_dtype_to_string(xla::ffi::DataType dtype) { + const std::unordered_map map = { + {xla::ffi::DataType::INVALID, "INVALID"}, + {xla::ffi::DataType::PRED, "PRED"}, + {xla::ffi::DataType::S8, "S8"}, + {xla::ffi::DataType::S16, "S16"}, + {xla::ffi::DataType::S32, "S32"}, + {xla::ffi::DataType::S64, "S64"}, + {xla::ffi::DataType::U8, "U8"}, + {xla::ffi::DataType::U16, "U16"}, + {xla::ffi::DataType::U32, "U32"}, + {xla::ffi::DataType::U64, "U64"}, + {xla::ffi::DataType::F16, "F16"}, + {xla::ffi::DataType::F32, "F32"}, + {xla::ffi::DataType::F64, "F64"}, + {xla::ffi::DataType::BF16, "BF16"}, + {xla::ffi::DataType::C64, "C64"}, + {xla::ffi::DataType::C128, "C128"}, + {xla::ffi::DataType::TOKEN, "TOKEN"}, + {xla::ffi::DataType::F8E5M2, "F8E5M2"}, + {xla::ffi::DataType::F8E4M3, "F8E4M3"}, + {xla::ffi::DataType::F8E4M3FN, "F8E4M3FN"}, + {xla::ffi::DataType::F8E4M3B11FNUZ, "F8E4M3B11FNUZ"}, + {xla::ffi::DataType::F8E5M2FNUZ, "F8E5M2FNUZ"}, + {xla::ffi::DataType::F8E4M3FNUZ, "F8E4M3FNUZ"}, + {xla::ffi::DataType::F8E3M4, "F8E3M4"}, + {xla::ffi::DataType::F4E2M1FN, "F4E2M1FN"}, + {xla::ffi::DataType::F8E8M0FNU, "F8E8M0FNU"}, + }; + return map.at(dtype); +} + +inline void* data_ptr(ffi::AnyBuffer &buffer) { + return buffer.untyped_data(); +} + +inline void* data_ptr(ffi::Result &buffer) { + return data_ptr(*buffer); +} + +inline int byte_count(ffi::AnyBuffer &buffer) { + switch (buffer.element_type()) { + case xla::ffi::DataType::U32: + case xla::ffi::DataType::S32: + case xla::ffi::DataType::F32: + return 4; + case xla::ffi::DataType::F64: + case xla::ffi::DataType::S64: + return 8; + case xla::ffi::DataType::U8: + return 1; + default: + throw logic_error("Unsupported tensor datatype!"); + } +} + +#ifdef CUDA_BACKEND +void zero_buffer(ffi::AnyBuffer &buffer, stream_t stream) { + cudaMemsetAsync( + data_ptr(buffer), + 0, + buffer.element_count() * byte_count(buffer), + stream); +} +#endif +#ifdef HIP_BACKEND +void zero_buffer(ffi::AnyBuffer &buffer, stream_t stream) { + std::ignore = hipMemsetAsync( + data_ptr(buffer), + 0, + buffer.element_count() * byte_count(buffer), + stream); +} +#endif + +std::unordered_map parse_json_config(const json &j_obj) { + std::unordered_map result; + for (const auto &kv : j_obj.object_items()) { + result[kv.first] = static_cast(kv.second.number_value()); + } + return result; +} + +struct KernelProp { + int64_t L1_dim, L2_dim, L3_dim, weight_numel; + bool shared_weights; + xla::ffi::DataType irrep_dtype; + xla::ffi::DataType weight_dtype; + + int64_t workspace_size; // Convolution only + bool deterministic; + xla::ffi::DataType idx_dtype; + xla::ffi::DataType workspace_dtype; + + KernelProp() {} + + KernelProp( + std::unordered_map &kernel_dims, bool is_convolution): + L1_dim(kernel_dims.at("L1_dim")), + L2_dim(kernel_dims.at("L2_dim")), + L3_dim(kernel_dims.at("L3_dim")), + weight_numel(kernel_dims.at("weight_numel")), + shared_weights(kernel_dims.at("shared_weights")), + irrep_dtype(enum_to_xla_dtype(kernel_dims.at("irrep_dtype"))), + weight_dtype(enum_to_xla_dtype(kernel_dims.at("weight_dtype"))), + workspace_dtype(xla::ffi::DataType::U8) { + if(is_convolution) { + workspace_size = kernel_dims.at("workspace_size"); + deterministic = kernel_dims.at("deterministic"); + idx_dtype = enum_to_xla_dtype(kernel_dims.at("idx_dtype")); + } + } +}; + +std::unordered_map>, + KernelProp + >> tp_cache; + +std::unordered_map>, + KernelProp + >> conv_cache; +std::mutex mut; + +template +typename Cache::mapped_type &find_or_compile_cached( + Cache &cache, int64_t hash, Factory &&factory) { + const std::lock_guard lock(mut); + auto it = cache.find(hash); + if (it == cache.end()) { + it = cache.emplace(hash, std::forward(factory)()).first; + } + return it->second; +} + +std::pair*, KernelProp> + compile_tp_with_caching(std::string_view json_payload, + int64_t hash, + bool is_convolution) { + + auto &cached = find_or_compile_cached( + tp_cache, hash, [&] { + std::string err; + json root = json::parse(std::string(json_payload), err); + if (!err.empty()) throw std::runtime_error("JSON Parse Error: " + err); + + std::string kernel_src = root["kernel"].string_value(); + auto forward_cfg = parse_json_config(root["forward_config"]); + auto backward_cfg = parse_json_config(root["backward_config"]); + auto dbackward_cfg = parse_json_config(root["double_backward_config"]); + auto kernel_prop_map = parse_json_config(root["kernel_prop"]); + + auto jit_tp_impl = std::make_unique>( + kernel_src, + forward_cfg, + backward_cfg, + dbackward_cfg, + kernel_prop_map); + return std::make_pair(std::move(jit_tp_impl), + KernelProp(kernel_prop_map, is_convolution)); + }); + return {cached.first.get(), cached.second}; +} + +std::pair*, KernelProp> + compile_conv_with_caching(std::string_view json_payload, + int64_t hash, + bool is_convolution) { + + auto &cached = find_or_compile_cached( + conv_cache, hash, [&] { + std::string err; + json root = json::parse(std::string(json_payload), err); + if (!err.empty()) throw std::runtime_error("JSON Parse Error: " + err); + + std::string kernel_src = root["kernel"].string_value(); + auto forward_cfg = parse_json_config(root["forward_config"]); + auto backward_cfg = parse_json_config(root["backward_config"]); + auto dbackward_cfg = parse_json_config(root["double_backward_config"]); + auto kernel_prop_map = parse_json_config(root["kernel_prop"]); + + auto jit_conv_impl = std::make_unique>( + kernel_src, + forward_cfg, + backward_cfg, + dbackward_cfg, + kernel_prop_map); + return std::make_pair(std::move(jit_conv_impl), + KernelProp(kernel_prop_map, is_convolution)); + }); + return {cached.first.get(), cached.second}; +} + +inline void check_tensor(const ffi::AnyBuffer &buffer, + std::initializer_list expected_shape, + xla::ffi::DataType expected_dtype, + std::string tensor_name) { + const ffi::AnyBuffer::Dimensions dims = buffer.dimensions(); + if (dims.size() != expected_shape.size()) { + throw std::logic_error("Rank mismatch for tensor '" + + tensor_name + + "'. Expected rank " + + std::to_string(expected_shape.size()) + + ", got rank " + + std::to_string(dims.size())); + } + + for (size_t i = 0; i < dims.size(); i++) { + if (dims[i] != expected_shape.begin()[i]) { + throw std::logic_error("Shape mismatch for tensor '" + + tensor_name + + "'. Expected dimension " + + std::to_string(expected_shape.begin()[i]) + + " at index " + + std::to_string(i) + + ", got " + + std::to_string(dims[i])); + } + } + + if (buffer.element_type() != expected_dtype) { + throw std::logic_error("Datatype mismatch for tensor " + tensor_name + + ". Expected datatype " + xla_dtype_to_string(expected_dtype) + + ", got " + xla_dtype_to_string(buffer.element_type())); + } +} + +// --------------------- Tensor Products -------------------------- +ffi::Error tp_forward_impl( + ffi::AnyBuffer L1_in, + ffi::AnyBuffer L2_in, + ffi::AnyBuffer W, + ffi::Result L3_out, + stream_t stream, + std::string_view kernel_json, + int64_t hash) { + + auto [jit_kernel, k] = compile_tp_with_caching( + kernel_json, hash, false); + const int64_t num_batch = L1_in.dimensions()[0]; + + check_tensor(L1_in, {num_batch, k.L1_dim}, k.irrep_dtype, "L1_in"); + check_tensor(L2_in, {num_batch, k.L2_dim}, k.irrep_dtype, "L2_in"); + + if (k.shared_weights) + check_tensor(W, {k.weight_numel}, k.weight_dtype, "W"); + else + check_tensor(W, {num_batch, k.weight_numel}, k.weight_dtype, "W"); + + jit_kernel->exec_tensor_product( + num_batch, + data_ptr(L1_in), + data_ptr(L2_in), + data_ptr(L3_out), + data_ptr(W), + stream); + + return ffi::Error::Success(); +} + +ffi::Error tp_backward_impl( + ffi::AnyBuffer L1_in, + ffi::AnyBuffer L2_in, + ffi::AnyBuffer W, + ffi::AnyBuffer L3_grad, + ffi::Result L1_grad, + ffi::Result L2_grad, + ffi::Result W_grad, + stream_t stream, + std::string_view kernel_json, + int64_t hash) { + + auto [jit_kernel, k] = compile_tp_with_caching( + kernel_json, hash, false); + const int64_t num_batch = L1_in.dimensions()[0]; + check_tensor(L1_in, {num_batch, k.L1_dim}, k.irrep_dtype, "L1_in"); + check_tensor(L2_in, {num_batch, k.L2_dim}, k.irrep_dtype, "L2_in"); + check_tensor(L3_grad, {num_batch, k.L3_dim}, k.irrep_dtype, "L3_grad"); + + if (k.shared_weights) { + check_tensor(W, {k.weight_numel}, k.weight_dtype, "W"); + check_tensor(*W_grad, {k.weight_numel}, k.weight_dtype, "W_grad"); + } + else { + check_tensor(W, {num_batch, k.weight_numel}, k.weight_dtype, "W"); + check_tensor(*W_grad, {num_batch, k.weight_numel}, k.weight_dtype, "W_grad"); + } + + zero_buffer(*L1_grad, stream); + zero_buffer(*L2_grad, stream); + zero_buffer(*W_grad, stream); + + jit_kernel->backward( + num_batch, + data_ptr(L1_in), + data_ptr(L1_grad), + data_ptr(L2_in), + data_ptr(L2_grad), + data_ptr(W), + data_ptr(W_grad), + data_ptr(L3_grad), + stream); + return ffi::Error::Success(); +} + + +ffi::Error tp_double_backward_impl( + ffi::AnyBuffer L1_in, + ffi::AnyBuffer L2_in, + ffi::AnyBuffer W, + ffi::AnyBuffer L3_grad, + ffi::AnyBuffer L1_dgrad, + ffi::AnyBuffer L2_dgrad, + ffi::AnyBuffer W_dgrad, + ffi::Result L1_grad, + ffi::Result L2_grad, + ffi::Result W_grad, + ffi::Result L3_dgrad, + stream_t stream, + std::string_view kernel_json, + int64_t hash) { + + auto [jit_kernel, k] = compile_tp_with_caching( + kernel_json, hash, false); + const int64_t num_batch = L1_in.dimensions()[0]; + check_tensor(L1_in, {num_batch, k.L1_dim}, k.irrep_dtype, "L1_in"); + check_tensor(L2_in, {num_batch, k.L2_dim}, k.irrep_dtype, "L2_in"); + check_tensor(L3_grad, {num_batch, k.L3_dim}, k.irrep_dtype, "L3_grad"); + check_tensor(L1_dgrad, {num_batch, k.L1_dim}, k.irrep_dtype, "L1_dgrad"); + check_tensor(L2_dgrad, {num_batch, k.L2_dim}, k.irrep_dtype, "L2_dgrad"); + + if (k.shared_weights){ + check_tensor(W, {k.weight_numel}, k.weight_dtype, "W"); + check_tensor(W_dgrad, {k.weight_numel}, k.weight_dtype, "W_dgrad"); + } else { + check_tensor(W, {num_batch, k.weight_numel}, k.weight_dtype, "W"); + check_tensor(W_dgrad, {num_batch, k.weight_numel}, k.weight_dtype, "W_dgrad"); + } + + zero_buffer(*L1_grad, stream); + zero_buffer(*L2_grad, stream); + zero_buffer(*W_grad, stream); + zero_buffer(*L3_dgrad, stream); + + jit_kernel->double_backward( + num_batch, + data_ptr(L1_in), + data_ptr(L2_in), + data_ptr(W), + data_ptr(L3_grad), + data_ptr(L1_dgrad), + data_ptr(L2_dgrad), + data_ptr(W_dgrad), + data_ptr(L1_grad), + data_ptr(L2_grad), + data_ptr(W_grad), + data_ptr(L3_dgrad), + stream); + return ffi::Error::Success(); +} + +XLA_FFI_DEFINE_HANDLER_SYMBOL( + tp_forward, tp_forward_impl, + ffi::Ffi::Bind() + .Arg() + .Arg() + .Arg() + .Ret() + .Ctx>() + .Attr("kernel") + .Attr("hash"), + {xla::ffi::Traits::kCmdBufferCompatible}); // cudaGraph enabled + +XLA_FFI_DEFINE_HANDLER_SYMBOL( + tp_backward, tp_backward_impl, + ffi::Ffi::Bind() + .Arg() + .Arg() + .Arg() + .Arg() + .Ret() + .Ret() + .Ret() + .Ctx>() + .Attr("kernel") + .Attr("hash"), + {xla::ffi::Traits::kCmdBufferCompatible}); + +XLA_FFI_DEFINE_HANDLER_SYMBOL( + tp_double_backward, tp_double_backward_impl, + ffi::Ffi::Bind() + .Arg() + .Arg() + .Arg() + .Arg() + .Arg() + .Arg() + .Arg() + .Ret() + .Ret() + .Ret() + .Ret() + .Ctx>() + .Attr("kernel") + .Attr("hash"), + {xla::ffi::Traits::kCmdBufferCompatible}); + +// --------------------- Convolution -------------------------- +ffi::Error conv_forward_impl( + ffi::AnyBuffer L1_in, + ffi::AnyBuffer L2_in, + ffi::AnyBuffer W, + ffi::AnyBuffer rows, + ffi::AnyBuffer cols, + ffi::AnyBuffer workspace, + ffi::AnyBuffer transpose_perm, + ffi::Result L3_out, + stream_t stream, + std::string_view kernel_json, + int64_t hash) { + + auto [jit_kernel, k] = compile_conv_with_caching( + kernel_json, hash, true); + const int64_t nnz = rows.dimensions()[0]; + const int64_t node_count = L1_in.dimensions()[0]; + void* workspace_ptr = data_ptr(workspace); + + check_tensor(L1_in, {node_count, k.L1_dim}, k.irrep_dtype, "L1_in"); + check_tensor(L2_in, {nnz, k.L2_dim}, k.irrep_dtype, "L2_in"); + check_tensor(workspace, {k.workspace_size}, k.workspace_dtype, "workspace"); + check_tensor(rows, {nnz}, k.idx_dtype, "rows"); + check_tensor(cols, {nnz}, k.idx_dtype, "cols"); + + if (k.deterministic){ + check_tensor(transpose_perm, {nnz}, k.idx_dtype, "transpose perm"); + } + else { + workspace_ptr = nullptr; + } + zero_buffer(*L3_out, stream); + + if (k.shared_weights) + check_tensor(W, {k.weight_numel}, k.weight_dtype, "W"); + else + check_tensor(W, {nnz, k.weight_numel}, k.weight_dtype, "W"); + + jit_kernel->exec_conv( + data_ptr(L1_in), + data_ptr(L2_in), + data_ptr(W), + data_ptr(L3_out), + data_ptr(rows), + data_ptr(cols), + nnz, node_count, + workspace_ptr, + stream); + + return ffi::Error::Success(); +} + +ffi::Error conv_backward_impl( + ffi::AnyBuffer L1_in, + ffi::AnyBuffer L2_in, + ffi::AnyBuffer W, + ffi::AnyBuffer L3_grad, + ffi::Result L1_grad, + ffi::Result L2_grad, + ffi::Result W_grad, + ffi::AnyBuffer rows, + ffi::AnyBuffer cols, + ffi::AnyBuffer workspace, + ffi::AnyBuffer transpose_perm, + stream_t stream, + std::string_view kernel_json, + int64_t hash) { + + auto [jit_kernel, k] = compile_conv_with_caching( + kernel_json, hash, true); + const int64_t nnz = rows.dimensions()[0]; + const int64_t node_count = L1_in.dimensions()[0]; + void* workspace_ptr = data_ptr(workspace); + + check_tensor(L1_in, {node_count, k.L1_dim}, k.irrep_dtype, "L1_in"); + check_tensor(L2_in, {nnz, k.L2_dim}, k.irrep_dtype, "L2_in"); + check_tensor(L3_grad, {node_count, k.L3_dim}, k.irrep_dtype, "L3_grad"); + check_tensor(workspace, {k.workspace_size}, k.workspace_dtype, "workspace"); + check_tensor(rows, {nnz}, k.idx_dtype, "rows"); + check_tensor(cols, {nnz}, k.idx_dtype, "cols"); + + if (k.deterministic) { + check_tensor(transpose_perm, {nnz}, k.idx_dtype, "transpose perm"); + } + else { + workspace_ptr = nullptr; + } + zero_buffer(*L1_grad, stream); + zero_buffer(*L2_grad, stream); + zero_buffer(*W_grad, stream); + + if (k.shared_weights) { + check_tensor(W, {k.weight_numel}, k.weight_dtype, "W"); + check_tensor(*W_grad, {k.weight_numel}, k.weight_dtype, "W_grad"); + } + else { + check_tensor(W, {nnz, k.weight_numel}, k.weight_dtype, "W"); + check_tensor(*W_grad, {nnz, k.weight_numel}, k.weight_dtype, "W_grad"); + } + + jit_kernel->backward( + data_ptr(L1_in), + data_ptr(L1_grad), + data_ptr(L2_in), + data_ptr(L2_grad), + data_ptr(W), + data_ptr(W_grad), + data_ptr(L3_grad), + data_ptr(rows), + data_ptr(cols), + nnz, node_count, + workspace_ptr, + data_ptr(transpose_perm), + stream); + return ffi::Error::Success(); +} + +ffi::Error conv_double_backward_impl( + ffi::AnyBuffer L1_in, + ffi::AnyBuffer L2_in, + ffi::AnyBuffer W, + ffi::AnyBuffer L3_grad, + ffi::AnyBuffer L1_dgrad, + ffi::AnyBuffer L2_dgrad, + ffi::AnyBuffer W_dgrad, + ffi::Result L1_grad, + ffi::Result L2_grad, + ffi::Result W_grad, + ffi::Result L3_dgrad, + ffi::AnyBuffer rows, + ffi::AnyBuffer cols, + ffi::AnyBuffer workspace, + ffi::AnyBuffer transpose_perm, + stream_t stream, + std::string_view kernel_json, + int64_t hash) { + + auto [jit_kernel, k] = compile_conv_with_caching( + kernel_json, hash, true); + const int64_t nnz = rows.dimensions()[0]; + const int64_t node_count = L1_in.dimensions()[0]; + void* workspace_ptr = data_ptr(workspace); + + check_tensor(L1_in, {node_count, k.L1_dim}, k.irrep_dtype, "L1_in"); + check_tensor(L2_in, {nnz, k.L2_dim}, k.irrep_dtype, "L2_in"); + check_tensor(L3_grad, {node_count, k.L3_dim}, k.irrep_dtype, "L3_grad"); + check_tensor(L1_dgrad, {node_count, k.L1_dim}, k.irrep_dtype, "L1_dgrad"); + check_tensor(L2_dgrad, {nnz, k.L2_dim}, k.irrep_dtype, "L2_dgrad"); + check_tensor(workspace, {k.workspace_size}, k.workspace_dtype, "workspace"); + check_tensor(rows, {nnz}, k.idx_dtype, "rows"); + check_tensor(cols, {nnz}, k.idx_dtype, "cols"); + + if (k.deterministic) { + check_tensor(transpose_perm, {nnz}, k.idx_dtype, "transpose perm"); + } + else { + workspace_ptr = nullptr; + } + zero_buffer(*L1_grad, stream); + zero_buffer(*L2_grad, stream); + zero_buffer(*W_grad, stream); + zero_buffer(*L3_dgrad, stream); + + if (k.shared_weights) { + check_tensor(W, {k.weight_numel}, k.weight_dtype, "W"); + check_tensor(W_dgrad, {k.weight_numel}, k.weight_dtype, "W_dgrad"); + } else { + check_tensor(W, {nnz, k.weight_numel}, k.weight_dtype, "W"); + check_tensor(W_dgrad, {nnz, k.weight_numel}, k.weight_dtype, "W_dgrad"); + } + + jit_kernel->double_backward( + data_ptr(L1_in), + data_ptr(L2_in), + data_ptr(W), + data_ptr(L3_grad), + data_ptr(L1_dgrad), + data_ptr(L2_dgrad), + data_ptr(W_dgrad), + data_ptr(L1_grad), + data_ptr(L2_grad), + data_ptr(W_grad), + data_ptr(L3_dgrad), + data_ptr(rows), + data_ptr(cols), + nnz, node_count, + workspace_ptr, + data_ptr(transpose_perm), + stream); + return ffi::Error::Success(); +} + +// --------------------- FFI Bindings -------------------------- + +ffi::Error tp_initialize_impl(ffi::RemainingArgs, ffi::RemainingRets, stream_t, + std::string_view kernel_json, int64_t hash) { + compile_tp_with_caching(kernel_json, hash, false); + return ffi::Error::Success(); +} + +ffi::Error conv_initialize_impl(ffi::RemainingArgs, ffi::RemainingRets, stream_t, + std::string_view kernel_json, int64_t hash) { + compile_conv_with_caching(kernel_json, hash, true); + return ffi::Error::Success(); +} + +#define OEQ_STOCK_INITIALIZE_ATTRIBUTES \ + .RemainingArgs() \ + .RemainingRets() \ + .Ctx>() \ + .Attr("kernel") \ + .Attr("hash") + +XLA_FFI_DEFINE_HANDLER_SYMBOL( + tp_initialize, tp_initialize_impl, + ffi::Ffi::Bind() OEQ_STOCK_INITIALIZE_ATTRIBUTES); + +XLA_FFI_DEFINE_HANDLER_SYMBOL( + conv_initialize, conv_initialize_impl, + ffi::Ffi::Bind() OEQ_STOCK_INITIALIZE_ATTRIBUTES); + +#undef OEQ_STOCK_INITIALIZE_ATTRIBUTES + +XLA_FFI_DEFINE_HANDLER_SYMBOL( + conv_forward, conv_forward_impl, + ffi::Ffi::Bind() + .Arg() + .Arg() + .Arg() + .Arg() + .Arg() + .Arg() + .Arg() + .Ret() + .Ctx>() + .Attr("kernel") + .Attr("hash"), + {xla::ffi::Traits::kCmdBufferCompatible}); + +XLA_FFI_DEFINE_HANDLER_SYMBOL( + conv_backward, conv_backward_impl, + ffi::Ffi::Bind() + .Arg() + .Arg() + .Arg() + .Arg() + .Ret() + .Ret() + .Ret() + .Arg() + .Arg() + .Arg() + .Arg() + .Ctx>() + .Attr("kernel") + .Attr("hash"), + {xla::ffi::Traits::kCmdBufferCompatible}); + +XLA_FFI_DEFINE_HANDLER_SYMBOL( + conv_double_backward, conv_double_backward_impl, + ffi::Ffi::Bind() + .Arg() + .Arg() + .Arg() + .Arg() + .Arg() + .Arg() + .Arg() + .Ret() + .Ret() + .Ret() + .Ret() + .Arg() + .Arg() + .Arg() + .Arg() + .Ctx>() + .Attr("kernel") + .Attr("hash"), + {xla::ffi::Traits::kCmdBufferCompatible}); + +namespace { + +#define OEQ_FFI_HANDLER(NAME, INITIALIZE) \ + {#NAME, nullptr, nullptr, reinterpret_cast(INITIALIZE), \ + reinterpret_cast(NAME), OEQ_FFI_TRAIT_COMMAND_BUFFER_COMPATIBLE} + +const OeqFfiHandler kFfiHandlers[] = { + OEQ_FFI_HANDLER(tp_forward, tp_initialize), + OEQ_FFI_HANDLER(tp_backward, tp_initialize), + OEQ_FFI_HANDLER(tp_double_backward, tp_initialize), + OEQ_FFI_HANDLER(conv_forward, conv_initialize), + OEQ_FFI_HANDLER(conv_backward, conv_initialize), + OEQ_FFI_HANDLER(conv_double_backward, conv_initialize), +}; + +#undef OEQ_FFI_HANDLER + +const OeqFfiHandlerTable kFfiHandlerTable = { + OEQ_FFI_ABI_VERSION, + sizeof(kFfiHandlers) / sizeof(kFfiHandlers[0]), + kFfiHandlers, +}; + +} // namespace + +extern "C" const OeqFfiHandlerTable* oeq_ffi_handler_table() { + return &kFfiHandlerTable; +} diff --git a/openequivariance/openequivariance/extension/jax/ffi_provider.cpp b/openequivariance/openequivariance/extension/jax/ffi_provider.cpp new file mode 100644 index 00000000..52f0addb --- /dev/null +++ b/openequivariance/openequivariance/extension/jax/ffi_provider.cpp @@ -0,0 +1,71 @@ +#include +#include +#include + +#include "ffi_handler_table.h" +#include "xla/ffi/api/c_api.h" + +extern "C" OEQ_FFI_EXPORT int RegisterFFi( + const XLA_FFI_Api* api, + const char* platform_name) { + constexpr std::size_t kHandlerRegistrationSize = + offsetof(XLA_FFI_Api, XLA_FFI_Handler_Register) + + sizeof(api->XLA_FFI_Handler_Register); + if (api == nullptr || platform_name == nullptr || + api->struct_size < kHandlerRegistrationSize || + api->XLA_FFI_Handler_Register == nullptr || + api->XLA_FFI_Error_GetMessage == nullptr || + api->XLA_FFI_Error_Destroy == nullptr) { + return 1; + } + + const OeqFfiHandlerTable* handlers = oeq_ffi_handler_table(); + if (handlers == nullptr || handlers->abi_version != OEQ_FFI_ABI_VERSION) { + return 1; + } + for (uint32_t index = 0; index < handlers->handler_count; ++index) { + const OeqFfiHandler& handler = handlers->handlers[index]; + if (handler.execute == nullptr) { + std::cerr << "OpenEquivariance FFI target " << handler.name + << " has no execute handler" << std::endl; + return 1; + } + + XLA_FFI_Handler_Register_Args args{}; + args.struct_size = XLA_FFI_Handler_Register_Args_STRUCT_SIZE; + args.name = {handler.name, std::strlen(handler.name)}; + args.platform = {platform_name, std::strlen(platform_name)}; + args.bundle = { + reinterpret_cast(handler.instantiate), + reinterpret_cast(handler.prepare), + reinterpret_cast(handler.initialize), + reinterpret_cast(handler.execute), + }; + args.traits = static_cast(handler.traits); + + XLA_FFI_Error* error = api->XLA_FFI_Handler_Register(&args); + if (error == nullptr) { + std::cout << "[OpenEquivariance] Registered FFI target " + << handler.name << " for " << platform_name << std::endl; + continue; + } + + XLA_FFI_Error_GetMessage_Args message_args{}; + message_args.struct_size = XLA_FFI_Error_GetMessage_Args_STRUCT_SIZE; + message_args.error = error; + api->XLA_FFI_Error_GetMessage(&message_args); + std::cerr << "OpenEquivariance failed to register FFI target " + << handler.name << ": " + << (message_args.message == nullptr + ? "unknown XLA FFI error" + : message_args.message) + << std::endl; + + XLA_FFI_Error_Destroy_Args destroy_args{}; + destroy_args.struct_size = XLA_FFI_Error_Destroy_Args_STRUCT_SIZE; + destroy_args.error = error; + api->XLA_FFI_Error_Destroy(&destroy_args); + return 1; + } + return 0; +} diff --git a/openequivariance/openequivariance/extension/libtorch_tp_jit_stable.cpp b/openequivariance/openequivariance/extension/libtorch_tp_jit_stable.cpp index 7e99c0ce..2823c74d 100644 --- a/openequivariance/openequivariance/extension/libtorch_tp_jit_stable.cpp +++ b/openequivariance/openequivariance/extension/libtorch_tp_jit_stable.cpp @@ -139,23 +139,3 @@ bool tensor_is_on_gpu(const Tensor &tensor) { } } // namespace oeq - -// CMake passes EXTENSION_NAME to match the target name (e.g. oeq_stable_cuda13) -#ifndef EXTENSION_NAME - #ifdef CUDA_BACKEND - #define EXTENSION_NAME oeq_stable_cuda - #endif - #ifdef HIP_BACKEND - #define EXTENSION_NAME oeq_stable_hip - #endif - #ifdef SYCL_BACKEND - #define EXTENSION_NAME oeq_stable_sycl - #endif -#endif - -#ifdef INCLUDE_NB_EXTENSION - #include "nb_utilities.hpp" - NB_MODULE(EXTENSION_NAME, m) { - oeq::bind_utilities(m); - } -#endif diff --git a/openequivariance/openequivariance/extension/nb_utilities.hpp b/openequivariance/openequivariance/extension/utilities.cpp similarity index 83% rename from openequivariance/openequivariance/extension/nb_utilities.hpp rename to openequivariance/openequivariance/extension/utilities.cpp index e5c3d5cf..94e97b64 100644 --- a/openequivariance/openequivariance/extension/nb_utilities.hpp +++ b/openequivariance/openequivariance/extension/utilities.cpp @@ -1,13 +1,17 @@ -#pragma once - #include "nanobind/nanobind.h" #include "nanobind/stl/string.h" -namespace oeq { +#ifdef CUDA_BACKEND +#include "backend/backend_cuda.hpp" +#endif + +#ifdef HIP_BACKEND +#include "backend/backend_hip.hpp" +#endif -inline void bind_utilities(nanobind::module_ &m) { - namespace nb = nanobind; +namespace nb = nanobind; +NB_MODULE(OEQ_UTILITIES_MODULE_NAME, m) { if (nb::handle existing = nb::type()) { m.attr("DeviceProp") = existing; } else { @@ -31,5 +35,3 @@ inline void bind_utilities(nanobind::module_ &m) { .def("clear_L2_cache", &GPUTimer::clear_L2_cache); } } - -} // namespace oeq diff --git a/openequivariance/openequivariance/jax/extlib/__init__.py b/openequivariance/openequivariance/jax/extlib/__init__.py index 74e127ef..b5630f90 100644 --- a/openequivariance/openequivariance/jax/extlib/__init__.py +++ b/openequivariance/openequivariance/jax/extlib/__init__.py @@ -1,21 +1,89 @@ +import ctypes +import importlib +import importlib.util +import re + import jax -import openequivariance_extjax as oeq_extjax +import jax.extend -IS_HIP = oeq_extjax.is_hip() +FFI_ABI_VERSION = 2 +FFI_STAGES = ("instantiate", "prepare", "initialize", "execute") -DEVICE_TYPE = "cuda" -platform = "CUDA" -if IS_HIP: - platform = "ROCM" +class Handler(ctypes.Structure): + _fields_ = [ + ("name", ctypes.c_char_p), + ("instantiate", ctypes.c_void_p), + ("prepare", ctypes.c_void_p), + ("initialize", ctypes.c_void_p), + ("execute", ctypes.c_void_p), + ("traits", ctypes.c_uint32), + ] + + +class HandlerTable(ctypes.Structure): + _fields_ = [ + ("abi_version", ctypes.c_uint32), + ("handler_count", ctypes.c_uint32), + ("handlers", ctypes.POINTER(Handler)), + ] + + +def jax_backend(): + platform_version = jax.extend.backend.get_backend().platform_version + match = re.search(r"(cuda|rocm)\s+(\d+)", platform_version) + assert match is not None + platform, version = match.groups() + if platform == "rocm": + return "hip" + return f"cuda{int(version) // 1000}" + + +def load_library(): + backend = jax_backend() + spec = importlib.util.find_spec(f"{__name__}.liboeq_jax_{backend}") + if spec is None: + raise ImportError( + f"No OpenEquivariance JAX extension liboeq_jax_{backend}.so for " + f"this JAX installation." + ) + return backend, spec.origin, ctypes.CDLL(spec.origin) + + +def registrations(library): + table_fn = library.oeq_ffi_handler_table + table_fn.argtypes = [] + table_fn.restype = ctypes.POINTER(HandlerTable) + table = table_fn().contents + assert table.abi_version == FFI_ABI_VERSION + + targets = {} + for index in range(table.handler_count): + handler = table.handlers[index] + targets[handler.name.decode()] = { + stage: jax.ffi.pycapsule(getattr(handler, stage)) + for stage in FFI_STAGES + if getattr(handler, stage) + } + return targets + + +BACKEND, LIBRARY_PATH, library = load_library() +IS_HIP = BACKEND == "hip" +DEVICE_TYPE = "cuda" -for name, target in oeq_extjax.registrations().items(): +platform = "ROCM" if IS_HIP else "CUDA" +for name, target in registrations(library).items(): jax.ffi.register_ffi_target(name, target, platform=platform, api_version=1) -GPUTimer = oeq_extjax.GPUTimer -DeviceProp = oeq_extjax.DeviceProp +ext_module_utilities = importlib.import_module( + f"openequivariance.core.oeq_utilities_{BACKEND}" +) +DeviceProp = ext_module_utilities.DeviceProp +GPUTimer = ext_module_utilities.GPUTimer __all__ = [ + "BACKEND", "GPUTimer", "DeviceProp", "DEVICE_TYPE", diff --git a/openequivariance/pyproject.toml b/openequivariance/pyproject.toml index 4fee2c0a..60f3285e 100644 --- a/openequivariance/pyproject.toml +++ b/openequivariance/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools", "setuptools-scm", "scikit-build-core", "nanobind>=3.0"] +requires = ["setuptools", "setuptools-scm", "scikit-build-core", "nanobind>=3.0", "ninja"] build-backend = "scikit_build_core.build" [project] @@ -62,12 +62,6 @@ sycl = [ "ninja" ] -jax = [ - "scikit-build-core", - "setuptools-scm", - "nanobind" -] - [tool.setuptools.packages.find] include = ["openequivariance*"] @@ -82,7 +76,7 @@ lint.ignore = ["E741"] [tool.scikit-build] wheel.py-api = "cp311" build-dir = "build/{wheel_tag}" -editable.rebuild = true +editable.rebuild = false [tool.scikit-build.cmake.define] AARCH64 = {env="AARCH64", default="0"} @@ -90,6 +84,10 @@ LIBTORCH_URL_X86_64 = "https://download.pytorch.org/libtorch/cpu/libtorch-shared LIBTORCH_SHA256_X86_64 = "c5bf8efda9224a2d971b19d1ef6cf3ba6fee8ab53e69c49427db003d1d300496" LIBTORCH_URL_AARCH64 = "https://github.com/PASSIONLab/libtorch_aarch64/raw/742ffd07f47a31a94f6b6aae2cbe129e6b667d94/pkg-libtorch.tar.zst" LIBTORCH_SHA256_AARCH64 = "9fdebdb56766317c71eb4c9f1b750baef4ebc4e6ba85b427f701fdc20844f909" +XLA_FFI_URL = "https://raw.githubusercontent.com/openxla/xla/a7e58876b73dfa9bb6bc785f6981a332c133bb55/xla/ffi/api" +XLA_FFI_C_API_SHA256 = "aca130ddbe816672f9d302364d28813b1922c90baf7bc4818f3169d375c5ecc3" +XLA_FFI_API_SHA256 = "f7080e6a7d2578961c100c5042d1db8f85c57e1595ceb7a11f55422db808c62c" +XLA_FFI_FFI_SHA256 = "fe6b5bfbaa3f13e72f9bc46f5e5fc62ef782e3dd4a41bb08867904602ed615e7" [tool.cibuildwheel] build = "cp311-*" diff --git a/openequivariance_extjax/CMakeLists.txt b/openequivariance_extjax/CMakeLists.txt deleted file mode 100644 index 52b38c5a..00000000 --- a/openequivariance_extjax/CMakeLists.txt +++ /dev/null @@ -1,100 +0,0 @@ -cmake_minimum_required(VERSION 3.15...3.30) -project(${SKBUILD_PROJECT_NAME} LANGUAGES CXX) - -find_package(Python 3.10 REQUIRED COMPONENTS Interpreter Development.Module) - -# --- XLA CONFIGURATION --- -if(XLA_DIRECT_DOWNLOAD) - message(STATUS "XLA_DIRECT_DOWNLOAD is ON. Fetching XLA source...") - include(ExternalProject) - ExternalProject_Add( - xla - PREFIX ${CMAKE_BINARY_DIR}/xla - GIT_REPOSITORY https://github.com/openxla/xla.git - GIT_TAG main - GIT_SHALLOW TRUE - GIT_PROGRESS TRUE - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - LOG_DOWNLOAD ON - ) - ExternalProject_Get_Property(xla source_dir) - set(XLA_DIR ${source_dir}) -else() - message(STATUS "XLA_DIRECT_DOWNLOAD is OFF. Locating XLA via installed JAX...") - execute_process( - COMMAND "${Python_EXECUTABLE}" "-c" - "from jax import ffi; print(ffi.include_dir())" - OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE XLA_DIR - ) -endif() - -message(STATUS "XLA include directory: ${XLA_DIR}") -# ------------------------- - -execute_process( - COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir - OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_ROOT -) -message(STATUS "nanobind cmake directory: ${nanobind_ROOT}") - -find_package(nanobind CONFIG REQUIRED) - -if(JAX_HIP) - message(STATUS "JAX_HIP is set. Building with HIP backend.") - find_package(hip REQUIRED) -else() - message(STATUS "JAX_HIP is not set (or zero). Building with CUDA backend.") - find_package(CUDAToolkit REQUIRED) -endif() - -set(HEADER_DIR "src/extension") -set(OEQ_JAX_SOURCES - src/ffi_handlers.cpp - src/jax_ffi_registration.cpp - ${HEADER_DIR}/json11/json11.cpp -) - -set(OEQ_JAX_HEADERS - src/ffi_handler_table.h - ${HEADER_DIR}/convolution.hpp - ${HEADER_DIR}/tensorproducts.hpp - ${HEADER_DIR}/backend/backend_cuda.hpp - ${HEADER_DIR}/backend/backend_hip.hpp - ${HEADER_DIR}/json11/json11.hpp -) - -nanobind_add_module(openequivariance_extjax NB_STATIC ${OEQ_JAX_SOURCES} ${OEQ_JAX_HEADERS}) -target_include_directories(openequivariance_extjax PUBLIC ${XLA_DIR} ${HEADER_DIR}) -set_target_properties(openequivariance_extjax PROPERTIES POSITION_INDEPENDENT_CODE ON) -target_compile_options(openequivariance_extjax PRIVATE -Wno-attributes -Wno-return-type) - -# Ensure the module waits for XLA download if we are in direct download mode -if(XLA_DIRECT_DOWNLOAD) - add_dependencies(openequivariance_extjax xla) -endif() - -if(JAX_HIP) - target_link_libraries(openequivariance_extjax PRIVATE hiprtc) - target_compile_definitions(openequivariance_extjax PRIVATE HIP_BACKEND=1) - -else() - set_target_properties(openequivariance_extjax PROPERTIES CUDA_STANDARD 17) - - get_target_property(CUDA_LIB_DIR CUDA::nvrtc IMPORTED_LOCATION) - get_filename_component(CUDA_LIB_DIR ${CUDA_LIB_DIR} DIRECTORY) - - set_target_properties(openequivariance_extjax PROPERTIES - BUILD_RPATH "${CUDA_LIB_DIR}" - INSTALL_RPATH "${CUDA_LIB_DIR}" - ) - - target_link_libraries(openequivariance_extjax PRIVATE - CUDA::cudart - CUDA::cuda_driver - CUDA::nvrtc) - target_compile_definitions(openequivariance_extjax PRIVATE CUDA_BACKEND=1) -endif() - -install(TARGETS openequivariance_extjax LIBRARY DESTINATION .) diff --git a/openequivariance_extjax/LICENSE b/openequivariance_extjax/LICENSE deleted file mode 100644 index 2b293818..00000000 --- a/openequivariance_extjax/LICENSE +++ /dev/null @@ -1,28 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2025, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/openequivariance_extjax/MANIFEST.in b/openequivariance_extjax/MANIFEST.in deleted file mode 100644 index fcd76e59..00000000 --- a/openequivariance_extjax/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include CMakeLists.txt -include src/* \ No newline at end of file diff --git a/openequivariance_extjax/README.md b/openequivariance_extjax/README.md deleted file mode 100644 index a7145c2f..00000000 --- a/openequivariance_extjax/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# OpenEquivariance JAX Extension - -The JAX extension module for OpenEquivariance. - -## XLA FFI provider - -The `libjcn_ffi_openequivariance.so` Bazel target builds a Python-independent -FFI provider against the XLA and CUDA repositories selected by the caller's -Bazel configuration: - -```text -bazel build @openequivariance_src//openequivariance_extjax:libjcn_ffi_openequivariance.so -``` - -The handler implementation owns one ABI-versioned table with slots for every -XLA stage (`instantiate`, `prepare`, `initialize`, and `execute`) plus each -target's traits. The nanobind extension reads that table and publishes only -stages that have a handler. The shared library exports `RegisterFFi(const -XLA_FFI_Api*, const char*)`; a runtime loader obtains the API from its PJRT -plugin and calls this entry point. The provider uses -`XLA_FFI_Handler_Register` to register every target as a complete handler -bundle, passing every non-null stage and the traits from the table. JAX -registration uses the same table. diff --git a/openequivariance_extjax/pyproject.toml b/openequivariance_extjax/pyproject.toml deleted file mode 100644 index ec186955..00000000 --- a/openequivariance_extjax/pyproject.toml +++ /dev/null @@ -1,51 +0,0 @@ -[build-system] -requires = [ - "setuptools-scm", - "scikit-build-core", - "nanobind" -] -build-backend = "scikit_build_core.build" - -[project] -name = "openequivariance_extjax" -version = "0.0.1" - -authors = [ - { name="Austin Glover" }, - { name="Vivek Bharadwaj" }, - { name="Aydin Buluc" }, - { name="James Demmel" } -] -description = "JAX C++ Extension for OpenEquivariance" -requires-python = ">=3.10" - -dependencies = [] -readme = "README.md" - -license = "BSD-3-Clause" -license-files = ["LICENSE"] - -classifiers = [ - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", -] - -[project.urls] -homepage = "https://passionlab.github.io/OpenEquivariance/" -source = "https://github.com/PASSIONLab/OpenEquivariance" -issues = "https://github.com/PASSIONLab/OpenEquivariance/issues" - -[tool.scikit-build.cmake.define] -JAX_HIP = {env="JAX_HIP", default="0"} -XLA_DIRECT_DOWNLOAD = {env="XLA_DIRECT_DOWNLOAD", default="0"} - -[tool.pytest.ini_options] -addopts = [ - "--import-mode=importlib", -] - -[tool.ruff] -lint.ignore = ["E741"] \ No newline at end of file diff --git a/openequivariance_extjax/src/ffi_handler_table.h b/openequivariance_extjax/src/ffi_handler_table.h deleted file mode 100644 index 15287ef4..00000000 --- a/openequivariance_extjax/src/ffi_handler_table.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include - -#if defined(_WIN32) -#define OEQ_FFI_EXPORT __declspec(dllexport) -#elif defined(__GNUC__) || defined(__clang__) -#define OEQ_FFI_EXPORT __attribute__((visibility("default"))) -#else -#define OEQ_FFI_EXPORT -#endif - -// This is a stable, Python-free view of XLA FFI handlers. Consumers must -// reject a table with an unknown ABI version or entry layout. -#ifdef __cplusplus -extern "C" { -#endif - -#define OEQ_FFI_ABI_VERSION 2 - -#define OEQ_FFI_TRAIT_COMMAND_BUFFER_COMPATIBLE (1u << 0) - -typedef struct OeqFfiHandler { - const char* name; - void* instantiate; - void* prepare; - void* initialize; - void* execute; - uint32_t traits; -} OeqFfiHandler; - -typedef struct OeqFfiHandlerTable { - uint32_t abi_version; - uint32_t handler_count; - const OeqFfiHandler* handlers; -} OeqFfiHandlerTable; - -OEQ_FFI_EXPORT const OeqFfiHandlerTable* oeq_ffi_handler_table(); - -#ifdef __cplusplus -} -#endif diff --git a/openequivariance_extjax/src/ffi_handler_table.h b/openequivariance_extjax/src/ffi_handler_table.h new file mode 120000 index 00000000..c97be5e7 --- /dev/null +++ b/openequivariance_extjax/src/ffi_handler_table.h @@ -0,0 +1 @@ +../../openequivariance/openequivariance/extension/jax/ffi_handler_table.h \ No newline at end of file diff --git a/openequivariance_extjax/src/ffi_handlers.cpp b/openequivariance_extjax/src/ffi_handlers.cpp deleted file mode 100644 index fb3a702e..00000000 --- a/openequivariance_extjax/src/ffi_handlers.cpp +++ /dev/null @@ -1,771 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "xla/ffi/api/ffi.h" -#include "json11/json11.hpp" -#include "ffi_handler_table.h" - -namespace ffi = xla::ffi; -using json = json11::Json; - -#ifdef CUDA_BACKEND - #include - #include - - #include "backend/backend_cuda.hpp" - using JITKernel = CUJITKernel; - using GPU_Allocator = CUDA_Allocator; - using stream_t = cudaStream_t; -#endif - -#ifdef HIP_BACKEND - #include "backend/backend_hip.hpp" - using JITKernel = HIPJITKernel; - using GPU_Allocator = HIP_Allocator; - using stream_t = hipStream_t; -#endif - -#include "tensorproducts.hpp" -#include "convolution.hpp" - -xla::ffi::DataType enum_to_xla_dtype(int64_t i){ - switch(i) { - case 1: - return xla::ffi::DataType::F32; - case 2: - return xla::ffi::DataType::F64; - case 3: - return xla::ffi::DataType::S32; - case 4: - return xla::ffi::DataType::S64; - case 5: - return xla::ffi::DataType::U8; - } - throw logic_error("Unsupported tensor datatype!"); -} - -std::string xla_dtype_to_string(xla::ffi::DataType dtype) { - const std::unordered_map map = { - {xla::ffi::DataType::INVALID, "INVALID"}, - {xla::ffi::DataType::PRED, "PRED"}, - {xla::ffi::DataType::S8, "S8"}, - {xla::ffi::DataType::S16, "S16"}, - {xla::ffi::DataType::S32, "S32"}, - {xla::ffi::DataType::S64, "S64"}, - {xla::ffi::DataType::U8, "U8"}, - {xla::ffi::DataType::U16, "U16"}, - {xla::ffi::DataType::U32, "U32"}, - {xla::ffi::DataType::U64, "U64"}, - {xla::ffi::DataType::F16, "F16"}, - {xla::ffi::DataType::F32, "F32"}, - {xla::ffi::DataType::F64, "F64"}, - {xla::ffi::DataType::BF16, "BF16"}, - {xla::ffi::DataType::C64, "C64"}, - {xla::ffi::DataType::C128, "C128"}, - {xla::ffi::DataType::TOKEN, "TOKEN"}, - {xla::ffi::DataType::F8E5M2, "F8E5M2"}, - {xla::ffi::DataType::F8E4M3, "F8E4M3"}, - {xla::ffi::DataType::F8E4M3FN, "F8E4M3FN"}, - {xla::ffi::DataType::F8E4M3B11FNUZ, "F8E4M3B11FNUZ"}, - {xla::ffi::DataType::F8E5M2FNUZ, "F8E5M2FNUZ"}, - {xla::ffi::DataType::F8E4M3FNUZ, "F8E4M3FNUZ"}, - {xla::ffi::DataType::F8E3M4, "F8E3M4"}, - {xla::ffi::DataType::F4E2M1FN, "F4E2M1FN"}, - {xla::ffi::DataType::F8E8M0FNU, "F8E8M0FNU"}, - }; - return map.at(dtype); -} - -inline void* data_ptr(ffi::AnyBuffer &buffer) { - return buffer.untyped_data(); -} - -inline void* data_ptr(ffi::Result &buffer) { - return data_ptr(*buffer); -} - -inline int byte_count(ffi::AnyBuffer &buffer) { - switch (buffer.element_type()) { - case xla::ffi::DataType::U32: - case xla::ffi::DataType::S32: - case xla::ffi::DataType::F32: - return 4; - case xla::ffi::DataType::F64: - case xla::ffi::DataType::S64: - return 8; - case xla::ffi::DataType::U8: - return 1; - default: - throw logic_error("Unsupported tensor datatype!"); - } -} - -#ifdef CUDA_BACKEND -void zero_buffer(ffi::AnyBuffer &buffer, stream_t stream) { - cudaMemsetAsync( - data_ptr(buffer), - 0, - buffer.element_count() * byte_count(buffer), - stream); -} -#endif -#ifdef HIP_BACKEND -void zero_buffer(ffi::AnyBuffer &buffer, stream_t stream) { - std::ignore = hipMemsetAsync( - data_ptr(buffer), - 0, - buffer.element_count() * byte_count(buffer), - stream); -} -#endif - -std::unordered_map parse_json_config(const json &j_obj) { - std::unordered_map result; - for (const auto &kv : j_obj.object_items()) { - result[kv.first] = static_cast(kv.second.number_value()); - } - return result; -} - -struct KernelProp { - int64_t L1_dim, L2_dim, L3_dim, weight_numel; - bool shared_weights; - xla::ffi::DataType irrep_dtype; - xla::ffi::DataType weight_dtype; - - int64_t workspace_size; // Convolution only - bool deterministic; - xla::ffi::DataType idx_dtype; - xla::ffi::DataType workspace_dtype; - - KernelProp() {} - - KernelProp( - std::unordered_map &kernel_dims, bool is_convolution): - L1_dim(kernel_dims.at("L1_dim")), - L2_dim(kernel_dims.at("L2_dim")), - L3_dim(kernel_dims.at("L3_dim")), - weight_numel(kernel_dims.at("weight_numel")), - shared_weights(kernel_dims.at("shared_weights")), - irrep_dtype(enum_to_xla_dtype(kernel_dims.at("irrep_dtype"))), - weight_dtype(enum_to_xla_dtype(kernel_dims.at("weight_dtype"))), - workspace_dtype(xla::ffi::DataType::U8) { - if(is_convolution) { - workspace_size = kernel_dims.at("workspace_size"); - deterministic = kernel_dims.at("deterministic"); - idx_dtype = enum_to_xla_dtype(kernel_dims.at("idx_dtype")); - } - } -}; - -std::unordered_map>, - KernelProp - >> tp_cache; - -std::unordered_map>, - KernelProp - >> conv_cache; -std::mutex mut; - -template -typename Cache::mapped_type &find_or_compile_cached( - Cache &cache, int64_t hash, Factory &&factory) { - const std::lock_guard lock(mut); - auto it = cache.find(hash); - if (it == cache.end()) { - it = cache.emplace(hash, std::forward(factory)()).first; - } - return it->second; -} - -std::pair*, KernelProp> - compile_tp_with_caching(std::string_view json_payload, - int64_t hash, - bool is_convolution) { - - auto &cached = find_or_compile_cached( - tp_cache, hash, [&] { - std::string err; - json root = json::parse(std::string(json_payload), err); - if (!err.empty()) throw std::runtime_error("JSON Parse Error: " + err); - - std::string kernel_src = root["kernel"].string_value(); - auto forward_cfg = parse_json_config(root["forward_config"]); - auto backward_cfg = parse_json_config(root["backward_config"]); - auto dbackward_cfg = parse_json_config(root["double_backward_config"]); - auto kernel_prop_map = parse_json_config(root["kernel_prop"]); - - auto jit_tp_impl = std::make_unique>( - kernel_src, - forward_cfg, - backward_cfg, - dbackward_cfg, - kernel_prop_map); - return std::make_pair(std::move(jit_tp_impl), - KernelProp(kernel_prop_map, is_convolution)); - }); - return {cached.first.get(), cached.second}; -} - -std::pair*, KernelProp> - compile_conv_with_caching(std::string_view json_payload, - int64_t hash, - bool is_convolution) { - - auto &cached = find_or_compile_cached( - conv_cache, hash, [&] { - std::string err; - json root = json::parse(std::string(json_payload), err); - if (!err.empty()) throw std::runtime_error("JSON Parse Error: " + err); - - std::string kernel_src = root["kernel"].string_value(); - auto forward_cfg = parse_json_config(root["forward_config"]); - auto backward_cfg = parse_json_config(root["backward_config"]); - auto dbackward_cfg = parse_json_config(root["double_backward_config"]); - auto kernel_prop_map = parse_json_config(root["kernel_prop"]); - - auto jit_conv_impl = std::make_unique>( - kernel_src, - forward_cfg, - backward_cfg, - dbackward_cfg, - kernel_prop_map); - return std::make_pair(std::move(jit_conv_impl), - KernelProp(kernel_prop_map, is_convolution)); - }); - return {cached.first.get(), cached.second}; -} - -inline void check_tensor(const ffi::AnyBuffer &buffer, - std::initializer_list expected_shape, - xla::ffi::DataType expected_dtype, - std::string tensor_name) { - const ffi::AnyBuffer::Dimensions dims = buffer.dimensions(); - if (dims.size() != expected_shape.size()) { - throw std::logic_error("Rank mismatch for tensor '" - + tensor_name - + "'. Expected rank " - + std::to_string(expected_shape.size()) - + ", got rank " - + std::to_string(dims.size())); - } - - for (size_t i = 0; i < dims.size(); i++) { - if (dims[i] != expected_shape.begin()[i]) { - throw std::logic_error("Shape mismatch for tensor '" - + tensor_name - + "'. Expected dimension " - + std::to_string(expected_shape.begin()[i]) - + " at index " - + std::to_string(i) - + ", got " - + std::to_string(dims[i])); - } - } - - if (buffer.element_type() != expected_dtype) { - throw std::logic_error("Datatype mismatch for tensor " + tensor_name + - ". Expected datatype " + xla_dtype_to_string(expected_dtype) + - ", got " + xla_dtype_to_string(buffer.element_type())); - } -} - -// --------------------- Tensor Products -------------------------- -ffi::Error tp_forward_impl( - ffi::AnyBuffer L1_in, - ffi::AnyBuffer L2_in, - ffi::AnyBuffer W, - ffi::Result L3_out, - stream_t stream, - std::string_view kernel_json, - int64_t hash) { - - auto [jit_kernel, k] = compile_tp_with_caching( - kernel_json, hash, false); - const int64_t num_batch = L1_in.dimensions()[0]; - - check_tensor(L1_in, {num_batch, k.L1_dim}, k.irrep_dtype, "L1_in"); - check_tensor(L2_in, {num_batch, k.L2_dim}, k.irrep_dtype, "L2_in"); - - if (k.shared_weights) - check_tensor(W, {k.weight_numel}, k.weight_dtype, "W"); - else - check_tensor(W, {num_batch, k.weight_numel}, k.weight_dtype, "W"); - - jit_kernel->exec_tensor_product( - num_batch, - data_ptr(L1_in), - data_ptr(L2_in), - data_ptr(L3_out), - data_ptr(W), - stream); - - return ffi::Error::Success(); -} - -ffi::Error tp_backward_impl( - ffi::AnyBuffer L1_in, - ffi::AnyBuffer L2_in, - ffi::AnyBuffer W, - ffi::AnyBuffer L3_grad, - ffi::Result L1_grad, - ffi::Result L2_grad, - ffi::Result W_grad, - stream_t stream, - std::string_view kernel_json, - int64_t hash) { - - auto [jit_kernel, k] = compile_tp_with_caching( - kernel_json, hash, false); - const int64_t num_batch = L1_in.dimensions()[0]; - check_tensor(L1_in, {num_batch, k.L1_dim}, k.irrep_dtype, "L1_in"); - check_tensor(L2_in, {num_batch, k.L2_dim}, k.irrep_dtype, "L2_in"); - check_tensor(L3_grad, {num_batch, k.L3_dim}, k.irrep_dtype, "L3_grad"); - - if (k.shared_weights) { - check_tensor(W, {k.weight_numel}, k.weight_dtype, "W"); - check_tensor(*W_grad, {k.weight_numel}, k.weight_dtype, "W_grad"); - } - else { - check_tensor(W, {num_batch, k.weight_numel}, k.weight_dtype, "W"); - check_tensor(*W_grad, {num_batch, k.weight_numel}, k.weight_dtype, "W_grad"); - } - - zero_buffer(*L1_grad, stream); - zero_buffer(*L2_grad, stream); - zero_buffer(*W_grad, stream); - - jit_kernel->backward( - num_batch, - data_ptr(L1_in), - data_ptr(L1_grad), - data_ptr(L2_in), - data_ptr(L2_grad), - data_ptr(W), - data_ptr(W_grad), - data_ptr(L3_grad), - stream); - return ffi::Error::Success(); -} - - -ffi::Error tp_double_backward_impl( - ffi::AnyBuffer L1_in, - ffi::AnyBuffer L2_in, - ffi::AnyBuffer W, - ffi::AnyBuffer L3_grad, - ffi::AnyBuffer L1_dgrad, - ffi::AnyBuffer L2_dgrad, - ffi::AnyBuffer W_dgrad, - ffi::Result L1_grad, - ffi::Result L2_grad, - ffi::Result W_grad, - ffi::Result L3_dgrad, - stream_t stream, - std::string_view kernel_json, - int64_t hash) { - - auto [jit_kernel, k] = compile_tp_with_caching( - kernel_json, hash, false); - const int64_t num_batch = L1_in.dimensions()[0]; - check_tensor(L1_in, {num_batch, k.L1_dim}, k.irrep_dtype, "L1_in"); - check_tensor(L2_in, {num_batch, k.L2_dim}, k.irrep_dtype, "L2_in"); - check_tensor(L3_grad, {num_batch, k.L3_dim}, k.irrep_dtype, "L3_grad"); - check_tensor(L1_dgrad, {num_batch, k.L1_dim}, k.irrep_dtype, "L1_dgrad"); - check_tensor(L2_dgrad, {num_batch, k.L2_dim}, k.irrep_dtype, "L2_dgrad"); - - if (k.shared_weights){ - check_tensor(W, {k.weight_numel}, k.weight_dtype, "W"); - check_tensor(W_dgrad, {k.weight_numel}, k.weight_dtype, "W_dgrad"); - } else { - check_tensor(W, {num_batch, k.weight_numel}, k.weight_dtype, "W"); - check_tensor(W_dgrad, {num_batch, k.weight_numel}, k.weight_dtype, "W_dgrad"); - } - - zero_buffer(*L1_grad, stream); - zero_buffer(*L2_grad, stream); - zero_buffer(*W_grad, stream); - zero_buffer(*L3_dgrad, stream); - - jit_kernel->double_backward( - num_batch, - data_ptr(L1_in), - data_ptr(L2_in), - data_ptr(W), - data_ptr(L3_grad), - data_ptr(L1_dgrad), - data_ptr(L2_dgrad), - data_ptr(W_dgrad), - data_ptr(L1_grad), - data_ptr(L2_grad), - data_ptr(W_grad), - data_ptr(L3_dgrad), - stream); - return ffi::Error::Success(); -} - -XLA_FFI_DEFINE_HANDLER_SYMBOL( - tp_forward, tp_forward_impl, - ffi::Ffi::Bind() - .Arg() - .Arg() - .Arg() - .Ret() - .Ctx>() - .Attr("kernel") - .Attr("hash"), - {xla::ffi::Traits::kCmdBufferCompatible}); // cudaGraph enabled - -XLA_FFI_DEFINE_HANDLER_SYMBOL( - tp_backward, tp_backward_impl, - ffi::Ffi::Bind() - .Arg() - .Arg() - .Arg() - .Arg() - .Ret() - .Ret() - .Ret() - .Ctx>() - .Attr("kernel") - .Attr("hash"), - {xla::ffi::Traits::kCmdBufferCompatible}); - -XLA_FFI_DEFINE_HANDLER_SYMBOL( - tp_double_backward, tp_double_backward_impl, - ffi::Ffi::Bind() - .Arg() - .Arg() - .Arg() - .Arg() - .Arg() - .Arg() - .Arg() - .Ret() - .Ret() - .Ret() - .Ret() - .Ctx>() - .Attr("kernel") - .Attr("hash"), - {xla::ffi::Traits::kCmdBufferCompatible}); - -// --------------------- Convolution -------------------------- -ffi::Error conv_forward_impl( - ffi::AnyBuffer L1_in, - ffi::AnyBuffer L2_in, - ffi::AnyBuffer W, - ffi::AnyBuffer rows, - ffi::AnyBuffer cols, - ffi::AnyBuffer workspace, - ffi::AnyBuffer transpose_perm, - ffi::Result L3_out, - stream_t stream, - std::string_view kernel_json, - int64_t hash) { - - auto [jit_kernel, k] = compile_conv_with_caching( - kernel_json, hash, true); - const int64_t nnz = rows.dimensions()[0]; - const int64_t node_count = L1_in.dimensions()[0]; - void* workspace_ptr = data_ptr(workspace); - - check_tensor(L1_in, {node_count, k.L1_dim}, k.irrep_dtype, "L1_in"); - check_tensor(L2_in, {nnz, k.L2_dim}, k.irrep_dtype, "L2_in"); - check_tensor(workspace, {k.workspace_size}, k.workspace_dtype, "workspace"); - check_tensor(rows, {nnz}, k.idx_dtype, "rows"); - check_tensor(cols, {nnz}, k.idx_dtype, "cols"); - - if (k.deterministic){ - check_tensor(transpose_perm, {nnz}, k.idx_dtype, "transpose perm"); - } - else { - workspace_ptr = nullptr; - } - zero_buffer(*L3_out, stream); - - if (k.shared_weights) - check_tensor(W, {k.weight_numel}, k.weight_dtype, "W"); - else - check_tensor(W, {nnz, k.weight_numel}, k.weight_dtype, "W"); - - jit_kernel->exec_conv( - data_ptr(L1_in), - data_ptr(L2_in), - data_ptr(W), - data_ptr(L3_out), - data_ptr(rows), - data_ptr(cols), - nnz, node_count, - workspace_ptr, - stream); - - return ffi::Error::Success(); -} - -ffi::Error conv_backward_impl( - ffi::AnyBuffer L1_in, - ffi::AnyBuffer L2_in, - ffi::AnyBuffer W, - ffi::AnyBuffer L3_grad, - ffi::Result L1_grad, - ffi::Result L2_grad, - ffi::Result W_grad, - ffi::AnyBuffer rows, - ffi::AnyBuffer cols, - ffi::AnyBuffer workspace, - ffi::AnyBuffer transpose_perm, - stream_t stream, - std::string_view kernel_json, - int64_t hash) { - - auto [jit_kernel, k] = compile_conv_with_caching( - kernel_json, hash, true); - const int64_t nnz = rows.dimensions()[0]; - const int64_t node_count = L1_in.dimensions()[0]; - void* workspace_ptr = data_ptr(workspace); - - check_tensor(L1_in, {node_count, k.L1_dim}, k.irrep_dtype, "L1_in"); - check_tensor(L2_in, {nnz, k.L2_dim}, k.irrep_dtype, "L2_in"); - check_tensor(L3_grad, {node_count, k.L3_dim}, k.irrep_dtype, "L3_grad"); - check_tensor(workspace, {k.workspace_size}, k.workspace_dtype, "workspace"); - check_tensor(rows, {nnz}, k.idx_dtype, "rows"); - check_tensor(cols, {nnz}, k.idx_dtype, "cols"); - - if (k.deterministic) { - check_tensor(transpose_perm, {nnz}, k.idx_dtype, "transpose perm"); - } - else { - workspace_ptr = nullptr; - } - zero_buffer(*L1_grad, stream); - zero_buffer(*L2_grad, stream); - zero_buffer(*W_grad, stream); - - if (k.shared_weights) { - check_tensor(W, {k.weight_numel}, k.weight_dtype, "W"); - check_tensor(*W_grad, {k.weight_numel}, k.weight_dtype, "W_grad"); - } - else { - check_tensor(W, {nnz, k.weight_numel}, k.weight_dtype, "W"); - check_tensor(*W_grad, {nnz, k.weight_numel}, k.weight_dtype, "W_grad"); - } - - jit_kernel->backward( - data_ptr(L1_in), - data_ptr(L1_grad), - data_ptr(L2_in), - data_ptr(L2_grad), - data_ptr(W), - data_ptr(W_grad), - data_ptr(L3_grad), - data_ptr(rows), - data_ptr(cols), - nnz, node_count, - workspace_ptr, - data_ptr(transpose_perm), - stream); - return ffi::Error::Success(); -} - -ffi::Error conv_double_backward_impl( - ffi::AnyBuffer L1_in, - ffi::AnyBuffer L2_in, - ffi::AnyBuffer W, - ffi::AnyBuffer L3_grad, - ffi::AnyBuffer L1_dgrad, - ffi::AnyBuffer L2_dgrad, - ffi::AnyBuffer W_dgrad, - ffi::Result L1_grad, - ffi::Result L2_grad, - ffi::Result W_grad, - ffi::Result L3_dgrad, - ffi::AnyBuffer rows, - ffi::AnyBuffer cols, - ffi::AnyBuffer workspace, - ffi::AnyBuffer transpose_perm, - stream_t stream, - std::string_view kernel_json, - int64_t hash) { - - auto [jit_kernel, k] = compile_conv_with_caching( - kernel_json, hash, true); - const int64_t nnz = rows.dimensions()[0]; - const int64_t node_count = L1_in.dimensions()[0]; - void* workspace_ptr = data_ptr(workspace); - - check_tensor(L1_in, {node_count, k.L1_dim}, k.irrep_dtype, "L1_in"); - check_tensor(L2_in, {nnz, k.L2_dim}, k.irrep_dtype, "L2_in"); - check_tensor(L3_grad, {node_count, k.L3_dim}, k.irrep_dtype, "L3_grad"); - check_tensor(L1_dgrad, {node_count, k.L1_dim}, k.irrep_dtype, "L1_dgrad"); - check_tensor(L2_dgrad, {nnz, k.L2_dim}, k.irrep_dtype, "L2_dgrad"); - check_tensor(workspace, {k.workspace_size}, k.workspace_dtype, "workspace"); - check_tensor(rows, {nnz}, k.idx_dtype, "rows"); - check_tensor(cols, {nnz}, k.idx_dtype, "cols"); - - if (k.deterministic) { - check_tensor(transpose_perm, {nnz}, k.idx_dtype, "transpose perm"); - } - else { - workspace_ptr = nullptr; - } - zero_buffer(*L1_grad, stream); - zero_buffer(*L2_grad, stream); - zero_buffer(*W_grad, stream); - zero_buffer(*L3_dgrad, stream); - - if (k.shared_weights) { - check_tensor(W, {k.weight_numel}, k.weight_dtype, "W"); - check_tensor(W_dgrad, {k.weight_numel}, k.weight_dtype, "W_dgrad"); - } else { - check_tensor(W, {nnz, k.weight_numel}, k.weight_dtype, "W"); - check_tensor(W_dgrad, {nnz, k.weight_numel}, k.weight_dtype, "W_dgrad"); - } - - jit_kernel->double_backward( - data_ptr(L1_in), - data_ptr(L2_in), - data_ptr(W), - data_ptr(L3_grad), - data_ptr(L1_dgrad), - data_ptr(L2_dgrad), - data_ptr(W_dgrad), - data_ptr(L1_grad), - data_ptr(L2_grad), - data_ptr(W_grad), - data_ptr(L3_dgrad), - data_ptr(rows), - data_ptr(cols), - nnz, node_count, - workspace_ptr, - data_ptr(transpose_perm), - stream); - return ffi::Error::Success(); -} - -// --------------------- FFI Bindings -------------------------- - -ffi::Error tp_initialize_impl(ffi::RemainingArgs, ffi::RemainingRets, stream_t, - std::string_view kernel_json, int64_t hash) { - compile_tp_with_caching(kernel_json, hash, false); - return ffi::Error::Success(); -} - -ffi::Error conv_initialize_impl(ffi::RemainingArgs, ffi::RemainingRets, stream_t, - std::string_view kernel_json, int64_t hash) { - compile_conv_with_caching(kernel_json, hash, true); - return ffi::Error::Success(); -} - -#define OEQ_STOCK_INITIALIZE_ATTRIBUTES \ - .RemainingArgs() \ - .RemainingRets() \ - .Ctx>() \ - .Attr("kernel") \ - .Attr("hash") - -XLA_FFI_DEFINE_HANDLER_SYMBOL( - tp_initialize, tp_initialize_impl, - ffi::Ffi::Bind() OEQ_STOCK_INITIALIZE_ATTRIBUTES); - -XLA_FFI_DEFINE_HANDLER_SYMBOL( - conv_initialize, conv_initialize_impl, - ffi::Ffi::Bind() OEQ_STOCK_INITIALIZE_ATTRIBUTES); - -#undef OEQ_STOCK_INITIALIZE_ATTRIBUTES - -XLA_FFI_DEFINE_HANDLER_SYMBOL( - conv_forward, conv_forward_impl, - ffi::Ffi::Bind() - .Arg() - .Arg() - .Arg() - .Arg() - .Arg() - .Arg() - .Arg() - .Ret() - .Ctx>() - .Attr("kernel") - .Attr("hash"), - {xla::ffi::Traits::kCmdBufferCompatible}); - -XLA_FFI_DEFINE_HANDLER_SYMBOL( - conv_backward, conv_backward_impl, - ffi::Ffi::Bind() - .Arg() - .Arg() - .Arg() - .Arg() - .Ret() - .Ret() - .Ret() - .Arg() - .Arg() - .Arg() - .Arg() - .Ctx>() - .Attr("kernel") - .Attr("hash"), - {xla::ffi::Traits::kCmdBufferCompatible}); - -XLA_FFI_DEFINE_HANDLER_SYMBOL( - conv_double_backward, conv_double_backward_impl, - ffi::Ffi::Bind() - .Arg() - .Arg() - .Arg() - .Arg() - .Arg() - .Arg() - .Arg() - .Ret() - .Ret() - .Ret() - .Ret() - .Arg() - .Arg() - .Arg() - .Arg() - .Ctx>() - .Attr("kernel") - .Attr("hash"), - {xla::ffi::Traits::kCmdBufferCompatible}); - -namespace { - -#define OEQ_FFI_HANDLER(NAME, INITIALIZE) \ - {#NAME, nullptr, nullptr, reinterpret_cast(INITIALIZE), \ - reinterpret_cast(NAME), OEQ_FFI_TRAIT_COMMAND_BUFFER_COMPATIBLE} - -const OeqFfiHandler kFfiHandlers[] = { - OEQ_FFI_HANDLER(tp_forward, tp_initialize), - OEQ_FFI_HANDLER(tp_backward, tp_initialize), - OEQ_FFI_HANDLER(tp_double_backward, tp_initialize), - OEQ_FFI_HANDLER(conv_forward, conv_initialize), - OEQ_FFI_HANDLER(conv_backward, conv_initialize), - OEQ_FFI_HANDLER(conv_double_backward, conv_initialize), -}; - -#undef OEQ_FFI_HANDLER - -const OeqFfiHandlerTable kFfiHandlerTable = { - OEQ_FFI_ABI_VERSION, - sizeof(kFfiHandlers) / sizeof(kFfiHandlers[0]), - kFfiHandlers, -}; - -} // namespace - -extern "C" const OeqFfiHandlerTable* oeq_ffi_handler_table() { - return &kFfiHandlerTable; -} diff --git a/openequivariance_extjax/src/ffi_handlers.cpp b/openequivariance_extjax/src/ffi_handlers.cpp new file mode 120000 index 00000000..637be54e --- /dev/null +++ b/openequivariance_extjax/src/ffi_handlers.cpp @@ -0,0 +1 @@ +../../openequivariance/openequivariance/extension/jax/ffi_handlers.cpp \ No newline at end of file diff --git a/openequivariance_extjax/src/ffi_provider.cpp b/openequivariance_extjax/src/ffi_provider.cpp deleted file mode 100644 index 52f0addb..00000000 --- a/openequivariance_extjax/src/ffi_provider.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include -#include - -#include "ffi_handler_table.h" -#include "xla/ffi/api/c_api.h" - -extern "C" OEQ_FFI_EXPORT int RegisterFFi( - const XLA_FFI_Api* api, - const char* platform_name) { - constexpr std::size_t kHandlerRegistrationSize = - offsetof(XLA_FFI_Api, XLA_FFI_Handler_Register) + - sizeof(api->XLA_FFI_Handler_Register); - if (api == nullptr || platform_name == nullptr || - api->struct_size < kHandlerRegistrationSize || - api->XLA_FFI_Handler_Register == nullptr || - api->XLA_FFI_Error_GetMessage == nullptr || - api->XLA_FFI_Error_Destroy == nullptr) { - return 1; - } - - const OeqFfiHandlerTable* handlers = oeq_ffi_handler_table(); - if (handlers == nullptr || handlers->abi_version != OEQ_FFI_ABI_VERSION) { - return 1; - } - for (uint32_t index = 0; index < handlers->handler_count; ++index) { - const OeqFfiHandler& handler = handlers->handlers[index]; - if (handler.execute == nullptr) { - std::cerr << "OpenEquivariance FFI target " << handler.name - << " has no execute handler" << std::endl; - return 1; - } - - XLA_FFI_Handler_Register_Args args{}; - args.struct_size = XLA_FFI_Handler_Register_Args_STRUCT_SIZE; - args.name = {handler.name, std::strlen(handler.name)}; - args.platform = {platform_name, std::strlen(platform_name)}; - args.bundle = { - reinterpret_cast(handler.instantiate), - reinterpret_cast(handler.prepare), - reinterpret_cast(handler.initialize), - reinterpret_cast(handler.execute), - }; - args.traits = static_cast(handler.traits); - - XLA_FFI_Error* error = api->XLA_FFI_Handler_Register(&args); - if (error == nullptr) { - std::cout << "[OpenEquivariance] Registered FFI target " - << handler.name << " for " << platform_name << std::endl; - continue; - } - - XLA_FFI_Error_GetMessage_Args message_args{}; - message_args.struct_size = XLA_FFI_Error_GetMessage_Args_STRUCT_SIZE; - message_args.error = error; - api->XLA_FFI_Error_GetMessage(&message_args); - std::cerr << "OpenEquivariance failed to register FFI target " - << handler.name << ": " - << (message_args.message == nullptr - ? "unknown XLA FFI error" - : message_args.message) - << std::endl; - - XLA_FFI_Error_Destroy_Args destroy_args{}; - destroy_args.struct_size = XLA_FFI_Error_Destroy_Args_STRUCT_SIZE; - destroy_args.error = error; - api->XLA_FFI_Error_Destroy(&destroy_args); - return 1; - } - return 0; -} diff --git a/openequivariance_extjax/src/ffi_provider.cpp b/openequivariance_extjax/src/ffi_provider.cpp new file mode 120000 index 00000000..cd7483c3 --- /dev/null +++ b/openequivariance_extjax/src/ffi_provider.cpp @@ -0,0 +1 @@ +../../openequivariance/openequivariance/extension/jax/ffi_provider.cpp \ No newline at end of file diff --git a/openequivariance_extjax/src/jax_ffi_registration.cpp b/openequivariance_extjax/src/jax_ffi_registration.cpp deleted file mode 100644 index 83d8d12f..00000000 --- a/openequivariance_extjax/src/jax_ffi_registration.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include "nanobind/nanobind.h" - -#include "ffi_handler_table.h" - -#ifdef CUDA_BACKEND -#include "backend/backend_cuda.hpp" -#endif - -#ifdef HIP_BACKEND -#include "backend/backend_hip.hpp" -#endif - -#include "nb_utilities.hpp" - -namespace nb = nanobind; - -namespace { - -bool IsHip() { -#ifdef HIP_BACKEND - return true; -#else - return false; -#endif -} - -void AddStage(nb::dict& stages, const char* name, void* handler) { - if (handler != nullptr) { - stages[name] = nb::capsule(handler); - } -} - -} // namespace - -NB_MODULE(openequivariance_extjax, m) { - m.def("registrations", []() { - nb::dict registrations; - const auto* handlers = oeq_ffi_handler_table(); - for (uint32_t index = 0; index < handlers->handler_count; ++index) { - const auto& handler = handlers->handlers[index]; - nb::dict stages; - AddStage(stages, "instantiate", handler.instantiate); - AddStage(stages, "prepare", handler.prepare); - AddStage(stages, "initialize", handler.initialize); - AddStage(stages, "execute", handler.execute); - registrations[handler.name] = stages; - } - return registrations; - }); - m.def("is_hip", &IsHip); - - oeq::bind_utilities(m); -} diff --git a/tests/jax_ffi_abi_test.py b/tests/jax_ffi_abi_test.py index e483457a..4c1e2f0d 100644 --- a/tests/jax_ffi_abi_test.py +++ b/tests/jax_ffi_abi_test.py @@ -1,4 +1,4 @@ -"""Runtime checks for the OEQ-owned C handler table and nanobind registry.""" +"""Runtime checks for the OEQ-owned C handler table and its JAX registrations.""" import ctypes from pathlib import Path @@ -19,7 +19,7 @@ COMMAND_BUFFER_COMPATIBLE = 1 -class _Handler(ctypes.Structure): +class Handler(ctypes.Structure): _fields_ = [ ("name", ctypes.c_char_p), ("instantiate", ctypes.c_void_p), @@ -30,23 +30,23 @@ class _Handler(ctypes.Structure): ] -class _HandlerTable(ctypes.Structure): +class HandlerTable(ctypes.Structure): _fields_ = [ ("abi_version", ctypes.c_uint32), ("handler_count", ctypes.c_uint32), - ("handlers", ctypes.POINTER(_Handler)), + ("handlers", ctypes.POINTER(Handler)), ] -def _handler_table(): - ext = pytest.importorskip("openequivariance_extjax") - library = ctypes.CDLL(ext.__file__) +def handler_table(): + ext = pytest.importorskip("openequivariance.jax.extlib") + library = ctypes.CDLL(ext.LIBRARY_PATH) table_fn = library.oeq_ffi_handler_table - table_fn.restype = ctypes.POINTER(_HandlerTable) + table_fn.restype = ctypes.POINTER(HandlerTable) return ext, table_fn().contents -def _capsule_pointer(capsule): +def capsule_pointer(capsule): get_pointer = ctypes.pythonapi.PyCapsule_GetPointer get_pointer.argtypes = [ctypes.py_object, ctypes.c_char_p] get_pointer.restype = ctypes.c_void_p @@ -58,7 +58,7 @@ def test_exported_handler_table_matches_manifest(with_jax): if not with_jax: pytest.skip("JAX ABI checks require --jax") - _, table = _handler_table() + _, table = handler_table() names = [ table.handlers[index].name.decode() for index in range(table.handler_count) ] @@ -79,13 +79,13 @@ def test_exported_handler_table_matches_manifest(with_jax): ) -def test_nanobind_registrations_match_handler_table(with_jax): - """Nanobind exposes each non-null ABI stage as a capsule.""" +def test_registrations_match_handler_table(with_jax): + """Each non-null ABI stage is registered as a capsule.""" if not with_jax: pytest.skip("JAX ABI checks require --jax") - ext, table = _handler_table() - registrations = ext.registrations() + ext, table = handler_table() + registrations = ext.registrations(ext.library) assert tuple(registrations) == FFI_TARGETS assert len(registrations) == table.handler_count == 6 for index, name in enumerate(FFI_TARGETS): @@ -94,7 +94,7 @@ def test_nanobind_registrations_match_handler_table(with_jax): expected_stages = {stage for stage in FFI_STAGES if getattr(handler, stage)} assert set(registration) == expected_stages for stage in expected_stages: - assert _capsule_pointer(registration[stage]) == getattr(handler, stage) + assert capsule_pointer(registration[stage]) == getattr(handler, stage) def test_handler_families_share_one_initializer(with_jax): @@ -102,7 +102,7 @@ def test_handler_families_share_one_initializer(with_jax): if not with_jax: pytest.skip("JAX ABI checks require --jax") - _, table = _handler_table() + _, table = handler_table() handlers = { table.handlers[index].name.decode(): table.handlers[index] for index in range(table.handler_count)