diff --git a/.github/workflows/build-ais.yml b/.github/workflows/build-ais.yml index 5a9ead9e..1578821f 100644 --- a/.github/workflows/build-ais.yml +++ b/.github/workflows/build-ais.yml @@ -137,6 +137,7 @@ jobs: -w /ais/hipFile/build \ "${AIS_CONTAINER_NAME}" \ /bin/bash -c ' + source /opt/venv/bin/activate export CPACK_DEBIAN_PACKAGE_RELEASE="${JOB_DESIGNATOR}${SLES_BUILD_ID_PREFIX}${BUILD_ID}~$(source /etc/os-release && echo ${VERSION_ID})" export CPACK_RPM_PACKAGE_RELEASE="${JOB_DESIGNATOR}${SLES_BUILD_ID_PREFIX}${BUILD_ID}" cmake \ diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 27311aed..e0606ca1 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -18,11 +18,12 @@ jobs: - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 #v6.2.0 - - name: Set up pylint and black + - name: Install packages run: | python -m pip install --upgrade pip python -m pip install pylint python -m pip install black + python -m pip install rocm-docs-core - name: Lint all Python files run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..f35d5c2f --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,18 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +sphinx: + configuration: docs/conf.py + +formats: [] + +python: + install: + - requirements: docs/sphinx/requirements.txt + +build: + os: ubuntu-22.04 + tools: + python: "3.10" diff --git a/CHANGELOG.md b/CHANGELOG.md index 2707832a..20455c23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ * `AIS_USE_SANITIZERS` now also enables the following sanitizers: integer, float-divide-by-zero, local-bounds, vptr, nullability (in addition to address, leak, and undefined). Sanitizers should also now emit usable stack trace info. * Added check in the Fastpath/AIS backend to ensure the HIP Runtime is initialized. This avoids causing a segfault in the HIP Runtime. * The default CMake build type was changed from `Debug` to `RelWithDebInfo` +* The docs directory has been reorganized and `AIS_BUILD_DOCS` will now generate Sphinx output for the web. ### Removed * The rocFile library has been completely removed and the code is now a part of hipFile. diff --git a/INSTALL.md b/INSTALL.md index b2e9cc97..35485392 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -116,6 +116,9 @@ sudo systemctl reboot * `libmount-dev` (Debian/Ubuntu) * `libmount-devel` (RHEL/Fedora/openSUSE) +Please the the README.md file in the docs directory for requirements +for building the documentation. + ### Configure You do not need to set the `HIP_PLATFORM` environment variable, as @@ -126,7 +129,7 @@ that will be set by CMake. Options |Option|Default|Purpose| |------|-------|-------| -|AIS\_BUILD\_DOCS|OFF|Build API documentation (requires Doxygen)| +|AIS\_BUILD\_DOCS|OFF|Build API documentation (see `docs/README.md`)| |AIS\_INSTALL\_EXAMPLES|ON|Install example programs| |AIS\_USE\_CLANG\_TIDY|OFF|Run the `clang-tidy` tool (clang only)| |AIS\_USE\_CODE\_COVERAGE|OFF|Generate code coverage information when tests are run (clang only)| @@ -191,18 +194,5 @@ The results will be wrote to `/build`, in the `coverage-report.txt` and `coverage-lines.txt` files. ### Documentation -The API documentation is built using Doxygen. To build it, use the -`AIS_BUILD_DOCS` option. This will build the documentation for any -libraries that have been configured. As a special case, configuring -the documentation without any library will build the documentation -for BOTH libraries, allowing for a docs-only build. - -The documentation will be built with the libraries and appear in -`docs/(hip|roc)file`. We build HTML, XML, and LaTeX docs. If you -want a pdf, run `make pdf` in the `latex` directory, which will -create a file named refman.pdf that you can rename. - -If you want to build the docs without compiling the libraries, -you can just build the `doc` target (if you've set `AIS_BUILD_DOCS`): - `cmake --build . --target doc` +See `docs/README.md` diff --git a/cmake/AISDocumentation.cmake b/cmake/AISDocumentation.cmake index 2407978f..2bb9596d 100644 --- a/cmake/AISDocumentation.cmake +++ b/cmake/AISDocumentation.cmake @@ -13,6 +13,17 @@ option(AIS_BUILD_DOCS "Build the hipFile docs (requires Doxygen, Sphinx, and Bre if(AIS_BUILD_DOCS) find_package(Doxygen REQUIRED) + find_package(Python3 REQUIRED COMPONENTS Interpreter) + + # Verify Sphinx + rocm_docs are available + execute_process( + COMMAND "${Python3_EXECUTABLE}" -c "import rocm_docs" + RESULT_VARIABLE _rocm_docs_found + OUTPUT_QUIET ERROR_QUIET + ) + if(NOT _rocm_docs_found EQUAL 0) + message(FATAL_ERROR "rocm_docs Python package not found. Install it with: pip install rocm-docs-core") + endif() # Set Doxygen input (pasted into Doxyfile.in) set(AIS_DOXYFILE_INPUT "${HIPFILE_ROOT_PATH}/include") @@ -20,20 +31,70 @@ if(AIS_BUILD_DOCS) # Set the path to the documentation set(AIS_DOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/docs") + ########### + # Doxygen # + ########### + + # The Doxygen HTML, XML, etc. goes in docs/doxygen/ + # Set the Doxyfile install location - set(AIS_DOXYFILE ${AIS_DOC_PATH}/Doxyfile) + set(AIS_DOXYFILE "${AIS_DOC_PATH}/doxygen/Doxyfile") + + # Doxygen XML output dir (consumed by Breathe inside Sphinx) + set(AIS_DOXYGEN_XML_DIR "${AIS_DOC_PATH}/doxygen/xml") # Create the Doxyfile from the input file - configure_file("docs/Doxyfile.in" ${AIS_DOXYFILE}) + configure_file("docs/doxygen/Doxyfile.in" ${AIS_DOXYFILE}) + + ########## + # Sphinx # + ########## + + # The Sphinx HTML, XML, etc. goes in docs/sphinx/ + + # Set the target for the config file and toc file + set(AIS_SPHINX_CONF_FILE "${AIS_DOC_PATH}/sphinx/conf.py") + set(AIS_SPHINX_TOC_FILE "${AIS_DOC_PATH}/sphinx/_toc.yml.in") + + # Sphinx HTML output dir + set(AIS_SPHINX_BUILD_DIR "${AIS_DOC_PATH}/sphinx/html") - # Set the output directory - set(DOXYGEN_OUT ${AIS_DOC_PATH}) + # Copy conf.py and the table of contents file + # Need to do this so the toc file doesn't dirty the repo + # _toc.yml.in is transformed by rocm_docs, not CMake + configure_file("docs/sphinx/conf.py" ${AIS_SPHINX_CONF_FILE} COPYONLY) + configure_file("docs/sphinx/_toc.yml.in" ${AIS_SPHINX_TOC_FILE} COPYONLY) - # Configure the documentation build - add_custom_target("doc" - COMMAND ${DOXYGEN_EXECUTABLE} ${AIS_DOXYFILE} - WORKING_DIRECTORY ${AIS_DOC_PATH} - COMMENT "Generating hipFile API documentation with Doxygen" + # Build docs: Doxygen first, then Sphinx (which pulls in rocm_docs via conf.py) + add_custom_target(doc + # Step 1: Run Doxygen to produce XML for Breathe + COMMAND "${DOXYGEN_EXECUTABLE}" "${AIS_DOXYFILE}" + # Step 2: Run Sphinx (html), injecting the Doxygen paths via environment + COMMAND + ${CMAKE_COMMAND} -E env + "DOXYFILE_PATH=${AIS_DOXYFILE}" + "DOXYGEN_ROOT=${AIS_DOC_PATH}/doxygen" + "DOXYGEN_XML_DIR=${AIS_DOXYGEN_XML_DIR}" + "${Python3_EXECUTABLE}" -m sphinx + -b html + -c "${AIS_DOC_PATH}/sphinx" + "${HIPFILE_ROOT_PATH}/docs" + "${AIS_SPHINX_BUILD_DIR}" + -v + # Step 3: Run Sphinx (LaTeX), injecting the Doxygen paths via environment + COMMAND + ${CMAKE_COMMAND} -E env + "DOXYFILE_PATH=${AIS_DOXYFILE}" + "DOXYGEN_ROOT=${AIS_DOC_PATH}/doxygen" + "DOXYGEN_XML_DIR=${AIS_DOXYGEN_XML_DIR}" + "${Python3_EXECUTABLE}" -m sphinx + -b latex + -c "${AIS_DOC_PATH}/sphinx" + "${HIPFILE_ROOT_PATH}/docs" + "${AIS_SPHINX_BUILD_DIR}" + -v + WORKING_DIRECTORY "${AIS_DOC_PATH}" + COMMENT "Generating hipFile API documentation with Doxygen + Sphinx (rocm_docs)" VERBATIM ) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..6b156561 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,79 @@ +# hipFile Documentation + +hipFile is documented using Doxygen markup in the public header +files and Sphinx .rst files. + +## Building the Documentation + +hipFile can generate two documentation products: + +1. API documentation +2. Sphinx HTML documentation (includes #1) + +API documentation is generated from Doxygen and limited to markup +in the header files. This will generate HTML, XML, and LaTeX output. +The LaTeX can be used to create a pdf document. + +The Sphinx documentation incorporates the Doxygen API markup and +adds reStructured text file content to generate HTML documentation +for the web. + +The documentation CMake target currently builds both. There is +no way to select "just Doxygen". + +### Requirements + +* CMake >= 3.21 +* Doxygen (we use 1.9.8, other versions are untested) +* Python 3.12 (earlier versions might work but are untested) +* The following Python packages: + * breathe + * rocm-docs-core + * sphinx + * sphinx-rtd-theme + +If you want to build a pdf, you will need LaTeX and pdflatex. + +### Generation + +Building the documentation is done via CMake by turning on the docs +option (which is off by default). + + `cmake -DAIS_BUILD_DOCS=ON ` + +You can then build the `doc` target. + + `cmake --build . --target doc` + +The documentation will be in the `docs` subdirectory of the build +directory: + +``` + ├── doxygen + │   ├── html + │   ├── latex + │   └── xml + └── sphinx + └── html +``` + +To generate a pdf for the API docs, navigate to `doxygen/latex` and run `make pdf` to +generate a pdf named `refman.pdf`. + +To generate a pdf for the Sphinx docs, navigate to `sphinx/html` and run `make` in +the directory. This will generate a pdf named `rocshmem.pdf` (we are stealing their +configs until `rocm-docs-core` is updated). + +## Adding to the Documentation + +### API Documentation (Doxygen) + +* Make your markup look like the rest of the file +* Doxygen macros can be found in `docs/doxygen/Doxyfile.in` +* All public API types, functions, etc. MUST have Doxygen markup + +### Other Documentation (Sphinx) + +All other documentation goes in Sphinx .rst documents. API-specific documentation +belongs in `docs/API` and other documentation goes in `docs`. Be sure to update +`sphinx/_toc.yml.in` if you add any new files. diff --git a/docs/async.rst b/docs/api/async.rst similarity index 100% rename from docs/async.rst rename to docs/api/async.rst diff --git a/docs/batch.rst b/docs/api/batch.rst similarity index 100% rename from docs/batch.rst rename to docs/api/batch.rst diff --git a/docs/core.rst b/docs/api/core.rst similarity index 100% rename from docs/core.rst rename to docs/api/core.rst diff --git a/docs/driver.rst b/docs/api/driver.rst similarity index 100% rename from docs/driver.rst rename to docs/api/driver.rst diff --git a/docs/errors.rst b/docs/api/errors.rst similarity index 100% rename from docs/errors.rst rename to docs/api/errors.rst diff --git a/docs/file.rst b/docs/api/file.rst similarity index 100% rename from docs/file.rst rename to docs/api/file.rst diff --git a/docs/misc_api.rst b/docs/api/misc_api.rst similarity index 100% rename from docs/misc_api.rst rename to docs/api/misc_api.rst diff --git a/docs/rdma.rst b/docs/api/rdma.rst similarity index 100% rename from docs/rdma.rst rename to docs/api/rdma.rst diff --git a/docs/Doxyfile.in b/docs/doxygen/Doxyfile.in similarity index 99% rename from docs/Doxyfile.in rename to docs/doxygen/Doxyfile.in index ad39635e..47b2b360 100644 --- a/docs/Doxyfile.in +++ b/docs/doxygen/Doxyfile.in @@ -68,7 +68,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = +OUTPUT_DIRECTORY = @AIS_DOC_PATH@/doxygen # If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096 # sub-directories (in 2 levels) under the output directory of each output format diff --git a/docs/hipify.rst b/docs/hipify.rst new file mode 100644 index 00000000..f47bdfee --- /dev/null +++ b/docs/hipify.rst @@ -0,0 +1,2 @@ +hipify +====== diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..73c3ef1f --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,43 @@ +.. meta:: + :description: hipFile does hipFile stuff + :keywords: hipFile, ROCm, storage, library, API, DMA + +********************* +hipFile documentation +********************* + +hipFile introduction goes here. For more information, see :doc:`introduction`. + +The hipFile public repository is located at ``_. + +.. grid:: 2 + :gutter: 3 + + .. grid-item-card:: Build and install + + * :doc:`Configure and build <./config_build>` + * :doc:`Sanitizers <./sanitizers>` + + .. grid-item-card:: Using hipFile + + * :doc:`hipify <./hipify>` + * :doc:`NVIDIA compatibility <./nvidia_compat>` + * :doc:`fio <./fio>` + * :doc:`Collecting statistics <./stats_collection>` + + .. grid-item-card:: API reference + + * :doc:`Errors <./api/errors>` + * :doc:`Core API <./api/core>` + * :doc:`Drivers <./api/driver>` + * :doc:`File operations <./api/file>` + * :doc:`Asynchronous operations <./api/async>` + * :doc:`Batch operations <./api/batch>` + * :doc:`RDMA <./api/rdma>` + * :doc:`Other <./api/misc_api>` + +To contribute to the documentation, refer to +`Contributing to ROCm `_. + +You can find licensing information on the +`Licensing `_ page. diff --git a/docs/sphinx/_toc.yml.in b/docs/sphinx/_toc.yml.in new file mode 100644 index 00000000..c4d32d37 --- /dev/null +++ b/docs/sphinx/_toc.yml.in @@ -0,0 +1,45 @@ +defaults: + numbered: False +root: index +subtrees: + +- entries: + - file: introduction.rst + title: Introduction + +- caption: Build and install + entries: + - file: config_build.rst + title: Configure and build + - file: sanitizers.rst + title: Sanitizers + +- caption: Using hipFile + entries: + - file: hipify.rst + title: hipify + - file: nvidia_compat.rst + title: cuFile compatibility + - file: fio.rst + title: fio support + - file: stats_collection.rst + title: Collecting statistics + +- caption: API reference + entries: + - file: api/errors.rst + title: Errors + - file: api/core.rst + title: Core + - file: api/driver.rst + title: Drivers + - file: api/file.rst + title: File operations + - file: api/async.rst + title: Asynchronous operations + - file: api/batch.rst + title: Batch operations + - file: api/rdma.rst + title: RDMA + - file: api/misc_api.rst + title: Other diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py new file mode 100644 index 00000000..52c5f78e --- /dev/null +++ b/docs/sphinx/conf.py @@ -0,0 +1,47 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# Pylint is NOT happy with the naming scheme ROCm chose +# pylint: disable=invalid-name + +"""This file is for configuring hipFile documentation""" + +import os +from rocm_docs import ROCmDocs + +version_number = "0.2.0" +left_nav_title = f"hipFile {version_number} documentation" + +# for PDF output on Read the Docs +project = "rocSHMEM" +author = "Advanced Micro Devices, Inc." +# pylint: disable=redefined-builtin +copyright = "Copyright (c) Advanced Micro Devices, Inc. All rights reserved." +# pylint: enable=redefined-builtin +version = version_number +release = version_number +external_projects_current_project = "rocshmem" +extensions = ["sphinx_design"] +external_toc_path = "./sphinx/_toc.yml" + +docs_core = ROCmDocs(left_nav_title) + +doxygen_root = os.environ.get("DOXYGEN_ROOT") +doxygen_xml_dir = os.environ.get("DOXYGEN_XML_DIR") + +if not doxygen_root or not doxygen_xml_dir: + raise RuntimeError( + "DOXYGEN_ROOT and DOXYGEN_XML_DIR must be set. " + "Build this documentation via CMake, not directly with sphinx-build." + ) + +docs_core.run_doxygen(doxygen_root=doxygen_root, doxygen_path=doxygen_xml_dir) + +docs_core.setup() + +# Transfer all Sphinx config variables into this module's global scope +for sphinx_var in ROCmDocs.SPHINX_VARS: + globals()[sphinx_var] = getattr(docs_core, sphinx_var) diff --git a/docs/stats_collection.rst b/docs/stats_collection.rst index e9d5e597..f8a9782b 100644 --- a/docs/stats_collection.rst +++ b/docs/stats_collection.rst @@ -4,10 +4,9 @@ Stats Collection Tool Command-line Tool ----------------- ``ais-stats`` can be run two ways: -* ``$ ais-stats -p [-i]`` will collect stats from a running process. - ``-i`` will report immediately rather than wait for the process to exit. -* ``$ ais-stats [args...]`` will launch ```` with the - provided arguments and report the collected stats when it exits. + +* ``$ ais-stats -p [-i]`` will collect stats from a running process. ``-i`` will report immediately rather than wait for the process to exit. +* ``$ ais-stats [args...]`` will launch ```` with the provided arguments and report the collected stats when it exits. Configuration ------------- @@ -24,4 +23,4 @@ Value Description Stats Collected --------------- * Basic: Bytes read/written on the fastpath backend -* Basic: Bytes read/written on the fallback backend \ No newline at end of file +* Basic: Bytes read/written on the fallback backend diff --git a/util/docker/DOCKERFILE.ais_ci_rocky b/util/docker/DOCKERFILE.ais_ci_rocky index 37a5f391..9d9d38f4 100644 --- a/util/docker/DOCKERFILE.ais_ci_rocky +++ b/util/docker/DOCKERFILE.ais_ci_rocky @@ -71,8 +71,18 @@ RUN dnf makecache && \ hip-devel \ libmount-devel \ llvm-devel \ + python3 \ + python3-pip \ rpm-build +# Install Python packages (don't source venv/bin/activate) +RUN python3 -m venv /opt/venv +RUN /opt/venv/bin/pip3 install \ + sphinx \ + breathe \ + sphinx-rtd-theme \ + rocm-docs-core + # Configure system linker for ROCm RUN cat < /etc/ld.so.conf.d/rocm.conf /opt/rocm/lib diff --git a/util/docker/DOCKERFILE.ais_ci_suse b/util/docker/DOCKERFILE.ais_ci_suse index 7e766415..c654c9e6 100644 --- a/util/docker/DOCKERFILE.ais_ci_suse +++ b/util/docker/DOCKERFILE.ais_ci_suse @@ -68,9 +68,21 @@ RUN zypper install -y \ hip-devel \ libmount-devel \ llvm19-devel \ + python312 \ + python312-pip \ rocm-device-libs \ rpm-build +# Install Python packages (don't source venv/bin/activate) +RUN python3.12 --version +RUN python3.12 -m venv /opt/venv +RUN /opt/venv/bin/pip3 install --upgrade pip +RUN /opt/venv/bin/pip3 install \ + sphinx \ + breathe \ + sphinx-rtd-theme \ + rocm-docs-core + # Configure system linker for ROCm RUN cat < /etc/ld.so.conf.d/rocm.conf /opt/rocm/lib diff --git a/util/docker/DOCKERFILE.ais_ci_ubuntu b/util/docker/DOCKERFILE.ais_ci_ubuntu index 0696bedb..288cfe20 100644 --- a/util/docker/DOCKERFILE.ais_ci_ubuntu +++ b/util/docker/DOCKERFILE.ais_ci_ubuntu @@ -51,8 +51,19 @@ RUN apt update && \ libboost-program-options-dev \ libmount-dev \ llvm-dev \ + python3 \ + python3-pip \ + python3-venv \ rocm-llvm-dev +# Install Python packages (don't source venv/bin/activate) +RUN python3 -m venv /opt/venv +RUN /opt/venv/bin/pip3 install \ + sphinx \ + breathe \ + sphinx-rtd-theme \ + rocm-docs-core + # Configure system linker for ROCm RUN cat < /etc/ld.so.conf.d/rocm.conf /opt/rocm/lib