Add Sphinx support to documentation

.github/workflows/build-ais.yml

@@ -137,6 +137,7 @@ jobs:
             -w /ais/hipFile/build \
             "${AIS_CONTAINER_NAME}" \
             /bin/bash -c '
+              source /opt/venv/bin/activate
               export CPACK_DEBIAN_PACKAGE_RELEASE="${JOB_DESIGNATOR}${SLES_BUILD_ID_PREFIX}${BUILD_ID}~$(source /etc/os-release && echo ${VERSION_ID})"
               export CPACK_RPM_PACKAGE_RELEASE="${JOB_DESIGNATOR}${SLES_BUILD_ID_PREFIX}${BUILD_ID}"
               cmake \

.github/workflows/pylint.yml

@@ -18,11 +18,12 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 #v6.2.0
 
-      - name: Set up pylint and black
+      - name: Install packages
         run: |
           python -m pip install --upgrade pip
           python -m pip install pylint
           python -m pip install black
+          python -m pip install rocm-docs-core
 
       - name: Lint all Python files
         run: |

.readthedocs.yaml

@@ -0,0 +1,18 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+formats: []
+
+python:
+  install:
+    - requirements: docs/sphinx/requirements.txt
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"

CHANGELOG.md

@@ -14,6 +14,7 @@
 * `AIS_USE_SANITIZERS` now also enables the following sanitizers: integer, float-divide-by-zero, local-bounds, vptr, nullability (in addition to address, leak, and undefined). Sanitizers should also now emit usable stack trace info.
 * Added check in the Fastpath/AIS backend to ensure the HIP Runtime is initialized. This avoids causing a segfault in the HIP Runtime.
 * The default CMake build type was changed from `Debug` to `RelWithDebInfo`
+* The docs directory has been reorganized and `AIS_BUILD_DOCS` will now generate Sphinx output for the web.
 
 ### Removed
 * The rocFile library has been completely removed and the code is now a part of hipFile.

INSTALL.md

@@ -116,6 +116,9 @@ sudo systemctl reboot
     * `libmount-dev` (Debian/Ubuntu)
     * `libmount-devel` (RHEL/Fedora/openSUSE)
 
+Please the the README.md file in the docs directory for requirements
+for building the documentation.
+
 ### Configure
 
 You do not need to set the `HIP_PLATFORM` environment variable, as
@@ -126,7 +129,7 @@ that will be set by CMake.
 Options
 |Option|Default|Purpose|
 |------|-------|-------|
-|AIS\_BUILD\_DOCS|OFF|Build API documentation (requires Doxygen)|
+|AIS\_BUILD\_DOCS|OFF|Build API documentation (see `docs/README.md`)|
 |AIS\_INSTALL\_EXAMPLES|ON|Install example programs|
 |AIS\_USE\_CLANG\_TIDY|OFF|Run the `clang-tidy` tool (clang only)|
 |AIS\_USE\_CODE\_COVERAGE|OFF|Generate code coverage information when tests are run (clang only)|
@@ -191,18 +194,5 @@ The results will be wrote to `<path/to/repo>/build`, in the
 `coverage-report.txt` and `coverage-lines.txt` files.
 
 ### Documentation
-The API documentation is built using Doxygen. To build it, use the
-`AIS_BUILD_DOCS` option. This will build the documentation for any
-libraries that have been configured. As a special case, configuring
-the documentation without any library will build the documentation
-for BOTH libraries, allowing for a docs-only build.
-
-The documentation will be built with the libraries and appear in
-`docs/(hip|roc)file`. We build HTML, XML, and LaTeX docs. If you
-want a pdf, run `make pdf` in the `latex` directory, which will
-create a file named refman.pdf that you can rename.
-
-If you want to build the docs without compiling the libraries,
-you can just build the `doc` target (if you've set `AIS_BUILD_DOCS`):
 
-    `cmake --build . --target doc`
+See `docs/README.md`

cmake/AISDocumentation.cmake

@@ -13,27 +13,88 @@ option(AIS_BUILD_DOCS "Build the hipFile docs (requires Doxygen, Sphinx, and Bre
 
 if(AIS_BUILD_DOCS)
     find_package(Doxygen REQUIRED)
+    find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+    # Verify Sphinx + rocm_docs are available
+    execute_process(
+        COMMAND "${Python3_EXECUTABLE}" -c "import rocm_docs"
+        RESULT_VARIABLE _rocm_docs_found
+        OUTPUT_QUIET ERROR_QUIET
+    )
+    if(NOT _rocm_docs_found EQUAL 0)
+        message(FATAL_ERROR "rocm_docs Python package not found. Install it with: pip install rocm-docs-core")
+    endif()
 
     # Set Doxygen input (pasted into Doxyfile.in)
     set(AIS_DOXYFILE_INPUT "${HIPFILE_ROOT_PATH}/include")
 
     # Set the path to the documentation
     set(AIS_DOC_PATH "${CMAKE_CURRENT_BINARY_DIR}/docs")
 
+    ###########
+    # Doxygen #
+    ###########
+
+    # The Doxygen HTML, XML, etc. goes in docs/doxygen/
+
     # Set the Doxyfile install location
-    set(AIS_DOXYFILE ${AIS_DOC_PATH}/Doxyfile)
+    set(AIS_DOXYFILE "${AIS_DOC_PATH}/doxygen/Doxyfile")
+
+    # Doxygen XML output dir (consumed by Breathe inside Sphinx)
+    set(AIS_DOXYGEN_XML_DIR "${AIS_DOC_PATH}/doxygen/xml")
 
     # Create the Doxyfile from the input file
-    configure_file("docs/Doxyfile.in" ${AIS_DOXYFILE})
+    configure_file("docs/doxygen/Doxyfile.in" ${AIS_DOXYFILE})
+
+    ##########
+    # Sphinx #
+    ##########
+
+    # The Sphinx HTML, XML, etc. goes in docs/sphinx/
+
+    # Set the target for the config file and toc file
+    set(AIS_SPHINX_CONF_FILE "${AIS_DOC_PATH}/sphinx/conf.py")
+    set(AIS_SPHINX_TOC_FILE "${AIS_DOC_PATH}/sphinx/_toc.yml.in")
+
+    # Sphinx HTML output dir
+    set(AIS_SPHINX_BUILD_DIR "${AIS_DOC_PATH}/sphinx/html")
 
-    # Set the output directory
-    set(DOXYGEN_OUT ${AIS_DOC_PATH})
+    # Copy conf.py and the table of contents file
+    # Need to do this so the toc file doesn't dirty the repo
+    # _toc.yml.in is transformed by rocm_docs, not CMake
+    configure_file("docs/sphinx/conf.py" ${AIS_SPHINX_CONF_FILE} COPYONLY)
+    configure_file("docs/sphinx/_toc.yml.in" ${AIS_SPHINX_TOC_FILE} COPYONLY)
 
-    # Configure the documentation build
-    add_custom_target("doc"
-        COMMAND ${DOXYGEN_EXECUTABLE} ${AIS_DOXYFILE}
-        WORKING_DIRECTORY ${AIS_DOC_PATH}
-        COMMENT "Generating hipFile API documentation with Doxygen"
+    # Build docs: Doxygen first, then Sphinx (which pulls in rocm_docs via conf.py)
+    add_custom_target(doc
+        # Step 1: Run Doxygen to produce XML for Breathe
+        COMMAND "${DOXYGEN_EXECUTABLE}" "${AIS_DOXYFILE}"
+        # Step 2: Run Sphinx (html), injecting the Doxygen paths via environment
+        COMMAND
+            ${CMAKE_COMMAND} -E env
+                "DOXYFILE_PATH=${AIS_DOXYFILE}"
+                "DOXYGEN_ROOT=${AIS_DOC_PATH}/doxygen"
+                "DOXYGEN_XML_DIR=${AIS_DOXYGEN_XML_DIR}"
+            "${Python3_EXECUTABLE}" -m sphinx
+                -b html
+                -c "${AIS_DOC_PATH}/sphinx"
+                "${HIPFILE_ROOT_PATH}/docs"
+                "${AIS_SPHINX_BUILD_DIR}"
+                -v
+        # Step 3: Run Sphinx (LaTeX), injecting the Doxygen paths via environment
+        COMMAND
+            ${CMAKE_COMMAND} -E env
+                "DOXYFILE_PATH=${AIS_DOXYFILE}"
+                "DOXYGEN_ROOT=${AIS_DOC_PATH}/doxygen"
+                "DOXYGEN_XML_DIR=${AIS_DOXYGEN_XML_DIR}"
+            "${Python3_EXECUTABLE}" -m sphinx
+                -b latex
+                -c "${AIS_DOC_PATH}/sphinx"
+                "${HIPFILE_ROOT_PATH}/docs"
+                "${AIS_SPHINX_BUILD_DIR}"
+                -v
+        WORKING_DIRECTORY "${AIS_DOC_PATH}"
+        COMMENT "Generating hipFile API documentation with Doxygen + Sphinx (rocm_docs)"
         VERBATIM
     )
 

docs/README.md

@@ -0,0 +1,79 @@
+# hipFile Documentation
+
+hipFile is documented using Doxygen markup in the public header
+files and Sphinx .rst files.
+
+## Building the Documentation
+
+hipFile can generate two documentation products:
+
+1. API documentation
+2. Sphinx HTML documentation (includes #1)
+
+API documentation is generated from Doxygen and limited to markup
+in the header files. This will generate HTML, XML, and LaTeX output.
+The LaTeX can be used to create a pdf document.
+
+The Sphinx documentation incorporates the Doxygen API markup and
+adds reStructured text file content to generate HTML documentation
+for the web.
+
+The documentation CMake target currently builds both. There is
+no way to select "just Doxygen".
+
+### Requirements
+
+* CMake >= 3.21
+* Doxygen (we use 1.9.8, other versions are untested)
+* Python 3.12 (earlier versions might work but are untested)
+* The following Python packages:
+    * breathe
+    * rocm-docs-core
+    * sphinx
+    * sphinx-rtd-theme
+
+If you want to build a pdf, you will need LaTeX and pdflatex.
+
+### Generation
+
+Building the documentation is done via CMake by turning on the docs
+option (which is off by default).
+
+    `cmake -DAIS_BUILD_DOCS=ON <options> <path/to/source>`
+
+You can then build the `doc` target.
+
+    `cmake --build . --target doc`
+
+The documentation will be in the `docs` subdirectory of the build
+directory:
+
+```
+    ├── doxygen
+    │   ├── html
+    │   ├── latex
+    │   └── xml
+    └── sphinx
+        └── html
+```
+
+To generate a pdf for the API docs, navigate to `doxygen/latex` and run `make pdf` to
+generate a pdf named `refman.pdf`.
+
+To generate a pdf for the Sphinx docs, navigate to `sphinx/html` and run `make` in
+the directory. This will generate a pdf named `rocshmem.pdf` (we are stealing their
+configs until `rocm-docs-core` is updated).
+
+## Adding to the Documentation
+
+### API Documentation (Doxygen)
+
+* Make your markup look like the rest of the file
+* Doxygen macros can be found in `docs/doxygen/Doxyfile.in`
+* All public API types, functions, etc. MUST have Doxygen markup
+
+### Other Documentation (Sphinx)
+
+All other documentation goes in Sphinx .rst documents. API-specific documentation
+belongs in `docs/API` and other documentation goes in `docs`. Be sure to update
+`sphinx/_toc.yml.in` if you add any new files.

docs/Doxyfile.in → docs/doxygen/Doxyfile.in

@@ -68,7 +68,7 @@ PROJECT_LOGO           =
 # entered, it will be relative to the location where doxygen was started. If
 # left blank the current directory will be used.
 
-OUTPUT_DIRECTORY       =
+OUTPUT_DIRECTORY       = @AIS_DOC_PATH@/doxygen
 
 # If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096
 # sub-directories (in 2 levels) under the output directory of each output format

docs/hipify.rst

@@ -0,0 +1,2 @@
+hipify
+======

docs/index.rst

@@ -0,0 +1,43 @@
+.. meta::
+  :description: hipFile does hipFile stuff
+  :keywords: hipFile, ROCm, storage, library, API, DMA
+
+*********************
+hipFile documentation
+*********************
+
+hipFile introduction goes here. For more information, see :doc:`introduction`.
+
+The hipFile public repository is located at `<https://github.com/ROCm/hipFile>`_.
+
+.. grid:: 2
+  :gutter: 3
+
+  .. grid-item-card:: Build and install
+
+      * :doc:`Configure and build <./config_build>`
+      * :doc:`Sanitizers <./sanitizers>`
+
+  .. grid-item-card:: Using hipFile
+
+      * :doc:`hipify <./hipify>`
+      * :doc:`NVIDIA compatibility <./nvidia_compat>`
+      * :doc:`fio <./fio>`
+      * :doc:`Collecting statistics <./stats_collection>`
+
+  .. grid-item-card:: API reference
+
+      * :doc:`Errors <./api/errors>`
+      * :doc:`Core API <./api/core>`
+      * :doc:`Drivers <./api/driver>`
+      * :doc:`File operations <./api/file>`
+      * :doc:`Asynchronous operations <./api/async>`
+      * :doc:`Batch operations <./api/batch>`
+      * :doc:`RDMA <./api/rdma>`
+      * :doc:`Other <./api/misc_api>`
+
+To contribute to the documentation, refer to
+`Contributing to ROCm <https://rocm.docs.amd.com/en/latest/contribute/contributing.html>`_.
+
+You can find licensing information on the
+`Licensing <https://rocm.docs.amd.com/en/latest/about/license.html>`_ page.

docs/sphinx/_toc.yml.in

@@ -0,0 +1,45 @@
+defaults:
+  numbered: False
+root: index
+subtrees:
+
+- entries:
+  - file: introduction.rst
+    title: Introduction
+
+- caption: Build and install
+  entries:
+  - file: config_build.rst
+    title: Configure and build
+  - file: sanitizers.rst
+    title: Sanitizers
+
+- caption: Using hipFile
+  entries:
+  - file: hipify.rst
+    title: hipify
+  - file: nvidia_compat.rst
+    title: cuFile compatibility
+  - file: fio.rst
+    title: fio support
+  - file: stats_collection.rst
+    title: Collecting statistics
+
+- caption: API reference
+  entries:
+  - file: api/errors.rst
+    title: Errors
+  - file: api/core.rst
+    title: Core
+  - file: api/driver.rst
+    title: Drivers
+  - file: api/file.rst
+    title: File operations
+  - file: api/async.rst
+    title: Asynchronous operations
+  - file: api/batch.rst
+    title: Batch operations
+  - file: api/rdma.rst
+    title: RDMA
+  - file: api/misc_api.rst
+    title: Other