diff --git a/code/doxygen-config.in b/code/doxygen-config.in index f5a2794bb..99f385e8d 100644 --- a/code/doxygen-config.in +++ b/code/doxygen-config.in @@ -1316,8 +1316,10 @@ IGNORE_PREFIX = # If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output # The default value is: YES. +# Disabled: the API Reference is rendered by breathe from the XML output below, and this standalone HTML +# documentation is never published (it is neither installed by CMakeLists.txt nor added to html_extra_path). -GENERATE_HTML = YES +GENERATE_HTML = NO # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of @@ -1886,8 +1888,9 @@ MATHJAX_CODEFILE = # option. # The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. +# Disabled along with GENERATE_HTML. -SEARCHENGINE = YES +SEARCHENGINE = NO # When the SERVER_BASED_SEARCH tag is enabled the search engine will be # implemented using a web server instead of a web client using JavaScript. There @@ -2263,8 +2266,10 @@ XML_OUTPUT = xml # of the XML output. # The default value is: YES. # This tag requires that the tag GENERATE_XML is set to YES. +# Disabled: the program listings account for a third of the XML that breathe has to parse on every build, and +# none of the doxygen directives used in the API Reference renders them. -XML_PROGRAMLISTING = YES +XML_PROGRAMLISTING = NO # If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include # namespace members in file scope as well, matching the HTML output. diff --git a/docs/conf.py b/docs/conf.py index 8ab20839f..428e33f44 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -222,7 +222,44 @@ def get_git_branch(): """ rtd_type = os.environ.get("READTHEDOCS_VERSION_TYPE") if rtd_type in ("branch", "tag"): +<<<<<<< HEAD return os.environ.get("READTHEDOCS_VERSION") +======= + version_name = os.environ.get("READTHEDOCS_VERSION") + if version_name in ("latest", "stable"): + # "latest" and "stable" are RTD pseudo-names, but RTD also exposes the git ref it actually + # checked out for them (e.g. "3.6.x" or "v3.6.1"). Preferring it keeps the API Reference and the + # GitHub links on the branch the documentation is really being built from instead of master. + # The ref is validated against the remote before being used for the checkout, so an unexpected + # value simply falls back to master as before. + git_identifier = os.environ.get("READTHEDOCS_GIT_IDENTIFIER") + if git_identifier and git_identifier not in ("latest", "stable"): + return git_identifier + if version_name == "latest": + # "latest" is an RTD pseudo-name, not a real branch → fall back to master. + return None + if version_name == "stable": + # "stable" is an RTD pseudo-name. Find the latest vX.Y.Z tag in the repo. + # Release tags live on dedicated branches (not main), so we scan all tags + # rather than restricting to those reachable from HEAD. + path_to_here = os.path.abspath(os.path.dirname(__file__)) + try: + p = subprocess.Popen( + ["git", "tag", "--sort=-version:refname"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=path_to_here, + ) + tags = p.communicate()[0].decode().splitlines() + release_tag_re = re.compile(r'^v\d+\.\d+\.\d+$') + for tag in tags: + if release_tag_re.match(tag.strip()): + return tag.strip() + except Exception: + pass + return None + return version_name +>>>>>>> 3fee709 (Reduce Read the Docs build time and fix version branch resolution (#1306)) if rtd_type == "external": return None @@ -301,6 +338,96 @@ def configure_doxyfile( file.write(filedata) +def resolve_remote_ref(url, preferred_ref, display_name): + """ + Resolve the branch or tag a repository must be checked out at, without downloading it. + + ``git ls-remote`` is used so that falling back to master, when the remote does not have + ``preferred_ref``, does not require cloning anything first. + + :param url: URL of the repository. + :param preferred_ref: Branch or tag to use if the remote has it. + :param display_name: Repository name, used for logging. + :return: Tuple with the branch or tag to use and the commit it points to. The commit is ``None`` if the + remote could not be queried. + """ + ref = preferred_ref + try: + remote_refs = git.cmd.Git().ls_remote("--heads", "--tags", url, ref, "master") + except git.GitCommandError as e: + print("Failed to list the refs of {}. Git Error: {}".format(display_name, e)) + return ref, None + + # ``git ls-remote`` matches the pattern against the tail of the ref path, so asking for "master" also + # matches a branch named "feature/some-work/master". Require an exact branch or tag match, otherwise the + # clone would be attempted with a ref that does not exist and fail instead of falling back. + commits = {} + for line in remote_refs.splitlines(): + commit, _, ref_path = line.partition("\t") + commits[ref_path] = commit + + for candidate in (ref, "master"): + for ref_path in ( + "refs/heads/{}".format(candidate), + "refs/tags/{}".format(candidate), + ): + if ref_path in commits: + return candidate, commits[ref_path] + print( + '{} does not have branch or tag "{}"; falling back to master'.format( + display_name, candidate + ) + ) + + return "master", None + + +def repo_is_at_commit(path, commit, display_name): + """ + Check whether an already cloned repository is checked out at a given commit. + + :param path: Local path of the repository. + :param commit: Commit that the repository is expected to be checked out at. + :param display_name: Repository name, used for logging. + :return: True only if the repository exists and its HEAD is that commit. + """ + if not commit or not os.path.isdir(path): + return False + + try: + head_commit = git.Repo(path).head.commit.hexsha + except Exception as e: + print("Could not read the HEAD of {}. Error: {}".format(display_name, e)) + return False + + if head_commit != commit: + print( + "{} is checked out at {} instead of {}".format( + display_name, head_commit[:10], commit[:10] + ) + ) + return False + + return True + + +def clone_repo_at_ref(url, path, ref, display_name): + """ + Clone a repository at a given branch or tag, downloading as little as possible. + + Only the tip of the ref is fetched (``--depth 1``), which is all that doxygen and SWIG need: the full + history of Fast DDS is over 100 MB and none of it is used. + + :param url: URL of the repository to clone. + :param path: Local path where the repository will be cloned. + :param ref: Branch or tag to check out, as resolved by ``resolve_remote_ref``. + :param display_name: Repository name, used for logging. + :return: The cloned repository. + """ + print('Cloning {} at "{}"'.format(display_name, ref)) + return git.Repo.clone_from(url, path, branch=ref, depth=1, single_branch=True) + + script_path = os.path.abspath(pathlib.Path(__file__).parent.absolute()) # Project directories project_source_dir = os.path.abspath("{}/../code".format(script_path)) @@ -342,6 +469,7 @@ def configure_doxyfile( if read_the_docs_build: print("Read the Docs environment detected!") +<<<<<<< HEAD fastdds_repo_name = os.path.abspath("{}/fastdds".format(project_binary_dir)) fastdds_python_repo_name = os.path.abspath( @@ -355,20 +483,17 @@ def configure_doxyfile( if os.path.isdir(fastdds_python_repo_name): print("Removing existing repository in {}".format(fastdds_python_repo_name)) shutil.rmtree(fastdds_python_repo_name) - - # Create necessary directory path - os.makedirs(os.path.dirname(fastdds_repo_name), exist_ok=True) - os.makedirs(os.path.dirname(fastdds_python_repo_name), exist_ok=True) - - # Clone repositories - - # - Fast DDS - print("Cloning Fast DDS") - fastdds = git.Repo.clone_from( - "https://github.com/eProsima/Fast-DDS.git", - fastdds_repo_name, +======= + fastdds_url = "https://github.com/eProsima/Fast-DDS.git" + fastdds_python_url = "https://github.com/eProsima/Fast-DDS-python.git" +>>>>>>> 3fee709 (Reduce Read the Docs build time and fix version branch resolution (#1306)) + + doxygen_index = os.path.join(output_dir, "xml", "index.xml") + swig_output = os.path.join( + fastdds_python_repo_name, "fastdds_python", "src", "swig", "fastddsPYTHON_wrap.cxx" ) +<<<<<<< HEAD # Verify the desired branch/tag actually exists in the cloned remote, falling back to master if not. fastdds_branch = fastdds_fallback_branch if fastdds.refs.__contains__("origin/{}".format(fastdds_branch)): @@ -377,11 +502,35 @@ def configure_doxyfile( # GitPython exposes tags by bare name, e.g. "v3.6.2". pass else: +======= + # Branch or tag, and the commit it currently points to, that each repository must be checked out at + fastdds_ref, fastdds_commit = resolve_remote_ref( + fastdds_url, fastdds_fallback_branch, "Fast DDS" + ) + fastdds_python_ref, fastdds_python_commit = resolve_remote_ref( + fastdds_python_url, fastdds_python_fallback_branch, "Fast DDS Python Bindings" + ) + + # Read the Docs runs a separate sphinx-build, which imports this file again, for every enabled output + # format. Cloning the repositories and running doxygen and SWIG once per format would repeat several + # minutes of work, so the preparation is skipped when a previous run of this build already completed it. + # The repositories must be checked out at the expected commit, and not merely exist, for their doxygen + # documentation and SWIG code to be the ones this build needs. + if ( + repo_is_at_commit(fastdds_repo_name, fastdds_commit, "Fast DDS") + and repo_is_at_commit( + fastdds_python_repo_name, fastdds_python_commit, "Fast DDS Python Bindings" + ) + and os.path.isfile(doxygen_index) + and os.path.isfile(swig_output) + ): +>>>>>>> 3fee709 (Reduce Read the Docs build time and fix version branch resolution (#1306)) print( - 'Fast DDS does not have branch or tag "{}"; falling back to master'.format( - fastdds_branch + "Reusing the repositories, doxygen documentation and SWIG code already generated in {}".format( + project_binary_dir ) ) +<<<<<<< HEAD fastdds_branch = "origin/3.2.x" # Actual checkout @@ -402,59 +551,75 @@ def configure_doxyfile( elif fastdds_python.tags.__contains__(fastdds_python_branch): # GitPython exposes tags by bare name, e.g. "v3.6.1". pass +======= +>>>>>>> 3fee709 (Reduce Read the Docs build time and fix version branch resolution (#1306)) else: - print( - 'Fast DDS Python does not have branch or tag "{}"; falling back to master'.format( - fastdds_python_branch - ) + # Remove the repositories left behind by an incomplete attempt, as cloning needs an empty directory + if os.path.isdir(fastdds_repo_name): + print("Removing existing repository in {}".format(fastdds_repo_name)) + shutil.rmtree(fastdds_repo_name) + if os.path.isdir(fastdds_python_repo_name): + print("Removing existing repository in {}".format(fastdds_python_repo_name)) + shutil.rmtree(fastdds_python_repo_name) + + # Create necessary directory path + os.makedirs(os.path.dirname(fastdds_repo_name), exist_ok=True) + os.makedirs(os.path.dirname(fastdds_python_repo_name), exist_ok=True) + + # Clone repositories at the branch or tag this documentation is being built from + clone_repo_at_ref(fastdds_url, fastdds_repo_name, fastdds_ref, "Fast DDS") + clone_repo_at_ref( + fastdds_python_url, + fastdds_python_repo_name, + fastdds_python_ref, + "Fast DDS Python Bindings", ) +<<<<<<< HEAD fastdds_python_branch = "origin/2.2.x" +======= +>>>>>>> 3fee709 (Reduce Read the Docs build time and fix version branch resolution (#1306)) + + os.makedirs(os.path.dirname(output_dir), exist_ok=True) + os.makedirs(os.path.dirname(doxygen_html), exist_ok=True) + + # Configure Doxyfile + configure_doxyfile( + doxyfile_in, + doxyfile_out, + input_dir, + output_dir, + project_binary_dir, + project_source_dir, + ) + # Generate doxygen documentation + doxygen_ret = subprocess.call("doxygen {}".format(doxyfile_out), shell=True) + if doxygen_ret != 0: + print("Doxygen failed with return code {}".format(doxygen_ret)) + sys.exit(doxygen_ret) + + # Generate SWIG code. + swig_ret = subprocess.call( + "swig \ + -python \ + -doxygen \ + -I{}/include \ + -DFASTDDS_DOCS_BUILD \ + -outdir {}/fastdds_python/src/swig \ + -c++ \ + -interface _fastdds_python \ + -o {}/fastdds_python/src/swig/fastddsPYTHON_wrap.cxx \ + {}/fastdds_python/src/swig/fastdds.i".format( + fastdds_repo_name, + fastdds_python_repo_name, + fastdds_python_repo_name, + fastdds_python_repo_name, + ), + shell=True, + ) - # Actual checkout - print('Checking out Fast DDS Python branch "{}"'.format(fastdds_python_branch)) - fastdds_python.git.checkout(fastdds_python_branch) - - os.makedirs(os.path.dirname(output_dir), exist_ok=True) - os.makedirs(os.path.dirname(doxygen_html), exist_ok=True) - - # Configure Doxyfile - configure_doxyfile( - doxyfile_in, - doxyfile_out, - input_dir, - output_dir, - project_binary_dir, - project_source_dir, - ) - # Generate doxygen documentation - doxygen_ret = subprocess.call("doxygen {}".format(doxyfile_out), shell=True) - if doxygen_ret != 0: - print("Doxygen failed with return code {}".format(doxygen_ret)) - sys.exit(doxygen_ret) - - # Generate SWIG code. - swig_ret = subprocess.call( - "swig \ - -python \ - -doxygen \ - -I{}/include \ - -DFASTDDS_DOCS_BUILD \ - -outdir {}/fastdds_python/src/swig \ - -c++ \ - -interface _fastdds_python \ - -o {}/fastdds_python/src/swig/fastddsPYTHON_wrap.cxx \ - {}/fastdds_python/src/swig/fastdds.i".format( - fastdds_repo_name, - fastdds_python_repo_name, - fastdds_python_repo_name, - fastdds_python_repo_name, - ), - shell=True, - ) - - if swig_ret != 0: - print("SWIG failed with return code {}".format(swig_ret)) - sys.exit(swig_ret) + if swig_ret != 0: + print("SWIG failed with return code {}".format(swig_ret)) + sys.exit(swig_ret) fastdds_python_imported_location = "{}/fastdds_python/src/swig".format( fastdds_python_repo_name diff --git a/readthedocs.yaml b/readthedocs.yaml index bdba5d0b2..175e58d5c 100644 --- a/readthedocs.yaml +++ b/readthedocs.yaml @@ -12,6 +12,15 @@ build: apt_packages: - plantuml - swig + jobs: + build: + # By default Read the Docs builds the downloadable documentation with an extra + # `sphinx-build -b singlehtml` run, which means building the whole documentation a second time (over 8 + # minutes). The `html` format is built first and its output is already the complete documentation, so + # the downloadable file is simply that output compressed, which takes about a second. + htmlzip: + - mkdir -p $READTHEDOCS_OUTPUT/htmlzip + - python -m zipfile -c $READTHEDOCS_OUTPUT/htmlzip/fast-dds-docs.zip $READTHEDOCS_OUTPUT/html # Build documentation in the docs/ directory with Sphinx sphinx: @@ -21,8 +30,12 @@ sphinx: #mkdocs: # configuration: mkdocs.yml -# Optionally build your docs in additional formats such as PDF and ePub -formats: all +# Read the Docs builds each additional format with its own `sphinx-build` invocation, so every one of them +# re-runs the whole documentation build. Only `htmlzip` is kept, as it provides the downloadable +# documentation for offline reading, and it is generated from the already built HTML (see `build.jobs` +# above) instead of running Sphinx again; `pdf` and `epub` are not built. +formats: + - htmlzip # Optionally set the version of Python and requirements required to build your docs python: