[25592] Reduce Read the Docs build time and fix version branch resolution (backport #1306)

code/doxygen-config.in

@@ -1316,8 +1316,10 @@ IGNORE_PREFIX          =
 
 # If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output
 # The default value is: YES.
+# Disabled: the API Reference is rendered by breathe from the XML output below, and this standalone HTML
+# documentation is never published (it is neither installed by CMakeLists.txt nor added to html_extra_path).
 
-GENERATE_HTML          = YES
+GENERATE_HTML          = NO
 
 # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
 # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
@@ -1886,8 +1888,9 @@ MATHJAX_CODEFILE       =
 # option.
 # The default value is: YES.
 # This tag requires that the tag GENERATE_HTML is set to YES.
+# Disabled along with GENERATE_HTML.
 
-SEARCHENGINE           = YES
+SEARCHENGINE           = NO
 
 # When the SERVER_BASED_SEARCH tag is enabled the search engine will be
 # implemented using a web server instead of a web client using JavaScript. There
@@ -2263,8 +2266,10 @@ XML_OUTPUT             = xml
 # of the XML output.
 # The default value is: YES.
 # This tag requires that the tag GENERATE_XML is set to YES.
+# Disabled: the program listings account for a third of the XML that breathe has to parse on every build, and
+# none of the doxygen directives used in the API Reference renders them.
 
-XML_PROGRAMLISTING     = YES
+XML_PROGRAMLISTING     = NO
 
 # If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include
 # namespace members in file scope as well, matching the HTML output.

docs/conf.py

@@ -222,7 +222,44 @@ def get_git_branch():
     """
     rtd_type = os.environ.get("READTHEDOCS_VERSION_TYPE")
     if rtd_type in ("branch", "tag"):
+<<<<<<< HEAD
         return os.environ.get("READTHEDOCS_VERSION")
+=======
+        version_name = os.environ.get("READTHEDOCS_VERSION")
+        if version_name in ("latest", "stable"):
+            # "latest" and "stable" are RTD pseudo-names, but RTD also exposes the git ref it actually
+            # checked out for them (e.g. "3.6.x" or "v3.6.1"). Preferring it keeps the API Reference and the
+            # GitHub links on the branch the documentation is really being built from instead of master.
+            # The ref is validated against the remote before being used for the checkout, so an unexpected
+            # value simply falls back to master as before.
+            git_identifier = os.environ.get("READTHEDOCS_GIT_IDENTIFIER")
+            if git_identifier and git_identifier not in ("latest", "stable"):
+                return git_identifier
+        if version_name == "latest":
+            # "latest" is an RTD pseudo-name, not a real branch → fall back to master.
+            return None
+        if version_name == "stable":
+            # "stable" is an RTD pseudo-name. Find the latest vX.Y.Z tag in the repo.
+            # Release tags live on dedicated branches (not main), so we scan all tags
+            # rather than restricting to those reachable from HEAD.
+            path_to_here = os.path.abspath(os.path.dirname(__file__))
+            try:
+                p = subprocess.Popen(
+                    ["git", "tag", "--sort=-version:refname"],
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    cwd=path_to_here,
+                )
+                tags = p.communicate()[0].decode().splitlines()
+                release_tag_re = re.compile(r'^v\d+\.\d+\.\d+$')
+                for tag in tags:
+                    if release_tag_re.match(tag.strip()):
+                        return tag.strip()
+            except Exception:
+                pass
+            return None
+        return version_name
+>>>>>>> 3fee709 (Reduce Read the Docs build time and fix version branch resolution (#1306))
     if rtd_type == "external":
         return None
 
@@ -301,6 +338,96 @@ def configure_doxyfile(
         file.write(filedata)
 
 
+def resolve_remote_ref(url, preferred_ref, display_name):
+    """
+    Resolve the branch or tag a repository must be checked out at, without downloading it.
+
+    ``git ls-remote`` is used so that falling back to master, when the remote does not have
+    ``preferred_ref``, does not require cloning anything first.
+
+    :param url: URL of the repository.
+    :param preferred_ref: Branch or tag to use if the remote has it.
+    :param display_name: Repository name, used for logging.
+    :return: Tuple with the branch or tag to use and the commit it points to. The commit is ``None`` if the
+        remote could not be queried.
+    """
+    ref = preferred_ref
+    try:
+        remote_refs = git.cmd.Git().ls_remote("--heads", "--tags", url, ref, "master")
+    except git.GitCommandError as e:
+        print("Failed to list the refs of {}. Git Error: {}".format(display_name, e))
+        return ref, None
+
+    # ``git ls-remote`` matches the pattern against the tail of the ref path, so asking for "master" also
+    # matches a branch named "feature/some-work/master". Require an exact branch or tag match, otherwise the
+    # clone would be attempted with a ref that does not exist and fail instead of falling back.
+    commits = {}
+    for line in remote_refs.splitlines():
+        commit, _, ref_path = line.partition("\t")
+        commits[ref_path] = commit
+
+    for candidate in (ref, "master"):
+        for ref_path in (
+            "refs/heads/{}".format(candidate),
+            "refs/tags/{}".format(candidate),
+        ):
+            if ref_path in commits:
+                return candidate, commits[ref_path]
+        print(
+            '{} does not have branch or tag "{}"; falling back to master'.format(
+                display_name, candidate
+            )
+        )
+
+    return "master", None
+
+
+def repo_is_at_commit(path, commit, display_name):
+    """
+    Check whether an already cloned repository is checked out at a given commit.
+
+    :param path: Local path of the repository.
+    :param commit: Commit that the repository is expected to be checked out at.
+    :param display_name: Repository name, used for logging.
+    :return: True only if the repository exists and its HEAD is that commit.
+    """
+    if not commit or not os.path.isdir(path):
+        return False
+
+    try:
+        head_commit = git.Repo(path).head.commit.hexsha
+    except Exception as e:
+        print("Could not read the HEAD of {}. Error: {}".format(display_name, e))
+        return False
+
+    if head_commit != commit:
+        print(
+            "{} is checked out at {} instead of {}".format(
+                display_name, head_commit[:10], commit[:10]
+            )
+        )
+        return False
+
+    return True
+
+
+def clone_repo_at_ref(url, path, ref, display_name):
+    """
+    Clone a repository at a given branch or tag, downloading as little as possible.
+
+    Only the tip of the ref is fetched (``--depth 1``), which is all that doxygen and SWIG need: the full
+    history of Fast DDS is over 100 MB and none of it is used.
+
+    :param url: URL of the repository to clone.
+    :param path: Local path where the repository will be cloned.
+    :param ref: Branch or tag to check out, as resolved by ``resolve_remote_ref``.
+    :param display_name: Repository name, used for logging.
+    :return: The cloned repository.
+    """
+    print('Cloning {} at "{}"'.format(display_name, ref))
+    return git.Repo.clone_from(url, path, branch=ref, depth=1, single_branch=True)
+
+
 script_path = os.path.abspath(pathlib.Path(__file__).parent.absolute())
 # Project directories
 project_source_dir = os.path.abspath("{}/../code".format(script_path))
@@ -342,6 +469,7 @@ def configure_doxyfile(
 if read_the_docs_build:
     print("Read the Docs environment detected!")
 
+<<<<<<< HEAD
     fastdds_repo_name = os.path.abspath("{}/fastdds".format(project_binary_dir))
 
     fastdds_python_repo_name = os.path.abspath(
@@ -355,20 +483,17 @@ def configure_doxyfile(
     if os.path.isdir(fastdds_python_repo_name):
         print("Removing existing repository in {}".format(fastdds_python_repo_name))
         shutil.rmtree(fastdds_python_repo_name)
-
-    # Create necessary directory path
-    os.makedirs(os.path.dirname(fastdds_repo_name), exist_ok=True)
-    os.makedirs(os.path.dirname(fastdds_python_repo_name), exist_ok=True)
-
-    # Clone repositories
-
-    # - Fast DDS
-    print("Cloning Fast DDS")
-    fastdds = git.Repo.clone_from(
-        "https://github.com/eProsima/Fast-DDS.git",
-        fastdds_repo_name,
+=======
+    fastdds_url = "https://github.com/eProsima/Fast-DDS.git"
+    fastdds_python_url = "https://github.com/eProsima/Fast-DDS-python.git"
+>>>>>>> 3fee709 (Reduce Read the Docs build time and fix version branch resolution (#1306))
+
+    doxygen_index = os.path.join(output_dir, "xml", "index.xml")
+    swig_output = os.path.join(
+        fastdds_python_repo_name, "fastdds_python", "src", "swig", "fastddsPYTHON_wrap.cxx"
     )
 
+<<<<<<< HEAD
     # Verify the desired branch/tag actually exists in the cloned remote, falling back to master if not.
     fastdds_branch = fastdds_fallback_branch
     if fastdds.refs.__contains__("origin/{}".format(fastdds_branch)):
@@ -377,11 +502,35 @@ def configure_doxyfile(
         # GitPython exposes tags by bare name, e.g. "v3.6.2".
         pass
     else:
+=======
+    # Branch or tag, and the commit it currently points to, that each repository must be checked out at
+    fastdds_ref, fastdds_commit = resolve_remote_ref(
+        fastdds_url, fastdds_fallback_branch, "Fast DDS"
+    )
+    fastdds_python_ref, fastdds_python_commit = resolve_remote_ref(
+        fastdds_python_url, fastdds_python_fallback_branch, "Fast DDS Python Bindings"
+    )
+
+    # Read the Docs runs a separate sphinx-build, which imports this file again, for every enabled output
+    # format. Cloning the repositories and running doxygen and SWIG once per format would repeat several
+    # minutes of work, so the preparation is skipped when a previous run of this build already completed it.
+    # The repositories must be checked out at the expected commit, and not merely exist, for their doxygen
+    # documentation and SWIG code to be the ones this build needs.
+    if (
+        repo_is_at_commit(fastdds_repo_name, fastdds_commit, "Fast DDS")
+        and repo_is_at_commit(
+            fastdds_python_repo_name, fastdds_python_commit, "Fast DDS Python Bindings"
+        )
+        and os.path.isfile(doxygen_index)
+        and os.path.isfile(swig_output)
+    ):
+>>>>>>> 3fee709 (Reduce Read the Docs build time and fix version branch resolution (#1306))
         print(
-            'Fast DDS does not have branch or tag "{}"; falling back to master'.format(
-                fastdds_branch
+            "Reusing the repositories, doxygen documentation and SWIG code already generated in {}".format(
+                project_binary_dir
             )
         )
+<<<<<<< HEAD
         fastdds_branch = "origin/3.2.x"
 
     # Actual checkout
@@ -402,59 +551,75 @@ def configure_doxyfile(
     elif fastdds_python.tags.__contains__(fastdds_python_branch):
         # GitPython exposes tags by bare name, e.g. "v3.6.1".
         pass
+=======
+>>>>>>> 3fee709 (Reduce Read the Docs build time and fix version branch resolution (#1306))
     else:
-        print(
-            'Fast DDS Python does not have branch or tag "{}"; falling back to master'.format(
-                fastdds_python_branch
-            )
+        # Remove the repositories left behind by an incomplete attempt, as cloning needs an empty directory
+        if os.path.isdir(fastdds_repo_name):
+            print("Removing existing repository in {}".format(fastdds_repo_name))
+            shutil.rmtree(fastdds_repo_name)
+        if os.path.isdir(fastdds_python_repo_name):
+            print("Removing existing repository in {}".format(fastdds_python_repo_name))
+            shutil.rmtree(fastdds_python_repo_name)
+
+        # Create necessary directory path
+        os.makedirs(os.path.dirname(fastdds_repo_name), exist_ok=True)
+        os.makedirs(os.path.dirname(fastdds_python_repo_name), exist_ok=True)
+
+        # Clone repositories at the branch or tag this documentation is being built from
+        clone_repo_at_ref(fastdds_url, fastdds_repo_name, fastdds_ref, "Fast DDS")
+        clone_repo_at_ref(
+            fastdds_python_url,
+            fastdds_python_repo_name,
+            fastdds_python_ref,
+            "Fast DDS Python Bindings",
         )
+<<<<<<< HEAD
         fastdds_python_branch = "origin/2.2.x"
+=======
+>>>>>>> 3fee709 (Reduce Read the Docs build time and fix version branch resolution (#1306))
+
+        os.makedirs(os.path.dirname(output_dir), exist_ok=True)
+        os.makedirs(os.path.dirname(doxygen_html), exist_ok=True)
+
+        # Configure Doxyfile
+        configure_doxyfile(
+            doxyfile_in,
+            doxyfile_out,
+            input_dir,
+            output_dir,
+            project_binary_dir,
+            project_source_dir,
+        )
+        # Generate doxygen documentation
+        doxygen_ret = subprocess.call("doxygen {}".format(doxyfile_out), shell=True)
+        if doxygen_ret != 0:
+            print("Doxygen failed with return code {}".format(doxygen_ret))
+            sys.exit(doxygen_ret)
+
+        # Generate SWIG code.
+        swig_ret = subprocess.call(
+            "swig \
+            -python \
+            -doxygen \
+            -I{}/include \
+            -DFASTDDS_DOCS_BUILD \
+            -outdir {}/fastdds_python/src/swig \
+            -c++ \
+            -interface _fastdds_python \
+            -o {}/fastdds_python/src/swig/fastddsPYTHON_wrap.cxx \
+            {}/fastdds_python/src/swig/fastdds.i".format(
+                fastdds_repo_name,
+                fastdds_python_repo_name,
+                fastdds_python_repo_name,
+                fastdds_python_repo_name,
+            ),
+            shell=True,
+        )
 
-    # Actual checkout
-    print('Checking out Fast DDS Python branch "{}"'.format(fastdds_python_branch))
-    fastdds_python.git.checkout(fastdds_python_branch)
-
-    os.makedirs(os.path.dirname(output_dir), exist_ok=True)
-    os.makedirs(os.path.dirname(doxygen_html), exist_ok=True)
-
-    # Configure Doxyfile
-    configure_doxyfile(
-        doxyfile_in,
-        doxyfile_out,
-        input_dir,
-        output_dir,
-        project_binary_dir,
-        project_source_dir,
-    )
-    # Generate doxygen documentation
-    doxygen_ret = subprocess.call("doxygen {}".format(doxyfile_out), shell=True)
-    if doxygen_ret != 0:
-        print("Doxygen failed with return code {}".format(doxygen_ret))
-        sys.exit(doxygen_ret)
-
-    # Generate SWIG code.
-    swig_ret = subprocess.call(
-        "swig \
-        -python \
-        -doxygen \
-        -I{}/include \
-        -DFASTDDS_DOCS_BUILD \
-        -outdir {}/fastdds_python/src/swig \
-        -c++ \
-        -interface _fastdds_python \
-        -o {}/fastdds_python/src/swig/fastddsPYTHON_wrap.cxx \
-        {}/fastdds_python/src/swig/fastdds.i".format(
-            fastdds_repo_name,
-            fastdds_python_repo_name,
-            fastdds_python_repo_name,
-            fastdds_python_repo_name,
-        ),
-        shell=True,
-    )
-
-    if swig_ret != 0:
-        print("SWIG failed with return code {}".format(swig_ret))
-        sys.exit(swig_ret)
+        if swig_ret != 0:
+            print("SWIG failed with return code {}".format(swig_ret))
+            sys.exit(swig_ret)
 
     fastdds_python_imported_location = "{}/fastdds_python/src/swig".format(
         fastdds_python_repo_name