fix: 0.11.70 release

.github/workflows/from_scratch.yml

@@ -407,15 +407,15 @@ jobs:
               exit 1
             fi
 
-            # A partially published latest release may not contain Windows
-            # artifacts. Fall back to the version shipped with tman, which is
-            # published as a complete, matching package set.
+            # The tman release itself may also be only partially published for
+            # Windows. Use the latest app from before that release so the
+            # fallback does not select the same incomplete package set again.
             echo "⚠ Latest package set is unavailable for Windows"
-            echo "Falling back to transcriber_demo version $TMAN_VERSION"
+            echo "Falling back to the latest transcriber_demo before $TMAN_VERSION"
 
             cd "$WORK_DIR"
             mv transcriber_demo transcriber_demo_latest
-            tman install app "transcriber_demo@=$TMAN_VERSION"
+            tman install app "transcriber_demo@<$TMAN_VERSION"
 
             cd "$WORK_DIR/transcriber_demo"
             tman install

build/ten_common/rust/cargo_build_tests.py

@@ -6,11 +6,12 @@
 #
 import argparse
 import json
-import sys
 import os
-from build.scripts import cmd_exec, fs_utils, timestamp_proxy
+import sys
 from pathlib import Path
 
+from build.scripts import cmd_exec, fs_utils, timestamp_proxy
+
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 import target_dir_tag
 
@@ -30,7 +31,7 @@ def __init__(self):
         self.integration_test_output_name: str | None = None
 
 
-def get_crate_test_output_name(log_level: int) -> str:
+def get_crate_test_info(log_level: int) -> tuple[str, str]:
     cmd = ["cargo", "metadata", "--no-deps", "--format-version", "1"]
     returncode, output = cmd_exec.run_cmd(cmd, log_level)
     if returncode:
@@ -40,53 +41,79 @@ def get_crate_test_output_name(log_level: int) -> str:
     # messages from cargo.
     output = output.splitlines()[-1]
     metadata = json.loads(output)
-    for target in metadata["packages"][0]["targets"]:
+    package = metadata["packages"][0]
+    for target in package["targets"]:
         if target["test"]:
-            return target["name"]
+            return package["id"], target["name"]
 
     raise RuntimeError("Failed to get crate name from targets.")
 
 
-def copy_test_binaries(
-    source_dir: str,
-    base_name: str,
-    output_dir: str,
-    target_name: str,
-    log_level: int = 0,
-):
-    if log_level > 0:
-        print(
-            f"Copying test binary {base_name} from {source_dir} to {output_dir}"
-        )
+def parse_cargo_test_binaries(
+    cargo_output: str, package_id: str
+) -> tuple[dict[str, str], str]:
+    test_binaries: dict[str, str] = {}
+    console_lines: list[str] = []
 
-    for f in os.listdir(source_dir):
-        is_executable = os.access(os.path.join(source_dir, f), os.X_OK)
-        if not is_executable:
+    for line in cargo_output.splitlines():
+        try:
+            message = json.loads(line)
+        except json.JSONDecodeError:
+            console_lines.append(line)
             continue
 
-        # The file name generated by `cargo build` always has a hash suffix, ex:
-        # <library_name>-<hash>.<extension>. We need to remove the hash suffix.
+        if not isinstance(message, dict):
+            continue
 
-        # We have to deal with the extension of the executable file, as if the
-        # `crate-type` is `cdylib`, the extension is `.so` on linux, which is
-        # not what we want. The extension is not present on linux or macos, and
-        # the extension is '.exe' on windows.
-        extension = os.path.splitext(f)[1]
-        if extension != "" and extension != ".exe":
+        if message.get("reason") == "compiler-message":
+            rendered = message.get("message", {}).get("rendered")
+            if rendered:
+                console_lines.append(rendered.rstrip())
             continue
 
-        binary_name = f[: f.rindex("-") if "-" in f else -1]
-        if binary_name != base_name:
+        if (
+            message.get("reason") != "compiler-artifact"
+            or message.get("package_id") != package_id
+            or not message.get("profile", {}).get("test")
+            or not message.get("executable")
+        ):
             continue
 
-        dest = target_name + extension
+        target_name = message.get("target", {}).get("name")
+        if target_name:
+            test_binaries[target_name] = message["executable"]
+
+    return test_binaries, "\n".join(console_lines)
+
 
-        fs_utils.copy_file(
-            os.path.join(source_dir, f),
-            os.path.join(output_dir, dest),
-            True,
+def copy_test_binary(
+    test_binaries: dict[str, str],
+    cargo_target_name: str,
+    output_dir: str,
+    target_name: str,
+    log_level: int = 0,
+):
+    source = test_binaries.get(cargo_target_name)
+    if not source:
+        available_targets = ", ".join(sorted(test_binaries)) or "none"
+        raise RuntimeError(
+            f"Cargo did not report a test binary for {cargo_target_name}. "
+            f"Available test targets: {available_targets}"
         )
 
+    if log_level > 0:
+        print(
+            f"Copying test binary {cargo_target_name} from {source} "
+            f"to {output_dir}"
+        )
+
+    extension = ".exe" if source.lower().endswith(".exe") else ""
+    fs_utils.copy_file(
+        source,
+        os.path.join(output_dir, target_name + extension),
+        True,
+    )
+
 
 def run_clippy_static_checking(args: ArgumentInfo):
     cmd = [
@@ -161,6 +188,10 @@ def run_clippy_static_checking(args: ArgumentInfo):
 
         # run_clippy_static_checking(args)
 
+        # Cargo's JSON messages provide artifact paths without relying on its
+        # internal target directory layout.
+        package_id, unit_test_output_name = get_crate_test_info(args.log_level)
+
         # cargo build --tests: only compile the test source files, without
         # running the test cases.
         # cargo test: compile the test source files and run the test cases.
@@ -176,6 +207,7 @@ def run_clippy_static_checking(args: ArgumentInfo):
                 "--target",
                 args.target,
                 "--tests",
+                "--message-format=json-render-diagnostics",
             ]
         else:
             cmd = [
@@ -185,34 +217,29 @@ def run_clippy_static_checking(args: ArgumentInfo):
                 args.target_path,
                 "--target",
                 args.target,
+                "--message-format=json-render-diagnostics",
             ]
 
         if args.build_type == "release":
             cmd.append("--release")
 
         returncode, logs = cmd_exec.run_cmd(cmd, args.log_level)
+        test_binaries, console_logs = parse_cargo_test_binaries(
+            logs, package_id
+        )
         if returncode:
-            raise RuntimeError(f"Failed to build rust tests: {logs}")
-        else:
-            print(logs)
+            raise RuntimeError(
+                f"Failed to build rust tests: {console_logs or logs}"
+            )
+        if console_logs:
+            print(console_logs)
 
         target_dir_tag.ensure_cargo_target_dir_tag(args.target_path)
 
-        # The prefix of the unit test binary is the crate name (i.e., the
-        # 'name' in [[bin]] or [[lib]] in a crate).
-        unit_test_output_name = get_crate_test_output_name(args.log_level)
-
-        # The output of the tests will be in
-        # <target_path>/<target>/<build_type>/deps
         if args.test_output_dir != "":
             # Copy the unit test binary.
-            copy_test_binaries(
-                os.path.join(
-                    args.target_path,
-                    args.target,
-                    args.build_type,
-                    "deps",
-                ),
+            copy_test_binary(
+                test_binaries,
                 unit_test_output_name,
                 args.test_output_dir,
                 "unit_test",
@@ -221,13 +248,8 @@ def run_clippy_static_checking(args: ArgumentInfo):
 
             if args.integration_test_output_name:
                 # Copy the integration test binary.
-                copy_test_binaries(
-                    os.path.join(
-                        args.target_path,
-                        args.target,
-                        args.build_type,
-                        "deps",
-                    ),
+                copy_test_binary(
+                    test_binaries,
                     args.integration_test_output_name,
                     args.test_output_dir,
                     "integration_test",

tools/tman/upload-to-ppa.sh

@@ -269,27 +269,34 @@ EOF
     log_info "Uploading file: $changes_file"
 
     # Launchpad uploads can intermittently fail with transient FTP-side 550
-    # errors after a previous distro upload has already succeeded. Retry the
-    # upload a few times before treating it as a hard failure.
+    # errors in the middle of a multi-file source upload. Retry with a capped
+    # exponential backoff before treating it as a hard failure.
     upload_success=false
-    for attempt in 1 2 3; do
-        log_info "Upload attempt ${attempt}/3"
+    attempt=1
+    while [ "$attempt" -le 6 ]; do
+        log_info "Upload attempt ${attempt}/6"
 
         if dput "ppa:${LAUNCHPAD_ID}/${PPA_NAME}" "$changes_file" 2>&1; then
             log_info "Upload successful"
             upload_success=true
             break
         fi
 
-        if [ "$attempt" -lt 3 ]; then
-            sleep_seconds=$((attempt * 20))
+        if [ "$attempt" -lt 6 ]; then
+            sleep_seconds=$((20 * (2 ** (attempt - 1))))
+            if [ "$sleep_seconds" -gt 180 ]; then
+                sleep_seconds=180
+            fi
+
             log_error "Upload failed for ${SOURCE_PACKAGE_NAME}/${dist} on attempt ${attempt}. Retrying in ${sleep_seconds}s..."
             sleep "$sleep_seconds"
         fi
+
+        attempt=$((attempt + 1))
     done
 
     if [ "$upload_success" != true ]; then
-        log_error "Upload failed after 3 attempts: ${SOURCE_PACKAGE_NAME} ${VERSION} (${ARCH}, ${dist})"
+        log_error "Upload failed after 6 attempts: ${SOURCE_PACKAGE_NAME} ${VERSION} (${ARCH}, ${dist})"
         exit 1
     fi
 done