Text to Video Ref. Implementation

.github/workflows/build_wheels.yml

@@ -1,4 +1,5 @@
 name: Build loadgen wheels and release them into PYPI
+
 on:
   release:
     types: [published]
@@ -73,7 +74,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
+        os: [ubuntu-latest, windows-latest, macos-latest, macos-15]
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/test-resnet50.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.9" ]
+        python-version: [ "3.10", "3.13" ]
         backend: [ "onnxruntime", "tf" ]
         loadgen-flag: [ "", "--adr.loadgen.tags=_from-pip --pip_loadgen=yes" ]
 

.github/workflows/test-retinanet.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.10" ]
+        python-version: [ "3.10", "3.13" ]
         backend: [ "onnxruntime", "pytorch" ]
 
     steps:

.github/workflows/test-rgat.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.11" ]
+        python-version: [ "3.11", "3.12" ]
         backend: [ "pytorch" ]
         loadgen-flag: [ "" ]
 

.github/workflows/test-submission-checker.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.8", "3.11", "3.12" ]
+        python-version: [ "3.10", "3.12", "3.14" ]
 
     steps:
     - uses: actions/checkout@v3

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "language/deepseek-r1/submodules/LiveCodeBench"]
 	path = language/deepseek-r1/submodules/LiveCodeBench
 	url = https://github.com/LiveCodeBench/LiveCodeBench
+[submodule "text_to_video/wan2.2-t2v-14b/submodules/VBench"]
+	path = text_to_video/wan2.2-t2v-14b/submodules/VBench
+	url = https://github.com/Vchitect/VBench

graph/R-GAT/tools/download_igbh_full.sh

@@ -33,74 +33,93 @@ cd $1/full/processed
 mkdir paper
 cd paper
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper/node_feat.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper/node_feat.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper/node_label_19.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper/node_label_19.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper/node_label_2K.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper/node_label_2K.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper/paper_id_index_mapping.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper/paper_id_index_mapping.npy"; exit $?; }
 cd ..
 
 # paper__cites__paper
 mkdir paper__cites__paper
 cd paper__cites__paper
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper__cites__paper/edge_index.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper__cites__paper/edge_index.npy"; exit $?; }
 cd ..
 
 # author
 mkdir author
 cd author
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/author/author_id_index_mapping.npy
+test $? -eq 0 || { echo "❌ Failed to download: author/author_id_index_mapping.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/author/node_feat.npy
+test $? -eq 0 || { echo "❌ Failed to download: author/node_feat.npy"; exit $?; }
 cd ..
 
 # conference
 mkdir conference
 cd conference
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/conference/conference_id_index_mapping.npy
+test $? -eq 0 || { echo "❌ Failed to download: conference/conference_id_index_mapping.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/conference/node_feat.npy
+test $? -eq 0 || { echo "❌ Failed to download: conference/node_feat.npy"; exit $?; }
 cd ..
 
 # institute
 mkdir institute
 cd institute
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/institute/institute_id_index_mapping.npy
+test $? -eq 0 || { echo "❌ Failed to download: institute/institute_id_index_mapping.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/institute/node_feat.npy
+test $? -eq 0 || { echo "❌ Failed to download: institute/node_feat.npy"; exit $?; }
 cd ..
 
 # journal
 mkdir journal
 cd journal
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/journal/journal_id_index_mapping.npy
+test $? -eq 0 || { echo "❌ Failed to download: journal/journal_id_index_mapping.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/journal/node_feat.npy
+test $? -eq 0 || { echo "❌ Failed to download: journal/node_feat.npy"; exit $?; }
 cd ..
 
 # fos
 mkdir fos
 cd fos
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/fos/fos_id_index_mapping.npy
+test $? -eq 0 || { echo "❌ Failed to download: fos/fos_id_index_mapping.npy"; exit $?; }
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/fos/node_feat.npy
+test $? -eq 0 || { echo "❌ Failed to download: fos/node_feat.npy"; exit $?; }
 cd ..
 
 # author__affiliated_to__institute
 mkdir author__affiliated_to__institute
 cd author__affiliated_to__institute
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/author__affiliated_to__institute/edge_index.npy
+test $? -eq 0 || { echo "❌ Failed to download: author__affiliated_to__institute/edge_index.npy"; exit $?; }
 cd ..
 
 # paper__published__journal
 mkdir paper__published__journal
 cd paper__published__journal
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper__published__journal/edge_index.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper__published__journal/edge_index.npy"; exit $?; }
 cd ..
 
 # paper__topic__fos
 mkdir paper__topic__fos
 cd paper__topic__fos
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper__topic__fos/edge_index.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper__topic__fos/edge_index.npy"; exit $?; }
 cd ..
 
 # paper__venue__conference
 mkdir paper__venue__conference
 cd paper__venue__conference
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper__venue__conference/edge_index.npy
+test $? -eq 0 || { echo "❌ Failed to download: paper__venue__conference/edge_index.npy"; exit $?; }
 cd ..
 
 # paper__written_by__author
@@ -109,4 +128,4 @@ cd paper__written_by__author
 wget -c https://igb-public.s3.us-east-2.amazonaws.com/IGBH/processed/paper__written_by__author/edge_index.npy
 cd ..
 
-echo "IGBH-IGBH download complete"
+echo "IGBH-IGBH download complete"

language/deepseek-r1/README.md

@@ -23,6 +23,7 @@ bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/he
 ```
 
 To specify a custom download directory, use the `-d` flag:
+
 ```bash
 bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/heads/main/mlc-r2-downloader.sh) \
   -d /path/to/download/directory \
@@ -53,6 +54,7 @@ bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/he
 This will download the full preprocessed dataset file (`mlperf_deepseek_r1_dataset_4388_fp8_eval.pkl`) and the calibration dataset file (`mlperf_deepseek_r1_calibration_dataset_500_fp8_eval.pkl`).
 
 To specify a custom download directory, use the `-d` flag:
+
 ```bash
 bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/heads/main/mlc-r2-downloader.sh) \
   -d /path/to/download/directory \
@@ -103,6 +105,8 @@ Launch a Docker container with your preferred backend:
 - **vllm**: vLLM's LLM api-based inference
 - **sglang**: sglang's OpenAI endpoint-based inference
 
+**NOTE**: `sglang` backend uses `sglang==0.5.4` installed into `lmsysorg/sglang:v0.5.2-cu129-b200` base image.
+
 ## Backend-Specific Setup
 
 After launching any Docker container, run the setup script which automatically detects your backend:
@@ -115,6 +119,7 @@ setup.sh
 The setup script creates a virtual environment and configures it differently based on the backend:
 
 #### All Backends
+
 - Virtual environment is **activated** after `setup.sh`
 - Activate backend-specific venv using `source .venv_[pytorch|vllm|sglang]/bin/activate`
 - All commands are to be run using the virtual environment
@@ -159,6 +164,7 @@ The reference implementation includes full support for MLPerf inference benchmar
 ### Running MLPerf Benchmarks
 
 #### Offline Scenario
+
 ```bash
 (.venv_BACKEND) $ python run_mlperf.py \
     --mode offline \
@@ -167,13 +173,25 @@ The reference implementation includes full support for MLPerf inference benchmar
 ```
 
 #### Server Scenario
+
 ```bash
 (.venv_BACKEND) $ python run_mlperf.py \
     --mode server \
     --input-file <input_dataset>.pkl \
     --output-dir mlperf_results
 ```
 
+#### Interactive Scenario
+
+```bash
+(.venv_BACKEND) $ python run_mlperf.py \
+    --mode interactive \
+    --input-file <input_dataset>.pkl \
+    --output-dir mlperf_results
+```
+
+**NOTE:** to enable Speculative Decoding for Sglang Backend, toggle `BACKEND_REGISTRY['sglang']['enable_speculative_decode']` in `utils/backend_registry.py` (disabled by default).
+
 #### Pytorch Backend for Mlperf
 
 PyTorch backend uses distributed execution with `torchrun` and `run_mlperf_mpi.py`:
@@ -188,24 +206,36 @@ PyTorch backend uses distributed execution with `torchrun` and `run_mlperf_mpi.p
 
 ### MLPerf Command Line Options
 
-| Option         | Description                    | Default          |
-| -------------- | ------------------------------ | ---------------- |
-| `--mode`       | Scenario mode (offline/server) | `offline`        |
-| `--accuracy`   | Run accuracy test              | `False`          |
-| `--output-dir` | Output directory for results   | `mlperf_results` |
+| Option         | Description                                | Default          |
+| -------------- | ------------------------------------------ | ---------------- |
+| `--mode`       | Scenario mode (offline/server/interactive) | `offline`        |
+| `--accuracy`   | Run accuracy test                          | `False`          |
+| `--output-dir` | Output directory for results               | `mlperf_results` |
 
 ### Backend Support Matrix
 
 The following table shows which backends support different evaluation and MLPerf operations:
 
-| Backend     | `run_eval.py` | `run_mlperf.py --mode=offline` | `run_mlperf.py --mode=server` |
-| ----------- | ------------- | ------------------------------ | ----------------------------- |
-| pytorch-fp8 | x             | x                              |                               |
-| vllm-fp8    | x             | x                              |                               |
-| sglang-fp8  | x             | x                              | x                             |
+| Backend     | `run_eval.py` | `run_mlperf.py --mode=offline` | `run_mlperf.py --mode=server` | `run_mlperf.py --mode=interactive` |
+| ----------- | ------------- | ------------------------------ | ----------------------------- | ---------------------------------- |
+| pytorch-fp8 | x             | x                              |                               |                                    |
+| vllm-fp8    | x             | x                              |                               |                                    |
+| sglang-fp8  | x             | x                              | x                             | x                                  |
 
 > **Note**: For PyTorch backend, use the `_mpi` versions with `torchrun`. For vLLM and SGLang backends, use the single-process versions without `_mpi`.
 
+## Speculative Decoding
+
+For the DeepSeek-R1 Interactive Scenario, users can enable Speculative Decoding Optimization for the SGLANG Backend by setting the `enable_speculative_decode` flag to `True` in `language/deepseek-r1/utils/backend_registry.py`.
+
+When Enabled, SGLANG backend will run the allowed configuration as per [Inference Policies](https://github.com/mlcommons/inference_policies/blob/master/inference_rules.adoc) (appendix-speculative-decoding):
+
+| Benchmark   | Scenario    | Speculative Decoding Algorithm                             | Configuration                                           | MTP Head                                       |
+| :---------- | :---------- | :--------------------------------------------------------- | :------------------------------------------------------ | :--------------------------------------------- |
+| DeepSeek-r1 | Interactive | EAGLE-style decoding with deepseek-ai/deepseek-r1 MTP head | `speculative-num-steps=3`, `speculative-eagle-topk=1.0` | https://huggingface.co/deepseek-ai/DeepSeek-R1 |
+
+> Note: ONLY Sglang backend supports speculative-decoding
+
 ## Accuracy Evaluation
 
 **Using MLCFlow Automation**

language/deepseek-r1/backends/sglang_backend.py

@@ -84,6 +84,10 @@ def _setup_environment(self) -> None:
         # Use the utility function to get cache directory
         cache_base = get_cache_directory()
 
+        # Use models subdirectory to match user's example paths
+        self.cache_dir = cache_base.parent / 'models'
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
         # Set up HuggingFace cache environment variables
         setup_huggingface_cache()
 
@@ -121,7 +125,15 @@ def _build_server_command(self) -> List[str]:
         ]
 
         # Add optimization flags
-        if self.config['enable_torch_compile']:
+        if self.config['enable_speculative_decode']:
+            cmd.extend(['--speculative-algorithm',
+                        self.config['speculative_algorithm']])
+            cmd.extend(['--speculative-num-steps',
+                        str(self.config['speculative_num_steps'])])
+            cmd.extend(['--speculative-eagle-topk',
+                        str(self.config['speculative_topk'])])
+
+        elif self.config['enable_torch_compile']:
             cmd.append('--enable-torch-compile')
 
         if self.config['enable_flashinfer']:
@@ -134,7 +146,6 @@ def _build_server_command(self) -> List[str]:
 
         # Add performance settings
         cmd.extend([
-            '--cuda-graph-max-bs', str(self.config['cuda_graph_max_bs']),
             '--max-running-requests', str(self.config['max_running_requests'])
         ])
 
@@ -490,8 +501,7 @@ def generate(self,
 
             except Exception as e:
                 print(f"\nError generating completion: {e}")
-                raise RuntimeError(
-                    f"SGLang backend failed to generate tokens for prompt: {prompt[:100]}...")
+                results.append({'error': str(e)})
 
         return results
 
@@ -529,8 +539,7 @@ async def _async_generate_single(
 
             except Exception as e:
                 print(f"\nError generating completion for prompt {idx}: {e}")
-                raise RuntimeError(
-                    f"SGLang backend failed to generate tokens for prompt {idx}: {e}")
+                return idx, {'error': str(e)}
 
     @require_initialized
     def generate_async(self,

language/deepseek-r1/docker/Dockerfile.sglang

@@ -1,41 +1,17 @@
 # SGLang Backend Dockerfile
-FROM nvidia/cuda:12.6.0-devel-ubuntu22.04
+FROM lmsysorg/sglang:v0.5.2-cu129-b200
 
 # Set environment variables
 ENV DEBIAN_FRONTEND=noninteractive
 ENV PIP_BREAK_SYSTEM_PACKAGES=1
 ENV MLPERF_BACKEND=sglang
 
-# Install Python and system dependencies
-RUN apt-get update && apt-get install -y \
-    software-properties-common \
-    && add-apt-repository ppa:deadsnakes/ppa \
-    && apt-get update && apt-get install -y \
-    python3.10 \
-    python3.10-dev \
-    python3.10-distutils \
-    python3-pip \
-    git \
-    git-lfs \
-    curl \
-    wget \
-    ca-certificates \
-    cmake \
-    build-essential \
-    ninja-build \
-    pybind11-dev \
-    pkg-config \
-    sudo \
-    libnuma-dev \
-    htop \
-    && rm -rf /var/lib/apt/lists/*
-
-# Set Python 3.10 as default
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 && \
-    update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1
-
-# Install pip for Python 3.10
-RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
+# Make /root accessible to non-root users so they can access /root/.cargo and /root/.rustup
+# This is needed because flashinfer/tvm_ffi tries to search these paths
+RUN chmod a+rX /root && \
+    if [ -d /root/.cargo ]; then chmod -R a+rX /root/.cargo 2>/dev/null || true; fi && \
+    if [ -d /root/.rustup ]; then chmod -R a+rX /root/.rustup 2>/dev/null || true; fi && \
+    if [ -d /root/.cargo/bin ]; then cp -a /root/.cargo/bin/* /usr/local/bin/ 2>/dev/null || true; fi
 
 # Install UV package manager system-wide
 RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \