Qwen fewshot

configs/experiment/inference/predict_qwen.yaml

@@ -0,0 +1,52 @@
+# @package _global_
+
+# Base config for Qwen/vLLM direct-prompt downstream task inference.
+# Uses the same few-shot infrastructure as predict_gemini (FewShotSupportBuilder).
+#
+# Set data and prompt when calling:
+#   python src/main.py experiment=inference/predict_qwen data=cmul2arcticl1 prompt=l1cls_cmul2arctic task_name=dp_qwen_l1cls_cmul2arctic
+#
+# Few-shot: inference.inference_runner.prompting.mode=few_shot
+
+defaults:
+  - /prompt: ???  # REQUIRED: specify prompt config (e.g., l1cls_cmul2arctic)
+  - override /logger: csv
+  - override /model: null
+  - override /model/net: null
+
+task_name: ???
+tags: ["qwen", "inference", "zeroshot"]
+
+seed: 42
+
+train: false
+test: false
+distributed_predict: True
+
+data:
+  predict_splits: ["test"]
+
+inference:
+  num_workers: 1
+  port: 8000  # vLLM server port; override if needed
+  passthrough_keys: ["target", "split", "utt_id", "metadata_idx"]
+  out_file: ${paths.output_dir}/prediction.jsonl
+  limit_samples: null
+
+  inference_runner:
+    _target_: src.model.qwen.predict.QwenDirectPromptInference
+    prompting:
+      mode: zero_shot  # zero_shot | few_shot
+      seed: ${seed}
+      support_data_cfg: ${data}
+    client_config:
+      base_url: "http://localhost:${inference.port}/v1"
+      model_name: "Qwen/Qwen3-Omni-30B-A3B-Instruct"
+      api_key: "EMPTY"
+      temperature: 0.0
+      max_tokens: 256
+    cache_key_field: "metadata_idx"
+    cache_path: "${paths.output_dir}/prediction.cache.jsonl"
+    error_log_path: "${paths.output_dir}/prediction.errors.jsonl"
+    resume: True
+    timeout: 600.0

configs/experiment/inference/predict_qweninstruct.yaml

@@ -0,0 +1 @@
+predict_qwen.yaml

requirements-dai.txt

@@ -0,0 +1,58 @@
+# --------- pytorch --------- #
+--extra-index-url https://download.pytorch.org/whl/cu126
+torch==2.6.0
+torchaudio==2.6.0
+torchvision==0.21.0
+lightning>=2.0.0
+torchmetrics>=0.11.4
+
+# --------- hydra --------- #
+hydra-core==1.3.2
+hydra-colorlog==1.2.0
+hydra-optuna-sweeper==1.2.0
+
+# --------- loggers --------- #
+# wandb
+# neptune-client
+# mlflow
+# comet-ml
+# aim>=3.16.2  # no lower than 3.16.2, see https://github.com/aimhubio/aim/issues/2550
+
+# --------- others --------- #
+rootutils       # standardizing the project root setup
+pre-commit      # hooks for applying linters on commit
+rich            # beautiful text formatting in terminal
+pytest          # tests
+# sh            # for running bash commands in some tests (linux/macos only)
+
+# espnet==202509
+git+https://github.com/Shikhar-S/espnet.git@flashattn
+
+huggingface_hub==0.35.3
+datasets==4.4.1
+pyarrow==22.0.0
+pandas
+transformers
+phonemizer==3.3.0
+flashlight-text==0.0.7
+pytest-mock==3.15.1
+git+https://github.com/changelinglab/phone-inventory-metric@v0.2.4
+
+# OPTIONAL
+wandb==0.22.1
+buckeye==1.3
+panphon==0.22.2
+rapidfuzz==3.14.3
+deepspeed==0.18.6
+
+# ZIPA
+lhotse==1.32.1
+
+# viz
+matplotlib
+seaborn
+
+#easycall
+ipatok==0.1.0
+epitran==1.34.0
+tensorboard

scripts/run.sh

@@ -23,7 +23,7 @@ Usage: $0 [OPTIONS]
 
 Options:
   --model LIST        Models: powsm, powsm_ctc, ctag, lv60, xlsr53, zipactc, zipactc_ns, or "all"
-  --recipe LIST       Recipes: geo_in, l1cls_cmu, l1cls_ed, l2as, lid_fl, atyp_ec, atyp_ua, atyp_us, inference, cascade_rnn_cls
+  --recipe LIST       Recipes: geo_in, l1cls_cmu, l1cls_ed, l2as, lid_fl, atyp_ec, atyp_ua, atyp_us, inference, predict, cascade_rnn_cls
   --data LIST         Datasets: timit, geo_in, edacc, cmul2arctic, speechocean, fleurs, or "all"
   --cluster NAME      Cluster: babel (default: babel)
   --fft               Enable full fine-tuning
@@ -45,6 +45,8 @@ EOF
 setup="probing"
 if [ "$recipe" = "inference" ]; then
     setup="inference"
+elif [ "$recipe" = "predict" ]; then
+    setup="predict"
 elif [[ "$recipe" == *cascade* ]]; then
     setup="cascade"
 fi
@@ -62,6 +64,7 @@ summary_log="${exp_dir}/${run_name}.summary.log"
 # Cluster configurations
 declare -A cluster_configs=(
     ["babel"]="scripts/babel.batch"
+    ["vllm"]="scripts/vllm_dai.batch"
 )
 
 # Model configurations: base_model|hf_repo
@@ -92,11 +95,24 @@ declare -A recipe_configs=(
     ["atyp_ua"]="atypical_uaspeech"
     ["atyp_us"]="atypical_ultrasuite"
     ["inference"]="transcribe"
+    ["predict"]="predict"
     ["cascade_rnn_cls"]="rnn_classification"
     ["cascade_rnn_reg"]="rnn_regression"
     ["cascade_rnn_geo"]="rnn_geolocation"
 )
 
+# Dataset -> recipe code for predict (only prompt-capable datasets; reuse recipe_configs for prompt name)
+declare -A data_to_recipe=(
+    ["cmul2arctic"]="l1cls_cmu"
+    ["edacc"]="l1cls_ed"
+    ["speechocean"]="l2as"
+    ["fleurs"]="lid_fl"
+    ["geo_in"]="geo_in"
+    ["easycall"]="atyp_ec"
+    ["uaspeech"]="atyp_ua"
+    ["ultrasuite"]="atyp_us"
+)
+
 # Dataset configurations: dataset|num_classes (num_classes is empty if not applicable)
 declare -A dataset_configs=(
     ["timit"]="timit|"
@@ -144,6 +160,10 @@ generate_list() {
 construct_config_name() {
     local model_var=$1 recipe_code=$2
     local base=$(get_base_model "$model_var")
+    if [ "$setup" = "predict" ]; then
+        echo "configs/experiment/inference/predict_${base}.yaml"
+        return
+    fi
     local recipe_full="${recipe_configs[$recipe_code]}"
     if [ "$setup" = "cascade" ]; then
         echo "configs/experiment/${setup}/${recipe_full}.yaml"
@@ -192,6 +212,29 @@ construct_cmd_for_inference() {
     printf '%s\n' "${cmds[@]}"
 }
 
+construct_cmd_for_predict() {
+    local model_var=$1 config_file=$2
+    shift 2
+    local datasets=("$@")
+    local script="${cluster_configs[$cluster]}"
+    local config_basename="${config_file##*/}"
+    local model_short="${config_basename%.yaml}"
+    model_short="${model_short#predict_}"
+    local cmds=()
+    for dataset_code in "${datasets[@]}"; do
+        [ -z "$dataset_code" ] && continue
+        [[ ! -v data_to_recipe[$dataset_code] ]] && continue
+        local recipe_code="${data_to_recipe[$dataset_code]}"
+        local prompt_name="${recipe_configs[$recipe_code]}"
+        local dataset_name="${dataset_configs[$dataset_code]%%|*}"
+        local task_name="dp_${model_short}_${prompt_name}"
+        local cmd="sbatch${sbatch_args:+ $sbatch_args} $script experiment=inference/${config_basename} data=$dataset_name prompt=$prompt_name task_name=$task_name"
+        [ -n "$extra_args" ] && cmd+=" $extra_args"
+        cmds+=("$cmd")
+    done
+    printf '%s\n' "${cmds[@]}"
+}
+
 _construct_adhoc_args_for_cascade() {
     local dataset_name=$1 model_var=$2
     # Pick the latest non-empty transcription.json under the run tree
@@ -265,6 +308,13 @@ run_experiment() {
                 return 1
             }
             ;;
+        predict)
+            cmd_list=$(construct_cmd_for_predict "$model_var" "$config_file" "${datasets[@]}") || {
+                log "Error constructing commands for $model_var on $recipe_code (config: $config_file)"
+                echo "ERROR: $model_var on $recipe_code" >> "$summary_log"
+                return 1
+            }
+            ;;
         cascade)
             cmd_list=$(construct_cmd_for_cascade "$model_var" "$config_file" "${datasets[@]}") || {
                 log "Error constructing commands for $model_var on $recipe_code (config: $config_file)"

scripts/run_few_shot.sh

@@ -0,0 +1,49 @@
+scripts/run.sh --recipe predict \
+	--cluster vllm \
+	--model qweni \
+	--data ultrasuite \
+	--extra_args "inference.num_workers=40 \
+	run_folder=5shot \
+	inference.inference_runner.prompting.mode=few_shot \
+	+prompt.prompt_config.few_shot.policy=k_sample \
+	+prompt.prompt_config.few_shot.k=5"
+
+# cmul2arctic,edacc,easycall,uaspeech,ultrasuite,fleurs,speechocean,geo_in
+
+scripts/run.sh --recipe predict \
+	--cluster vllm \
+	--model qweni \
+	--data ultrasuite \
+	--extra_args "inference.num_workers=40 run_folder=fewshot inference.inference_runner.prompting.mode=few_shot"
+
+python -m src.metrics.zeroshot_eval \
+    --dataset easycall \
+    --predictions "exp/runs/dp_qweninstruct_atypical_easycall/5shot/prediction.0.jsonl"
+
+python -m src.metrics.zeroshot_eval \
+  --dataset uaspeech \
+  --predictions "exp/runs/dp_qweninstruct_atypical_uaspeech/5shot/prediction.0.jsonl"
+
+python -m src.metrics.zeroshot_eval \
+  --dataset ultrasuite_child \
+  --predictions "exp/runs/dp_qweninstruct_atypical_ultrasuite/fewshot/prediction.0.jsonl"
+
+python -m src.metrics.zeroshot_eval \
+  --dataset vaanigeo \
+  --predictions "exp/runs/dp_qweninstruct_geolocation_vaani/5shot/prediction.0.jsonl"
+
+python -m src.metrics.zeroshot_eval \
+  --dataset cmul2arcticl1 \
+  --predictions "exp/runs/dp_qweninstruct_l1cls_cmul2arctic/5shot/prediction.0.jsonl"
+
+python -m src.metrics.zeroshot_eval \
+  --dataset edacc \
+  --predictions "exp/runs/dp_qweninstruct_l1cls_edacc/5shot/prediction.0.jsonl"
+
+python -m src.metrics.zeroshot_eval \
+  --dataset speechocean \
+  --predictions "exp/runs/dp_qweninstruct_l2as_speechocean/5shot/prediction.0.jsonl"
+
+python -m src.metrics.zeroshot_eval \
+  --dataset fleurs \
+  --predictions "exp/runs/dp_qweninstruct_lid_fleurs/5shot/prediction.0.jsonl"

scripts/start_vllm.sh

@@ -53,7 +53,7 @@ VLLM_EXECUTABLE=exp/download/vllm_arm.sif
 # TOTAL_LENGTH=8192
 # MODEL=nvidia/audio-flamingo-3-hf
 MODEL=Qwen/Qwen3-Omni-30B-A3B-Instruct
-TOTAL_LENGTH=4096
+TOTAL_LENGTH=8192
 ###########################
 ###########################
 apptainer exec --cleanenv --nv \