Skip to content

Commit b09cbbd

Browse files
committed
fix(auto): skip blank lines when reading wav.scp and jsonl file lists
A blank or whitespace-only line in a file list crashes prepare_data_iterator: the id/data branch indexes lines[0] on an empty split (IndexError), and the .jsonl branch feeds an empty string to json.loads (JSONDecodeError). Skip blank lines so a trailing or separator line no longer aborts the whole batch.
1 parent 66d7a4c commit b09cbbd

2 files changed

Lines changed: 21 additions & 0 deletions

File tree

‎funasr/auto/auto_model.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,8 @@ def prepare_data_iterator(
469469
if file_extension in filelist: # filelist: wav.scp, file.jsonl;text.txt;
470470
with open(data_in, encoding="utf-8") as fin:
471471
for line in fin:
472+
if not line.strip():
473+
continue
472474
key = "rand_key_" + "".join(random.choice(chars) for _ in range(13))
473475
if data_in.endswith(".jsonl"): # file.jsonl: json.dumps({"source": data})
474476
lines = json.loads(line.strip())

‎tests/test_pcm_input_format.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,25 @@ def test_default_batch_behavior_is_unchanged():
9797
assert values is data
9898

9999

100+
def test_scp_filelist_skips_blank_lines(tmp_path):
101+
list_path = tmp_path / "wav.scp"
102+
list_path.write_text("utt1 /a.wav\n\n \t \nutt2 /b.wav\n", encoding="utf-8")
103+
keys, values = prepare_data_iterator(str(list_path))
104+
assert keys == ["utt1", "utt2"]
105+
assert values == ["/a.wav", "/b.wav"]
106+
107+
108+
def test_jsonl_filelist_skips_blank_lines(tmp_path):
109+
list_path = tmp_path / "data.jsonl"
110+
list_path.write_text(
111+
'{"key": "k1", "source": "/a.wav"}\n\n \t \n{"key": "k2", "source": "/b.wav"}\n',
112+
encoding="utf-8",
113+
)
114+
keys, values = prepare_data_iterator(str(list_path))
115+
assert keys == ["k1", "k2"]
116+
assert values == ["/a.wav", "/b.wav"]
117+
118+
100119
class _RecordingModel(torch.nn.Module):
101120
def __init__(self, vad=False):
102121
super().__init__()

0 commit comments

Comments
 (0)