Add skip_origin_metadata flag to optimize streaming startup time

src/datasets/builder.py

@@ -274,6 +274,9 @@ class DatasetBuilder:
             It defines the number of samples that are kept in memory before writing them
             and also the length of the arrow chunks.
             None means that the ArrowWriter will use its default value.
+        skip_origin_metadata (`bool`, *optional*):
+            Whether to skip fetching origin metadata (ETag/mtime) for the data files.
+            It should only be true when streaming is true.
         **config_kwargs (additional keyword arguments): Keyword arguments to be passed to the corresponding builder
             configuration class, set on the class attribute [`DatasetBuilder.BUILDER_CONFIG_CLASS`]. The builder
             configuration class is [`BuilderConfig`] or a subclass of it.
@@ -313,6 +316,7 @@ def __init__(
         storage_options: Optional[dict] = None,
         writer_batch_size: Optional[int] = None,
         config_id: Optional[str] = None,
+        skip_origin_metadata: bool = False,
         **config_kwargs,
     ):
         # DatasetBuilder name
@@ -330,6 +334,7 @@ def __init__(
                 sanitize_patterns(data_files),
                 base_path=base_path,
                 download_config=DownloadConfig(token=token, storage_options=self.storage_options),
+                skip_origin_metadata=skip_origin_metadata,
             )
 
         # Prepare config: DatasetConfig contains name, version and description but can be extended by each dataset

src/datasets/data_files.py

@@ -581,10 +581,11 @@ def from_hf_repo(
         base_path: Optional[str] = None,
         allowed_extensions: Optional[list[str]] = None,
         download_config: Optional[DownloadConfig] = None,
+        skip_origin_metadata: bool = False,
     ) -> "DataFilesList":
         base_path = f"hf://datasets/{dataset_info.id}@{dataset_info.sha}/{base_path or ''}".rstrip("/")
         return cls.from_patterns(
-            patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config
+            patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config, skip_origin_metadata=skip_origin_metadata
         )
 
     @classmethod
@@ -594,10 +595,11 @@ def from_local_or_remote(
         base_path: Optional[str] = None,
         allowed_extensions: Optional[list[str]] = None,
         download_config: Optional[DownloadConfig] = None,
+        skip_origin_metadata: bool = False,
     ) -> "DataFilesList":
         base_path = base_path if base_path is not None else Path().resolve().as_posix()
         return cls.from_patterns(
-            patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config
+            patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config, skip_origin_metadata=skip_origin_metadata
         )
 
     @classmethod
@@ -607,6 +609,7 @@ def from_patterns(
         base_path: Optional[str] = None,
         allowed_extensions: Optional[list[str]] = None,
         download_config: Optional[DownloadConfig] = None,
+        skip_origin_metadata: bool = False,
     ) -> "DataFilesList":
         base_path = base_path if base_path is not None else Path().resolve().as_posix()
         data_files = []
@@ -623,7 +626,10 @@ def from_patterns(
             except FileNotFoundError:
                 if not has_magic(pattern):
                     raise
-        origin_metadata = _get_origin_metadata(data_files, download_config=download_config)
+        if not skip_origin_metadata:
+            origin_metadata = _get_origin_metadata(data_files, download_config=download_config)
+        else:
+            origin_metadata = [()] * len(data_files)
         return cls(data_files, origin_metadata)
 
     def filter(
@@ -668,6 +674,7 @@ def from_local_or_remote(
         base_path: Optional[str] = None,
         allowed_extensions: Optional[list[str]] = None,
         download_config: Optional[DownloadConfig] = None,
+        skip_origin_metadata: bool = False,
     ) -> "DataFilesDict":
         out = cls()
         for key, patterns_for_key in patterns.items():
@@ -679,6 +686,7 @@ def from_local_or_remote(
                     base_path=base_path,
                     allowed_extensions=allowed_extensions,
                     download_config=download_config,
+                    skip_origin_metadata=skip_origin_metadata,
                 )
             )
         return out
@@ -691,6 +699,7 @@ def from_hf_repo(
         base_path: Optional[str] = None,
         allowed_extensions: Optional[list[str]] = None,
         download_config: Optional[DownloadConfig] = None,
+        skip_origin_metadata: bool = False,
     ) -> "DataFilesDict":
         out = cls()
         for key, patterns_for_key in patterns.items():
@@ -703,6 +712,7 @@ def from_hf_repo(
                     base_path=base_path,
                     allowed_extensions=allowed_extensions,
                     download_config=download_config,
+                    skip_origin_metadata=skip_origin_metadata,
                 )
             )
         return out
@@ -714,6 +724,7 @@ def from_patterns(
         base_path: Optional[str] = None,
         allowed_extensions: Optional[list[str]] = None,
         download_config: Optional[DownloadConfig] = None,
+        skip_origin_metadata: bool = False,
     ) -> "DataFilesDict":
         out = cls()
         for key, patterns_for_key in patterns.items():
@@ -725,6 +736,7 @@ def from_patterns(
                     base_path=base_path,
                     allowed_extensions=allowed_extensions,
                     download_config=download_config,
+                    skip_origin_metadata=skip_origin_metadata,
                 )
             )
         return out
@@ -766,6 +778,7 @@ def resolve(
         self,
         base_path: str,
         download_config: Optional[DownloadConfig] = None,
+        skip_origin_metadata: bool = False,
     ) -> "DataFilesList":
         base_path = base_path if base_path is not None else Path().resolve().as_posix()
         data_files = []
@@ -782,7 +795,10 @@ def resolve(
             except FileNotFoundError:
                 if not has_magic(pattern):
                     raise
-        origin_metadata = _get_origin_metadata(data_files, download_config=download_config)
+        if not skip_origin_metadata:
+            origin_metadata = _get_origin_metadata(data_files, download_config=download_config)
+        else:
+            origin_metadata = [()] * len(data_files)
         return DataFilesList(data_files, origin_metadata)
 
     def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsList":
@@ -816,10 +832,11 @@ def resolve(
         self,
         base_path: str,
         download_config: Optional[DownloadConfig] = None,
+        skip_origin_metadata: bool = False,
     ) -> "DataFilesDict":
         out = DataFilesDict()
         for key, data_files_patterns_list in self.items():
-            out[key] = data_files_patterns_list.resolve(base_path, download_config)
+            out[key] = data_files_patterns_list.resolve(base_path, download_config, skip_origin_metadata=skip_origin_metadata)
         return out
 
     def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsDict":

src/datasets/load.py

@@ -1679,6 +1679,12 @@ def load_dataset(
             "To parallelize streaming, you can wrap the dataset with a PyTorch DataLoader using `num_workers` > 1 instead."
         )
 
+    if not streaming and config_kwargs.get("skip_origin_metadata", False):
+        logger.warning(
+            "`skip_origin_metadata` is only supported when `streaming=True`. Forcing `skip_origin_metadata=False`."
+        )
+        config_kwargs["skip_origin_metadata"] = False
+
     download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS)
     verification_mode = VerificationMode(
         (verification_mode or VerificationMode.BASIC_CHECKS) if not save_infos else VerificationMode.ALL_CHECKS

tests/test_data_files.py

@@ -400,6 +400,17 @@ def test_DataFilesList_from_patterns_raises_FileNotFoundError(complex_data_dir):
         DataFilesList.from_patterns(["file_that_doesnt_exist.txt"], complex_data_dir)
 
 
+def test_DataFilesList_from_patterns_skip_origin_metadata(complex_data_dir, text_file):
+    data_files_list = DataFilesList.from_patterns([_TEST_URL, text_file.as_posix()], complex_data_dir, skip_origin_metadata=True)
+    assert list(data_files_list) == [_TEST_URL, text_file.as_posix()]
+    assert data_files_list.origin_metadata == [()] * 2
+
+
+def test_DataFilesList_from_patterns_skip_origin_metadata_raises_FileNotFoundError(complex_data_dir):
+    with pytest.raises(FileNotFoundError):
+        DataFilesList.from_patterns(["file_that_doesnt_exist.txt"], complex_data_dir, skip_origin_metadata=True)
+
+
 class TestDataFilesDict:
     def test_key_order_after_copy(self):
         data_files = DataFilesDict({"train": "train.csv", "test": "test.csv"})

tests/test_load.py

@@ -1125,6 +1125,11 @@ def test_load_streaming_private_dataset_with_zipped_data(hf_token, hf_private_da
     assert next(iter(ds)) is not None
 
 
+def test_load_dataset_skip_origin_metadata_warning(caplog, csv_path):
+    load_dataset("csv", data_files={"train": csv_path}, skip_origin_metadata=True)
+    assert "`skip_origin_metadata` is only supported when `streaming=True`" in caplog.text
+
+
 @require_pil
 @pytest.mark.integration
 def test_load_dataset_config_kwargs_passed_as_arguments():