diff --git a/src/datasets/builder.py b/src/datasets/builder.py index 604cb9cd208..36186c53abd 100644 --- a/src/datasets/builder.py +++ b/src/datasets/builder.py @@ -274,6 +274,9 @@ class DatasetBuilder: It defines the number of samples that are kept in memory before writing them and also the length of the arrow chunks. None means that the ArrowWriter will use its default value. + skip_origin_metadata (`bool`, *optional*): + Whether to skip fetching origin metadata (ETag/mtime) for the data files. + It should only be true when streaming is true. **config_kwargs (additional keyword arguments): Keyword arguments to be passed to the corresponding builder configuration class, set on the class attribute [`DatasetBuilder.BUILDER_CONFIG_CLASS`]. The builder configuration class is [`BuilderConfig`] or a subclass of it. @@ -313,6 +316,7 @@ def __init__( storage_options: Optional[dict] = None, writer_batch_size: Optional[int] = None, config_id: Optional[str] = None, + skip_origin_metadata: bool = False, **config_kwargs, ): # DatasetBuilder name @@ -330,6 +334,7 @@ def __init__( sanitize_patterns(data_files), base_path=base_path, download_config=DownloadConfig(token=token, storage_options=self.storage_options), + skip_origin_metadata=skip_origin_metadata, ) # Prepare config: DatasetConfig contains name, version and description but can be extended by each dataset diff --git a/src/datasets/data_files.py b/src/datasets/data_files.py index 3e63c3ac65f..4b00dbfa95d 100644 --- a/src/datasets/data_files.py +++ b/src/datasets/data_files.py @@ -581,10 +581,11 @@ def from_hf_repo( base_path: Optional[str] = None, allowed_extensions: Optional[list[str]] = None, download_config: Optional[DownloadConfig] = None, + skip_origin_metadata: bool = False, ) -> "DataFilesList": base_path = f"hf://datasets/{dataset_info.id}@{dataset_info.sha}/{base_path or ''}".rstrip("/") return cls.from_patterns( - patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config + patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config, skip_origin_metadata=skip_origin_metadata ) @classmethod @@ -594,10 +595,11 @@ def from_local_or_remote( base_path: Optional[str] = None, allowed_extensions: Optional[list[str]] = None, download_config: Optional[DownloadConfig] = None, + skip_origin_metadata: bool = False, ) -> "DataFilesList": base_path = base_path if base_path is not None else Path().resolve().as_posix() return cls.from_patterns( - patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config + patterns, base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config, skip_origin_metadata=skip_origin_metadata ) @classmethod @@ -607,6 +609,7 @@ def from_patterns( base_path: Optional[str] = None, allowed_extensions: Optional[list[str]] = None, download_config: Optional[DownloadConfig] = None, + skip_origin_metadata: bool = False, ) -> "DataFilesList": base_path = base_path if base_path is not None else Path().resolve().as_posix() data_files = [] @@ -623,7 +626,10 @@ def from_patterns( except FileNotFoundError: if not has_magic(pattern): raise - origin_metadata = _get_origin_metadata(data_files, download_config=download_config) + if not skip_origin_metadata: + origin_metadata = _get_origin_metadata(data_files, download_config=download_config) + else: + origin_metadata = [()] * len(data_files) return cls(data_files, origin_metadata) def filter( @@ -668,6 +674,7 @@ def from_local_or_remote( base_path: Optional[str] = None, allowed_extensions: Optional[list[str]] = None, download_config: Optional[DownloadConfig] = None, + skip_origin_metadata: bool = False, ) -> "DataFilesDict": out = cls() for key, patterns_for_key in patterns.items(): @@ -679,6 +686,7 @@ def from_local_or_remote( base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config, + skip_origin_metadata=skip_origin_metadata, ) ) return out @@ -691,6 +699,7 @@ def from_hf_repo( base_path: Optional[str] = None, allowed_extensions: Optional[list[str]] = None, download_config: Optional[DownloadConfig] = None, + skip_origin_metadata: bool = False, ) -> "DataFilesDict": out = cls() for key, patterns_for_key in patterns.items(): @@ -703,6 +712,7 @@ def from_hf_repo( base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config, + skip_origin_metadata=skip_origin_metadata, ) ) return out @@ -714,6 +724,7 @@ def from_patterns( base_path: Optional[str] = None, allowed_extensions: Optional[list[str]] = None, download_config: Optional[DownloadConfig] = None, + skip_origin_metadata: bool = False, ) -> "DataFilesDict": out = cls() for key, patterns_for_key in patterns.items(): @@ -725,6 +736,7 @@ def from_patterns( base_path=base_path, allowed_extensions=allowed_extensions, download_config=download_config, + skip_origin_metadata=skip_origin_metadata, ) ) return out @@ -766,6 +778,7 @@ def resolve( self, base_path: str, download_config: Optional[DownloadConfig] = None, + skip_origin_metadata: bool = False, ) -> "DataFilesList": base_path = base_path if base_path is not None else Path().resolve().as_posix() data_files = [] @@ -782,7 +795,10 @@ def resolve( except FileNotFoundError: if not has_magic(pattern): raise - origin_metadata = _get_origin_metadata(data_files, download_config=download_config) + if not skip_origin_metadata: + origin_metadata = _get_origin_metadata(data_files, download_config=download_config) + else: + origin_metadata = [()] * len(data_files) return DataFilesList(data_files, origin_metadata) def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsList": @@ -816,10 +832,11 @@ def resolve( self, base_path: str, download_config: Optional[DownloadConfig] = None, + skip_origin_metadata: bool = False, ) -> "DataFilesDict": out = DataFilesDict() for key, data_files_patterns_list in self.items(): - out[key] = data_files_patterns_list.resolve(base_path, download_config) + out[key] = data_files_patterns_list.resolve(base_path, download_config, skip_origin_metadata=skip_origin_metadata) return out def filter_extensions(self, extensions: list[str]) -> "DataFilesPatternsDict": diff --git a/src/datasets/load.py b/src/datasets/load.py index e938cb35e6d..cd3e1160328 100644 --- a/src/datasets/load.py +++ b/src/datasets/load.py @@ -1679,6 +1679,12 @@ def load_dataset( "To parallelize streaming, you can wrap the dataset with a PyTorch DataLoader using `num_workers` > 1 instead." ) + if not streaming and config_kwargs.get("skip_origin_metadata", False): + logger.warning( + "`skip_origin_metadata` is only supported when `streaming=True`. Forcing `skip_origin_metadata=False`." + ) + config_kwargs["skip_origin_metadata"] = False + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) verification_mode = VerificationMode( (verification_mode or VerificationMode.BASIC_CHECKS) if not save_infos else VerificationMode.ALL_CHECKS diff --git a/tests/test_data_files.py b/tests/test_data_files.py index 77c2974a6ca..776745aa6a2 100644 --- a/tests/test_data_files.py +++ b/tests/test_data_files.py @@ -400,6 +400,17 @@ def test_DataFilesList_from_patterns_raises_FileNotFoundError(complex_data_dir): DataFilesList.from_patterns(["file_that_doesnt_exist.txt"], complex_data_dir) +def test_DataFilesList_from_patterns_skip_origin_metadata(complex_data_dir, text_file): + data_files_list = DataFilesList.from_patterns([_TEST_URL, text_file.as_posix()], complex_data_dir, skip_origin_metadata=True) + assert list(data_files_list) == [_TEST_URL, text_file.as_posix()] + assert data_files_list.origin_metadata == [()] * 2 + + +def test_DataFilesList_from_patterns_skip_origin_metadata_raises_FileNotFoundError(complex_data_dir): + with pytest.raises(FileNotFoundError): + DataFilesList.from_patterns(["file_that_doesnt_exist.txt"], complex_data_dir, skip_origin_metadata=True) + + class TestDataFilesDict: def test_key_order_after_copy(self): data_files = DataFilesDict({"train": "train.csv", "test": "test.csv"}) diff --git a/tests/test_load.py b/tests/test_load.py index 051f94a7dcc..ca17f974fff 100644 --- a/tests/test_load.py +++ b/tests/test_load.py @@ -1125,6 +1125,11 @@ def test_load_streaming_private_dataset_with_zipped_data(hf_token, hf_private_da assert next(iter(ds)) is not None +def test_load_dataset_skip_origin_metadata_warning(caplog, csv_path): + load_dataset("csv", data_files={"train": csv_path}, skip_origin_metadata=True) + assert "`skip_origin_metadata` is only supported when `streaming=True`" in caplog.text + + @require_pil @pytest.mark.integration def test_load_dataset_config_kwargs_passed_as_arguments():