|
|
|
|
|
|
|
|
from typing import Any, Optional, Union |
|
|
|
|
|
from huggingface_hub import HfFileSystem |
|
|
|
|
|
from . import config |
|
|
from .table import CastError |
|
|
from .utils.track import TrackedIterableFromGenerator, tracked_list, tracked_str |
|
|
|
|
|
|
|
|
class DatasetsError(Exception): |
|
|
"""Base class for exceptions in this library.""" |
|
|
|
|
|
|
|
|
class DefunctDatasetError(DatasetsError): |
|
|
"""The dataset has been defunct.""" |
|
|
|
|
|
|
|
|
class FileNotFoundDatasetsError(DatasetsError, FileNotFoundError): |
|
|
"""FileNotFoundError raised by this library.""" |
|
|
|
|
|
|
|
|
class DataFilesNotFoundError(FileNotFoundDatasetsError): |
|
|
"""No (supported) data files found.""" |
|
|
|
|
|
|
|
|
class DatasetNotFoundError(FileNotFoundDatasetsError): |
|
|
"""Dataset not found. |
|
|
|
|
|
Raised when trying to access: |
|
|
- a missing dataset, or |
|
|
- a private/gated dataset and the user is not authenticated. |
|
|
""" |
|
|
|
|
|
|
|
|
class DatasetBuildError(DatasetsError): |
|
|
pass |
|
|
|
|
|
|
|
|
class ManualDownloadError(DatasetBuildError): |
|
|
pass |
|
|
|
|
|
|
|
|
class FileFormatError(DatasetBuildError): |
|
|
pass |
|
|
|
|
|
|
|
|
class DatasetGenerationError(DatasetBuildError): |
|
|
pass |
|
|
|
|
|
|
|
|
class DatasetGenerationCastError(DatasetGenerationError): |
|
|
@classmethod |
|
|
def from_cast_error( |
|
|
cls, |
|
|
cast_error: CastError, |
|
|
builder_name: str, |
|
|
gen_kwargs: dict[str, Any], |
|
|
token: Optional[Union[bool, str]], |
|
|
) -> "DatasetGenerationCastError": |
|
|
explanation_message = ( |
|
|
f"\n\nAll the data files must have the same columns, but at some point {cast_error.details()}" |
|
|
) |
|
|
formatted_tracked_gen_kwargs: list[str] = [] |
|
|
for gen_kwarg in gen_kwargs.values(): |
|
|
if not isinstance(gen_kwarg, (tracked_str, tracked_list, TrackedIterableFromGenerator)): |
|
|
continue |
|
|
while ( |
|
|
isinstance(gen_kwarg, (tracked_list, TrackedIterableFromGenerator)) and gen_kwarg.last_item is not None |
|
|
): |
|
|
gen_kwarg = gen_kwarg.last_item |
|
|
if isinstance(gen_kwarg, tracked_str): |
|
|
gen_kwarg = gen_kwarg.get_origin() |
|
|
if isinstance(gen_kwarg, str) and gen_kwarg.startswith("hf://"): |
|
|
resolved_path = HfFileSystem(endpoint=config.HF_ENDPOINT, token=token).resolve_path(gen_kwarg) |
|
|
gen_kwarg = "hf://" + resolved_path.unresolve() |
|
|
if "@" + resolved_path.revision in gen_kwarg: |
|
|
gen_kwarg = ( |
|
|
gen_kwarg.replace("@" + resolved_path.revision, "", 1) |
|
|
+ f" (at revision {resolved_path.revision})" |
|
|
) |
|
|
formatted_tracked_gen_kwargs.append(str(gen_kwarg)) |
|
|
if formatted_tracked_gen_kwargs: |
|
|
explanation_message += f"\n\nThis happened while the {builder_name} dataset builder was generating data using\n\n{', '.join(formatted_tracked_gen_kwargs)}" |
|
|
help_message = "\n\nPlease either edit the data files to have matching columns, or separate them into different configurations (see docs at https://hf.co/docs/hub/datasets-manual-configuration#multiple-configurations)" |
|
|
return cls("An error occurred while generating the dataset" + explanation_message + help_message) |
|
|
|
|
|
|
|
|
class ChecksumVerificationError(DatasetsError): |
|
|
"""Error raised during checksums verifications of downloaded files.""" |
|
|
|
|
|
|
|
|
class UnexpectedDownloadedFileError(ChecksumVerificationError): |
|
|
"""Some downloaded files were not expected.""" |
|
|
|
|
|
|
|
|
class ExpectedMoreDownloadedFilesError(ChecksumVerificationError): |
|
|
"""Some files were supposed to be downloaded but were not.""" |
|
|
|
|
|
|
|
|
class NonMatchingChecksumError(ChecksumVerificationError): |
|
|
"""The downloaded file checksum don't match the expected checksum.""" |
|
|
|
|
|
|
|
|
class SplitsVerificationError(DatasetsError): |
|
|
"""Error raised during splits verifications.""" |
|
|
|
|
|
|
|
|
class UnexpectedSplitsError(SplitsVerificationError): |
|
|
"""The expected splits of the downloaded file is missing.""" |
|
|
|
|
|
|
|
|
class ExpectedMoreSplitsError(SplitsVerificationError): |
|
|
"""Some recorded splits are missing.""" |
|
|
|
|
|
|
|
|
class NonMatchingSplitsSizesError(SplitsVerificationError): |
|
|
"""The splits sizes don't match the expected splits sizes.""" |
|
|
|
