Name	Size	Mode	Actions
__pycache__/	-	0755	rm
abc.py	881	0644	editdlrm
csv.py	4739	0644	editdlrm
json.py	5729	0644	editdlrm
parquet.py	3723	0644	editdlrm
text.py	1665	0644	editdlrm
__init__.py	0	0644	editdlrm

from typing import Optional from .. import Features, NamedSplit from ..packaged_modules.text.text import Text from ..utils.typing import NestedDataStructureLike, PathLike from .abc import AbstractDatasetReader class TextDatasetReader(AbstractDatasetReader): def __init__( self, path_or_paths: NestedDataStructureLike[PathLike], split: Optional[NamedSplit] = None, features: Optional[Features] = None, cache_dir: str = None, keep_in_memory: bool = False, **kwargs, ): super().__init__( path_or_paths, split=split, features=features, cache_dir=cache_dir, keep_in_memory=keep_in_memory, **kwargs ) path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths} self.builder = Text( cache_dir=cache_dir, data_files=path_or_paths, features=features, **kwargs, ) def read(self): download_config = None download_mode = None ignore_verifications = False use_auth_token = None base_path = None self.builder.download_and_prepare( download_config=download_config, download_mode=download_mode, ignore_verifications=ignore_verifications, # try_from_hf_gcs=try_from_hf_gcs, base_path=base_path, use_auth_token=use_auth_token, ) # Build dataset for splits dataset = self.builder.as_dataset( split=self.split, ignore_verifications=ignore_verifications, in_memory=self.keep_in_memory ) return dataset