FM / datasets

/usr/local/lib/python3.6/site-packages/datasets/

/usr/local/lib/python3.6/site-packages/datasets

Name	Size	Mode	Actions
commands/	-	0755	rm
download/	-	0755	rm
features/	-	0755	rm
filesystems/	-	0755	rm
formatting/	-	0755	rm
io/	-	0755	rm
packaged_modules/	-	0755	rm
tasks/	-	0755	rm
utils/	-	0755	rm
__pycache__/	-	0755	rm
arrow_dataset.py	234990	0644	edit dl rm
arrow_reader.py	25598	0644	edit dl rm
arrow_writer.py	29671	0644	edit dl rm
builder.py	70487	0644	edit dl rm
combine.py	6067	0644	edit dl rm
config.py	7540	0644	edit dl rm
dataset_dict.py	83882	0644	edit dl rm
data_files.py	32741	0644	edit dl rm
fingerprint.py	22293	0644	edit dl rm
info.py	17700	0644	edit dl rm
inspect.py	20683	0644	edit dl rm
iterable_dataset.py	66078	0644	edit dl rm
keyhash.py	3826	0644	edit dl rm
load.py	86750	0644	edit dl rm
metric.py	27568	0644	edit dl rm
naming.py	2877	0644	edit dl rm
py.typed	0	0644	edit dl rm
search.py	33397	0644	edit dl rm
splits.py	22107	0644	edit dl rm
streaming.py	4824	0644	edit dl rm
table.py	82372	0644	edit dl rm
__init__.py	3065	0644	edit dl rm

Edit: /usr/local/lib/python3.6/site-packages/datasets/naming.py (2877B)

# Lint as: python3
"""Utilities for file names."""

import itertools
import os
import re

_uppercase_uppercase_re = re.compile(r"([A-Z]+)([A-Z][a-z])")
_lowercase_uppercase_re = re.compile(r"([a-z\d])([A-Z])")

_single_underscore_re = re.compile(r"(?<!_)_(?!_)")
_multiple_underscores_re = re.compile(r"(_{2,})")

_split_re = r"^\w+(\.\w+)*$"

INVALID_WINDOWS_CHARACTERS_IN_PATH = r"<>:/\|?*"

def camelcase_to_snakecase(name):
    """Convert camel-case string to snake-case."""
    name = _uppercase_uppercase_re.sub(r"\1_\2", name)
    name = _lowercase_uppercase_re.sub(r"\1_\2", name)
    return name.lower()

def snakecase_to_camelcase(name):
    """Convert snake-case string to camel-case string."""
    name = _single_underscore_re.split(name)
    name = [_multiple_underscores_re.split(n) for n in name]
    return "".join(n.capitalize() for n in itertools.chain.from_iterable(name) if n != "")

def filename_prefix_for_name(name):
    if os.path.basename(name) != name:
        raise ValueError(f"Should be a dataset name, not a path: {name}")
    return camelcase_to_snakecase(name)

def filename_prefix_for_split(name, split):
    if os.path.basename(name) != name:
        raise ValueError(f"Should be a dataset name, not a path: {name}")
    if not re.match(_split_re, split):
        raise ValueError(f"Split name should match '{_split_re}'' but got '{split}'.")
    return f"{filename_prefix_for_name(name)}-{split}"

def filepattern_for_dataset_split(dataset_name, split, data_dir, filetype_suffix=None):
    prefix = filename_prefix_for_split(dataset_name, split)
    if filetype_suffix:
        prefix += f".{filetype_suffix}"
    filepath = os.path.join(data_dir, prefix)
    return f"{filepath}*"

def filename_for_dataset_split(dataset_name, split, filetype_suffix=None):
    prefix = filename_prefix_for_split(dataset_name, split)
    if filetype_suffix:
        prefix += f".{filetype_suffix}"
    return prefix

def filepath_for_dataset_split(dataset_name, split, data_dir, filetype_suffix=None):
    filename = filename_for_dataset_split(
        dataset_name=dataset_name,
        split=split,
        filetype_suffix=filetype_suffix,
    )
    filepath = os.path.join(data_dir, filename)
    return filepath

# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Lint as: python3 """Utilities for file names.""" import itertools import os import re _uppercase_uppercase_re = re.compile(r"([A-Z]+)([A-Z][a-z])") _lowercase_uppercase_re = re.compile(r"([a-z\d])([A-Z])") _single_underscore_re = re.compile(r"(?:/\|?*" def camelcase_to_snakecase(name): """Convert camel-case string to snake-case.""" name = _uppercase_uppercase_re.sub(r"\1_\2", name) name = _lowercase_uppercase_re.sub(r"\1_\2", name) return name.lower() def snakecase_to_camelcase(name): """Convert snake-case string to camel-case string.""" name = _single_underscore_re.split(name) name = [_multiple_underscores_re.split(n) for n in name] return "".join(n.capitalize() for n in itertools.chain.from_iterable(name) if n != "") def filename_prefix_for_name(name): if os.path.basename(name) != name: raise ValueError(f"Should be a dataset name, not a path: {name}") return camelcase_to_snakecase(name) def filename_prefix_for_split(name, split): if os.path.basename(name) != name: raise ValueError(f"Should be a dataset name, not a path: {name}") if not re.match(_split_re, split): raise ValueError(f"Split name should match '{_split_re}'' but got '{split}'.") return f"{filename_prefix_for_name(name)}-{split}" def filepattern_for_dataset_split(dataset_name, split, data_dir, filetype_suffix=None): prefix = filename_prefix_for_split(dataset_name, split) if filetype_suffix: prefix += f".{filetype_suffix}" filepath = os.path.join(data_dir, prefix) return f"{filepath}*" def filename_for_dataset_split(dataset_name, split, filetype_suffix=None): prefix = filename_prefix_for_split(dataset_name, split) if filetype_suffix: prefix += f".{filetype_suffix}" return prefix def filepath_for_dataset_split(dataset_name, split, data_dir, filetype_suffix=None): filename = filename_for_dataset_split( dataset_name=dataset_name, split=split, filetype_suffix=filetype_suffix, ) filepath = os.path.join(data_dir, filename) return filepath