FM / fsspec

/usr/local/lib/python3.6/site-packages/fsspec/

/usr/local/lib/python3.6/site-packages/fsspec

Name	Size	Mode	Actions
implementations/	-	0755	rm
__pycache__/	-	0755	rm
archive.py	2150	0644	edit dl rm
asyn.py	27386	0644	edit dl rm
caching.py	16731	0644	edit dl rm
callbacks.py	4827	0644	edit dl rm
compression.py	4884	0644	edit dl rm
config.py	2900	0644	edit dl rm
conftest.py	1183	0644	edit dl rm
core.py	22102	0644	edit dl rm
dircache.py	2684	0644	edit dl rm
exceptions.py	348	0644	edit dl rm
fuse.py	10048	0644	edit dl rm
gui.py	13766	0644	edit dl rm
mapping.py	7517	0644	edit dl rm
parquet.py	19530	0644	edit dl rm
registry.py	9054	0644	edit dl rm
spec.py	56033	0644	edit dl rm
transaction.py	2195	0644	edit dl rm
utils.py	16848	0644	edit dl rm
_version.py	501	0644	edit dl rm
__init__.py	1557	0644	edit dl rm

Edit: /usr/local/lib/python3.6/site-packages/fsspec/compression.py (4884B)

"""Helper functions for a standard streaming compression API"""
from bz2 import BZ2File
from zipfile import ZipFile

import fsspec.utils
from fsspec.spec import AbstractBufferedFile

def noop_file(file, mode, **kwargs):
    return file

# TODO: files should also be available as contexts
# should be functions of the form func(infile, mode=, **kwargs) -> file-like
compr = {None: noop_file}

def register_compression(name, callback, extensions, force=False):
    """Register an "inferable" file compression type.

Registers transparent file compression type for use with fsspec.open.
    Compression can be specified by name in open, or "infer"-ed for any files
    ending with the given extensions.

Args:
        name: (str) The compression type name. Eg. "gzip".
        callback: A callable of form (infile, mode, **kwargs) -> file-like.
            Accepts an input file-like object, the target mode and kwargs.
            Returns a wrapped file-like object.
        extensions: (str, Iterable[str]) A file extension, or list of file
            extensions for which to infer this compression scheme. Eg. "gz".
        force: (bool) Force re-registration of compression type or extensions.

Raises:
        ValueError: If name or extensions already registered, and not force.

"""
    if isinstance(extensions, str):
        extensions = [extensions]

# Validate registration
    if name in compr and not force:
        raise ValueError("Duplicate compression registration: %s" % name)

for ext in extensions:
        if ext in fsspec.utils.compressions and not force:
            raise ValueError(
                "Duplicate compression file extension: %s (%s)" % (ext, name)
            )

compr[name] = callback

for ext in extensions:
        fsspec.utils.compressions[ext] = name

def unzip(infile, mode="rb", filename=None, **kwargs):
    if "r" not in mode:
        filename = filename or "file"
        z = ZipFile(infile, mode="w", **kwargs)
        fo = z.open(filename, mode="w")
        fo.close = lambda closer=fo.close: closer() or z.close()
        return fo
    z = ZipFile(infile)
    if filename is None:
        filename = z.namelist()[0]
    return z.open(filename, mode="r", **kwargs)

register_compression("zip", unzip, "zip")
register_compression("bz2", BZ2File, "bz2")

try:  # pragma: no cover
    from isal import igzip

# igzip is meant to be used as a faster drop in replacement to gzip
    # so its api and functions are the same as the stdlib’s module. Except
    # where ISA-L does not support the same calls as zlib
    # (See https://python-isal.readthedocs.io/).

register_compression("gzip", igzip.IGzipFile, "gz")
except ImportError:
    from gzip import GzipFile

register_compression(
        "gzip", lambda f, **kwargs: GzipFile(fileobj=f, **kwargs), "gz"
    )

try:
    from lzma import LZMAFile

register_compression("lzma", LZMAFile, "xz")
    register_compression("xz", LZMAFile, "xz", force=True)
except ImportError:
    pass

try:
    import lzmaffi

register_compression("lzma", lzmaffi.LZMAFile, "xz", force=True)
    register_compression("xz", lzmaffi.LZMAFile, "xz", force=True)
except ImportError:
    pass

class SnappyFile(AbstractBufferedFile):
    def __init__(self, infile, mode, **kwargs):
        import snappy

super().__init__(
            fs=None, path="snappy", mode=mode.strip("b") + "b", size=999999999, **kwargs
        )
        self.infile = infile
        if "r" in mode:
            self.codec = snappy.StreamDecompressor()
        else:
            self.codec = snappy.StreamCompressor()

def _upload_chunk(self, final=False):
        self.buffer.seek(0)
        out = self.codec.add_chunk(self.buffer.read())
        self.infile.write(out)
        return True

def seek(self, loc, whence=0):
        raise NotImplementedError("SnappyFile is not seekable")

def seekable(self):
        return False

def _fetch_range(self, start, end):
        """Get the specified set of bytes from remote"""
        data = self.infile.read(end - start)
        return self.codec.decompress(data)

try:
    import snappy

snappy.compress
    # Snappy may use the .sz file extension, but this is not part of the
    # standard implementation.
    register_compression("snappy", SnappyFile, [])

except (ImportError, NameError):
    pass

try:
    import lz4.frame

register_compression("lz4", lz4.frame.open, "lz4")
except ImportError:
    pass

try:
    import zstandard as zstd

def zstandard_file(infile, mode="rb"):
        if "r" in mode:
            cctx = zstd.ZstdDecompressor()
            return cctx.stream_reader(infile)
        else:
            cctx = zstd.ZstdCompressor(level=10)
            return cctx.stream_writer(infile)

register_compression("zstd", zstandard_file, "zst")
except ImportError:
    pass

cmd: