FM / implementations

/usr/local/lib/python3.6/site-packages/fsspec/implementations/

/usr/local/lib/python3.6/site-packages/fsspec/implementations

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
arrow.py	6762	0644	edit dl rm
cached.py	27703	0644	edit dl rm
dask.py	4441	0644	edit dl rm
dbfs.py	14650	0644	edit dl rm
dvc.py	2156	0644	edit dl rm
ftp.py	10741	0644	edit dl rm
git.py	4034	0644	edit dl rm
github.py	7264	0644	edit dl rm
hdfs.py	7072	0644	edit dl rm
http.py	25774	0644	edit dl rm
jupyter.py	3816	0644	edit dl rm
libarchive.py	7073	0644	edit dl rm
local.py	11419	0644	edit dl rm
memory.py	8606	0644	edit dl rm
reference.py	18448	0644	edit dl rm
sftp.py	5209	0644	edit dl rm
smb.py	10023	0644	edit dl rm
tar.py	5356	0644	edit dl rm
webhdfs.py	15086	0644	edit dl rm
zip.py	3201	0644	edit dl rm
__init__.py	0	0644	edit dl rm

Edit: /usr/local/lib/python3.6/site-packages/fsspec/implementations/dvc.py (2156B)

import os

import dvc.repo

from fsspec.implementations.local import LocalFileSystem
from fsspec.spec import AbstractFileSystem

lfs = LocalFileSystem()

class DVCFileSystem(AbstractFileSystem):
    """DVC backend (experimental)

Load data files that are versioned using the `Data Version Control`_ system

.. _Data Version Control: https://dvc.org/

This interface is incomplete and experimental.
    """

root_marker = ""

def __init__(self, path=None, **kwargs):
        """

Parameters
        ----------
        path: str (optional)
            Location of the repo to access; defaults to the current directory.
        """
        super().__init__(**kwargs)
        self.repo = dvc.repo.Repo(path)
        self.path = self.repo.find_root()

@classmethod
    def _strip_protocol(cls, path):
        return super()._strip_protocol(path).lstrip("/")

def ls(self, path, detail=False, **kwargs):
        path = self._strip_protocol(path)
        allfiles = self.repo.tree.walk(os.path.join(self.repo.root_dir, path))
        dirname, dirs, files = next(allfiles)
        out = [os.path.join(path, f) for f in dirs + files]
        details = []

for f in out:
            full = os.path.join(self.repo.root_dir, f)
            file_info = lfs.info(full)
            if lfs.isdir(full):
                details.append(file_info)
            else:
                try:
                    extra = self.repo.find_out_by_relpath(full).dumpd()
                except dvc.exceptions.OutputNotFoundError:
                    continue
                details.append(dict(**extra, **file_info))
            details[-1]["name"] = f
        if detail:
            return details
        return [d["name"] for d in details]

def ukey(self, path):
        return self.info(path)["md5"]

def _open(
        self,
        path,
        mode="rb",
        block_size=None,
        autocommit=True,
        cache_options=None,
        **kwargs,
    ):
        # returns a context file object (i.e., needs to be used with ``with``
        path = self._strip_protocol(path)
        return self.repo.open_by_relpath(path)

import os import dvc.repo from fsspec.implementations.local import LocalFileSystem from fsspec.spec import AbstractFileSystem lfs = LocalFileSystem() class DVCFileSystem(AbstractFileSystem): """DVC backend (experimental) Load data files that are versioned using the `Data Version Control`_ system .. _Data Version Control: https://dvc.org/ This interface is incomplete and experimental. """ root_marker = "" def __init__(self, path=None, **kwargs): """ Parameters ---------- path: str (optional) Location of the repo to access; defaults to the current directory. """ super().__init__(**kwargs) self.repo = dvc.repo.Repo(path) self.path = self.repo.find_root() @classmethod def _strip_protocol(cls, path): return super()._strip_protocol(path).lstrip("/") def ls(self, path, detail=False, **kwargs): path = self._strip_protocol(path) allfiles = self.repo.tree.walk(os.path.join(self.repo.root_dir, path)) dirname, dirs, files = next(allfiles) out = [os.path.join(path, f) for f in dirs + files] details = [] for f in out: full = os.path.join(self.repo.root_dir, f) file_info = lfs.info(full) if lfs.isdir(full): details.append(file_info) else: try: extra = self.repo.find_out_by_relpath(full).dumpd() except dvc.exceptions.OutputNotFoundError: continue details.append(dict(**extra, **file_info)) details[-1]["name"] = f if detail: return details return [d["name"] for d in details] def ukey(self, path): return self.info(path)["md5"] def _open( self, path, mode="rb", block_size=None, autocommit=True, cache_options=None, **kwargs, ): # returns a context file object (i.e., needs to be used with ``with`` path = self._strip_protocol(path) return self.repo.open_by_relpath(path)