FM / fsspec

/usr/local/lib/python3.6/site-packages/fsspec/

/usr/local/lib/python3.6/site-packages/fsspec

Name	Size	Mode	Actions
implementations/	-	0755	rm
__pycache__/	-	0755	rm
archive.py	2150	0644	edit dl rm
asyn.py	27386	0644	edit dl rm
caching.py	16731	0644	edit dl rm
callbacks.py	4827	0644	edit dl rm
compression.py	4884	0644	edit dl rm
config.py	2900	0644	edit dl rm
conftest.py	1183	0644	edit dl rm
core.py	22102	0644	edit dl rm
dircache.py	2684	0644	edit dl rm
exceptions.py	348	0644	edit dl rm
fuse.py	10048	0644	edit dl rm
gui.py	13766	0644	edit dl rm
mapping.py	7517	0644	edit dl rm
parquet.py	19530	0644	edit dl rm
registry.py	9054	0644	edit dl rm
spec.py	56033	0644	edit dl rm
transaction.py	2195	0644	edit dl rm
utils.py	16848	0644	edit dl rm
_version.py	501	0644	edit dl rm
__init__.py	1557	0644	edit dl rm

Edit: /usr/local/lib/python3.6/site-packages/fsspec/config.py (2900B)

import configparser
import json
import os

conf = {}
default_conf_dir = os.path.join(os.path.expanduser("~"), ".config/fsspec")
conf_dir = os.environ.get("FSSPEC_CONFIG_DIR", default_conf_dir)

def set_conf_env(conf_dict, envdict=os.environ):
    """Set config values from environment variables

Looks for variable of the form ``FSSPEC_<protocol>_<kwarg>``.
    There is no attempt to convert strings, but the kwarg keys will
    be lower-cased.

Parameters
    ----------
    conf_dict : dict(str, dict)
        This dict will be mutated
    envdict : dict-like(str, str)
        Source for the values - usually the real environment
    """
    for key in envdict:
        if key.startswith("FSSPEC"):
            if key.count("_") < 2:
                continue
            _, proto, kwarg = key.split("_", 2)
            conf_dict.setdefault(proto.lower(), {})[kwarg.lower()] = envdict[key]

def set_conf_files(cdir, conf_dict):
    """Set config values from files

Scans for INI and JSON files in the given dictionary, and uses their
    contents to set the config. In case of repeated values, later values
    win.

In the case of INI files, all values are strings, and these will not
    be converted.

Parameters
    ----------
    cdir : str
        Directory to search
    conf_dict : dict(str, dict)
        This dict will be mutated
    """
    if not os.path.isdir(cdir):
        return
    allfiles = sorted(os.listdir(cdir))
    for fn in allfiles:
        if fn.endswith(".ini"):
            ini = configparser.ConfigParser()
            ini.read(os.path.join(cdir, fn))
            for key in ini:
                if key == "DEFAULT":
                    continue
                conf_dict.setdefault(key, {}).update(dict(ini[key]))
        if fn.endswith(".json"):
            js = json.load(open(os.path.join(cdir, fn)))
            for key in js:
                conf_dict.setdefault(key, {}).update(dict(js[key]))

def apply_config(cls, kwargs, conf_dict=None):
    """Supply default values for kwargs when instantiating class

Augments the passed kwargs, by finding entries in the config dict
    which match the classes ``.protocol`` attribute (one or more str)

Parameters
    ----------
    cls : file system implementation
    kwargs : dict
    conf_dict : dict of dict
        Typically this is the global configuration

Returns
    -------
    dict : the modified set of kwargs
    """
    if conf_dict is None:
        conf_dict = conf
    protos = cls.protocol if isinstance(cls.protocol, (tuple, list)) else [cls.protocol]
    kw = {}
    for proto in protos:
        # default kwargs from the current state of the config
        if proto in conf_dict:
            kw.update(conf_dict[proto])
    # explicit kwargs always win
    kw.update(**kwargs)
    kwargs = kw
    return kwargs

set_conf_files(conf_dir, conf)
set_conf_env(conf)

import configparser import json import os conf = {} default_conf_dir = os.path.join(os.path.expanduser("~"), ".config/fsspec") conf_dir = os.environ.get("FSSPEC_CONFIG_DIR", default_conf_dir) def set_conf_env(conf_dict, envdict=os.environ): """Set config values from environment variables Looks for variable of the form ``FSSPEC__``. There is no attempt to convert strings, but the kwarg keys will be lower-cased. Parameters ---------- conf_dict : dict(str, dict) This dict will be mutated envdict : dict-like(str, str) Source for the values - usually the real environment """ for key in envdict: if key.startswith("FSSPEC"): if key.count("_") < 2: continue _, proto, kwarg = key.split("_", 2) conf_dict.setdefault(proto.lower(), {})[kwarg.lower()] = envdict[key] def set_conf_files(cdir, conf_dict): """Set config values from files Scans for INI and JSON files in the given dictionary, and uses their contents to set the config. In case of repeated values, later values win. In the case of INI files, all values are strings, and these will not be converted. Parameters ---------- cdir : str Directory to search conf_dict : dict(str, dict) This dict will be mutated """ if not os.path.isdir(cdir): return allfiles = sorted(os.listdir(cdir)) for fn in allfiles: if fn.endswith(".ini"): ini = configparser.ConfigParser() ini.read(os.path.join(cdir, fn)) for key in ini: if key == "DEFAULT": continue conf_dict.setdefault(key, {}).update(dict(ini[key])) if fn.endswith(".json"): js = json.load(open(os.path.join(cdir, fn))) for key in js: conf_dict.setdefault(key, {}).update(dict(js[key])) def apply_config(cls, kwargs, conf_dict=None): """Supply default values for kwargs when instantiating class Augments the passed kwargs, by finding entries in the config dict which match the classes ``.protocol`` attribute (one or more str) Parameters ---------- cls : file system implementation kwargs : dict conf_dict : dict of dict Typically this is the global configuration Returns ------- dict : the modified set of kwargs """ if conf_dict is None: conf_dict = conf protos = cls.protocol if isinstance(cls.protocol, (tuple, list)) else [cls.protocol] kw = {} for proto in protos: # default kwargs from the current state of the config if proto in conf_dict: kw.update(conf_dict[proto]) # explicit kwargs always win kw.update(**kwargs) kwargs = kw return kwargs set_conf_files(conf_dir, conf) set_conf_env(conf)