/usr/local/lib/python3.6/site-packages/datasets/utils/__pycache__
Edit: /usr/local/lib/python3.6/site-packages/datasets/utils/__pycache__/extract.cpython-36.pyc (10868B)
3
<%Eg* @ sZ d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm Z m
Z
d dl m Z m
Z
mZ ddlmZ ddlmZ e rd dlZG dd d ZG d
d d e ZG d d
d
ee ZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZ G dd deZ G d d d eZ G d d d eZ G d d! d!Z dS )" N)ABCabstractmethod)
TYPE_CHECKINGOptionalUnion )config )FileLockc @ sV e Zd Zdee dddZeedddZeeedd d
Zdeeed d
dZ dS )ExtractManagerN) cache_dirc C s& |rt jj|tjntj| _t| _d S )N) ospathjoinr ZEXTRACTED_DATASETS_DIRZEXTRACTED_DATASETS_PATH extract_dir Extractor extractor)selfr r @/usr/local/lib/python3.6/site-packages/datasets/utils/extract.py__init__ s zExtractManager.__init__)r returnc C s, ddl m} tjj|}tjj| j||S )Nr )hash_url_to_filename)Z
file_utilsr r
r abspathr r )r r r abs_pathr r r _get_output_path s z ExtractManager._get_output_path) output_path
force_extractr c C s* |p(t jj| o(t jj|o&t j| S )N)r
r isfileisdirlistdir)r r r r r r _do_extract# s zExtractManager._do_extractF)
input_pathr r c C s> | j j|}|s|S | j|}| j||r:| j j||| |S )N)r infer_extractor_formatr r! extract)r r" r extractor_formatr r r r r$ ( s
zExtractManager.extract)N)F)
__name__
__module__ __qualname__r strr r boolr! r$ r r r r r s r c @ sV e Zd Zeeedef edddZe eedef edef ddddZ
dS )
BaseExtractorz pathlib.Path)r r c K s d S )Nr )clsr kwargsr r r is_extractable3 s z BaseExtractor.is_extractableN)r" r r c C s d S )Nr )r" r r r r r$ 8 s zBaseExtractor.extract) r& r' r( classmethodr r r) r* r. staticmethodr$ r r r r r+ 2 s r+ c @ sL e Zd ZdZeedef edddZe d
edef e
e dddZ d S ) MagicNumberBaseExtractor z pathlib.Path)r magic_number_lengthc C s t | d }|j|S Q R X d S )Nrb)openread)r r3 fr r r read_magic_numberA s z*MagicNumberBaseExtractor.read_magic_number)r magic_numberr c C s< |s0y| j |t| j}W n tk
r. dS X |j| jS )NF)r8 lenr9 OSError
startswith)r, r r9 r r r r. F s z'MagicNumberBaseExtractor.is_extractableN)r2 )
r& r' r( r9 r0 r r) intr8 r/ bytesr* r. r r r r r1 > s
r1 c @ sN e Zd Zeedef edddZeedef edef ddddZ dS ) TarExtractorz pathlib.Path)r r c K s
t j|S )N)tarfile
is_tarfile)r, r r- r r r r. Q s zTarExtractor.is_extractableN)r" r r c C s. t j|dd tj| }|j| |j d S )NT)exist_ok)r
makedirsr@ r5
extractallclose)r" r Ztar_filer r r r$ U s
zTarExtractor.extract)
r& r' r( r/ r r) r* r. r0 r$ r r r r r? P s r? c @ s6 e Zd ZdZeedef edef ddddZdS )
GzipExtractors z pathlib.PathN)r" r r c C s> t j| d(}t|d}tj|| W d Q R X W d Q R X d S )Nr4 wb)gzipr5 shutil copyfileobj)r" r Z gzip_fileextracted_filer r r r$ ` s zGzipExtractor.extract)r& r' r( r9 r0 r r) r$ r r r r rF ] s rF c @ sN e Zd Zeedef edddZeedef edef ddddZ dS ) ZipExtractorz pathlib.Path)r r c K s
t j|S )N)zipfile
is_zipfile)r, r r- r r r r. h s zZipExtractor.is_extractableN)r" r r c C s< t j|dd tj| d}|j| |j W d Q R X d S )NT)rB r)r
rC rM ZipFilerD rE )r" r zip_filer r r r$ l s
zZipExtractor.extract)
r& r' r( r/ r r) r* r. r0 r$ r r r r rL g s rL c @ s6 e Zd ZdZeedef edef ddddZdS ) XzExtractors 7zXZ z pathlib.PathN)r" r r c C s< t j| (}t|d}tj|| W d Q R X W d Q R X d S )NrG )lzmar5 rI rJ )r" r compressed_filerK r r r r$ w s zXzExtractor.extract)r& r' r( r9 r0 r r) r$ r r r r rR t s rR c @ sV e Zd ZdZdZeedef edddZ e
edef edef ddd d
Z dS ) RarExtractors Rar! s Rar! z pathlib.Path)r r c
K s< t |d}|jt| j}W dQ R X || jkp:|j| jS )z9https://github.com/markokr/rarfile/blob/master/rarfile.pyr4 N)r5 r6 r: RAR5_IDr< RAR_ID)r, r r- r7 r9 r r r r. s zRarExtractor.is_extractableN)r" r r c C sD t jstddd l}tj|dd |j| }|j| |j d S )NzPlease pip install rarfiler T)rB ) r ZRARFILE_AVAILABLEr; rarfiler
rC ZRarFilerD rE )r" r rX rfr r r r$ s
zRarExtractor.extract) r& r' r( rW rV r/ r r) r* r. r0 r$ r r r r rU ~ s rU c @ s6 e Zd ZdZeedef edef ddddZdS )
ZstdExtractors (/z pathlib.PathN)r" r r c C sZ t jstddd l}|j }t| d(}t|d}|j|| W d Q R X W d Q R X d S )Nz Please pip install zstandardr r4 rG )r ZZSTANDARD_AVAILABLEr; Z zstandardZZstdDecompressorr5 copy_stream)r" r zstdZdctxZifhZofhr r r r$ s zZstdExtractor.extract)r& r' r( r9 r0 r r) r$ r r r r rZ s rZ c @ s6 e Zd ZdZeedef edef ddddZdS )Bzip2Extractors BZhz pathlib.PathN)r" r r c C s> t j| d(}t|d}tj|| W d Q R X W d Q R X d S )Nr4 rG )bz2r5 rI rJ )r" r rT rK r r r r$ s zBzip2Extractor.extract)r& r' r( r9 r0 r r) r$ r r r r r] s r] c @ s6 e Zd ZdZeedef edef ddddZdS )SevenZipExtractors 7z' z pathlib.PathN)r" r r c C sJ t jstddd l}tj|dd |j| d}|j| W d Q R X d S )NzPlease pip install py7zrr T)rB rO )r ZPY7ZR_AVAILABLEr; py7zrr
rC Z SevenZipFilerD )r" r r` archiver r r r$ s zSevenZipExtractor.extract)r& r' r( r9 r0 r r) r$ r r r r r_ s r_ c @ s6 e Zd ZdZeedef edef ddddZdS ) Lz4Extractors "Mz pathlib.PathN)r" r r c C sV t jstddd l}|jj| d(}t|d}tj|| W d Q R X W d Q R X d S )NzPlease pip install lz4r r4 rG )r Z
LZ4_AVAILABLEr; Z lz4.frameframer5 rI rJ )r" r lz4rT rK r r r r$ s zLz4Extractor.extract)r& r' r( r9 r0 r r) r$ r r r r rb s rb c
@ s e Zd Zeeeeeee e
e d Z e
dd Zeedef edddZe
dedef eed d
d Ze
edef ed d
dZe
dedef edef ee ee ddddZdS )r ) tarrH zipxzZrarr\ r^ Z7zrd c C s@ d}x6| j j D ](}t|drt|j}||kr4|n|}qW |S )Nr r9 )
extractorsvalueshasattrr: r9 )r, magic_number_max_lengthr r3 r r r _get_magic_number_max_length s
z&Extractor._get_magic_number_max_lengthz pathlib.Path)r r3 c C s( yt j| |dS tk
r" dS X d S )N)r3 r2 )r1 r8 r; )r r3 r r r _read_magic_number s z Extractor._read_magic_numberF)r return_extractorr c C s> t jdtd | j|}|r2|s$dS d| j| fS |s:dS dS )Nz{Method 'is_extractable' was deprecated in version 2.4.0 and will be removed in 3.0.0. Use 'infer_extractor_format' instead.)categoryTF)FN)warningswarn
FutureWarningr# rh )r, r rn r% r r r r. s
zExtractor.is_extractable)r r c C sB | j }| j||}x(| jj D ]\}}|j||dr |S q W d S )N)r9 )rl rm rh itemsr. )r, r rk r9 r% r r r r r# s
z Extractor.infer_extractor_formatN
deprecated)r" r r% r r c C s t |d }t| tj|dd tjtjj|dd |sF|dkr|dksZt|t rzt j
dt d |dkrt|n|}n
| j | }|j
||S t j
dt d x(| j j D ]}|j|r|j
||S qW W d Q R X d S ) Nz.lockT)
ignore_errors)rB rt zsParameter 'extractor' was deprecated in version 2.4.0 and will be removed in 3.0.0. Use 'extractor_format' instead.)ro ztParameter 'extractor_format' was made required in version 2.4.0 and not passing it will raise an exception in 3.0.0.)r) r
rI rmtreer
rC r dirname
isinstancerp rq rr rh r$ ri r. )r, r" r r% r Z lock_pathr r r r$ s$
zExtractor.extract)F)Nrt )r& r' r( r? rF rL rR rU rZ r] r_ rb rh r/ rl r0 r r) r= rm r* r. r# r r+ r$ r r r r r s( "r )!r^ rH rS r
rI r@ rp rM abcr r typingr r r r Zfilelockr
pathlibr r+ r1 r? rF rL rR rU rZ r] r_ rb r r r r r
s4