/usr/local/lib64/python3.6/site-packages/caffe2/python/__pycache__/
/usr/local/lib64/python3.6/site-packages/caffe2/python/__pycache__
NameSizeModeActions
allcompare_test.cpython-36.pyc27100644editdlrm
attention.cpython-36.pyc49380644editdlrm
benchmark_generator.cpython-36.pyc39100644editdlrm
binarysize.cpython-36.pyc48530644editdlrm
brew.cpython-36.pyc38490644editdlrm
brew_test.cpython-36.pyc104520644editdlrm
build.cpython-36.pyc3400644editdlrm
cached_reader.cpython-36.pyc29540644editdlrm
caffe_translator.cpython-36.pyc241650644editdlrm
caffe_translator_test.cpython-36.pyc29290644editdlrm
checkpoint.cpython-36.pyc279730644editdlrm
checkpoint_test.cpython-36.pyc94620644editdlrm
cnn.cpython-36.pyc84350644editdlrm
context.cpython-36.pyc40080644editdlrm
context_test.cpython-36.pyc26650644editdlrm
control.cpython-36.pyc144660644editdlrm
control_ops_grad.cpython-36.pyc163650644editdlrm
control_ops_grad_test.cpython-36.pyc15020644editdlrm
control_ops_util.cpython-36.pyc83320644editdlrm
control_test.cpython-36.pyc118700644editdlrm
convert.cpython-36.pyc1480644editdlrm
convert_test.cpython-36.pyc5680644editdlrm
convnet_benchmarks.cpython-36.pyc129450644editdlrm
convnet_benchmarks_test.cpython-36.pyc10900644editdlrm
core.cpython-36.pyc944280644editdlrm
core_gradients_test.cpython-36.pyc248250644editdlrm
core_test.cpython-36.pyc339420644editdlrm
crf.cpython-36.pyc76220644editdlrm
crf_predict.cpython-36.pyc10440644editdlrm
crf_viterbi_test.cpython-36.pyc16770644editdlrm
dataio.cpython-36.pyc239780644editdlrm
dataio_test.cpython-36.pyc132670644editdlrm
dataset.cpython-36.pyc128790644editdlrm
data_parallel_model.cpython-36.pyc514160644editdlrm
data_parallel_model_test.cpython-36.pyc389230644editdlrm
data_workers.cpython-36.pyc131420644editdlrm
data_workers_test.cpython-36.pyc45220644editdlrm
db_file_reader.cpython-36.pyc53610644editdlrm
db_test.cpython-36.pyc13280644editdlrm
device_checker.cpython-36.pyc40140644editdlrm
dyndep.cpython-36.pyc15680644editdlrm
embedding_generation_benchmark.cpython-36.pyc42030644editdlrm
experiment_util.cpython-36.pyc32330644editdlrm
extension_loader.cpython-36.pyc5430644editdlrm
fakefp16_transform_lib.cpython-36.pyc5460644editdlrm
filler_test.cpython-36.pyc9250644editdlrm
functional.cpython-36.pyc35310644editdlrm
functional_test.cpython-36.pyc41320644editdlrm
fused_8bit_rowwise_conversion_ops_test.cpython-36.pyc36310644editdlrm
gradient_checker.cpython-36.pyc106220644editdlrm
gradient_check_test.cpython-36.pyc167880644editdlrm
gru_cell.cpython-36.pyc25960644editdlrm
hip_test_util.cpython-36.pyc6690644editdlrm
hsm_util.cpython-36.pyc18230644editdlrm
hypothesis_test.cpython-36.pyc843120644editdlrm
hypothesis_test_util.cpython-36.pyc200390644editdlrm
ideep_test_util.cpython-36.pyc10810644editdlrm
layers_test.cpython-36.pyc573720644editdlrm
layer_model_helper.cpython-36.pyc216410644editdlrm
layer_model_instantiator.cpython-36.pyc36650644editdlrm
layer_parameter_sharing_test.cpython-36.pyc56260644editdlrm
layer_test_util.cpython-36.pyc52730644editdlrm
lazy.cpython-36.pyc4240644editdlrm
lazy_dyndep.cpython-36.pyc24750644editdlrm
lazy_dyndep_test.cpython-36.pyc48070644editdlrm
lengths_reducer_fused_8bit_rowwise_ops_test.cpython-36.pyc43920644editdlrm
lengths_reducer_rowwise_8bit_ops_test.cpython-36.pyc36840644editdlrm
lstm_benchmark.cpython-36.pyc72820644editdlrm
memonger.cpython-36.pyc321700644editdlrm
memonger_test.cpython-36.pyc231000644editdlrm
mkl_test_util.cpython-36.pyc11910644editdlrm
model_device_test.cpython-36.pyc33760644editdlrm
model_helper.cpython-36.pyc186180644editdlrm
model_helper_test.cpython-36.pyc19910644editdlrm
modifier_context.cpython-36.pyc26500644editdlrm
muji.cpython-36.pyc55660644editdlrm
muji_test.cpython-36.pyc35670644editdlrm
net_builder.cpython-36.pyc267860644editdlrm
net_builder_test.cpython-36.pyc92550644editdlrm
net_drawer.cpython-36.pyc100560644editdlrm
net_printer.cpython-36.pyc143670644editdlrm
net_printer_test.cpython-36.pyc33230644editdlrm
nomnigraph.cpython-36.pyc51250644editdlrm
nomnigraph_test.cpython-36.pyc145910644editdlrm
nomnigraph_transformations.cpython-36.pyc24310644editdlrm
nomnigraph_transformations_test.cpython-36.pyc40730644editdlrm
normalizer.cpython-36.pyc19410644editdlrm
normalizer_context.cpython-36.pyc15530644editdlrm
normalizer_test.cpython-36.pyc8700644editdlrm
numa_benchmark.cpython-36.pyc18860644editdlrm
numa_test.cpython-36.pyc16260644editdlrm
observer_test.cpython-36.pyc41810644editdlrm
operator_fp_exceptions_test.cpython-36.pyc13050644editdlrm
optimizer.cpython-36.pyc456060644editdlrm
optimizer_context.cpython-36.pyc20080644editdlrm
optimizer_test.cpython-36.pyc247470644editdlrm
optimizer_test_util.cpython-36.pyc68080644editdlrm
parallelize_bmuf_distributed_test.cpython-36.pyc69900644editdlrm
parallel_workers.cpython-36.pyc91480644editdlrm
parallel_workers_test.cpython-36.pyc38240644editdlrm
pipeline.cpython-36.pyc129400644editdlrm
pipeline_test.cpython-36.pyc27150644editdlrm
predictor_constants.cpython-36.pyc3040644editdlrm
python_op_test.cpython-36.pyc105020644editdlrm
queue_util.cpython-36.pyc50550644editdlrm
record_queue.cpython-36.pyc42250644editdlrm
recurrent.cpython-36.pyc98930644editdlrm
regularizer.cpython-36.pyc186040644editdlrm
regularizer_context.cpython-36.pyc15620644editdlrm
regularizer_test.cpython-36.pyc88250644editdlrm
rnn_cell.cpython-36.pyc447850644editdlrm
schema.cpython-36.pyc417630644editdlrm
schema_test.cpython-36.pyc137870644editdlrm
scope.cpython-36.pyc26000644editdlrm
scope_test.cpython-36.pyc40680644editdlrm
session.cpython-36.pyc73460644editdlrm
session_test.cpython-36.pyc23530644editdlrm
sparse_to_dense_mask_test.cpython-36.pyc52980644editdlrm
sparse_to_dense_test.cpython-36.pyc30030644editdlrm
task.cpython-36.pyc222070644editdlrm
task_test.cpython-36.pyc11780644editdlrm
test_util.cpython-36.pyc37290644editdlrm
text_file_reader.cpython-36.pyc22390644editdlrm
timeout_guard.cpython-36.pyc31060644editdlrm
toy_regression_test.cpython-36.pyc23600644editdlrm
transformations.cpython-36.pyc18330644editdlrm
transformations_test.cpython-36.pyc96080644editdlrm
tt_core.cpython-36.pyc64680644editdlrm
tt_core_test.cpython-36.pyc18240644editdlrm
utils.cpython-36.pyc122450644editdlrm
utils_test.cpython-36.pyc14160644editdlrm
visualize.cpython-36.pyc60270644editdlrm
workspace.cpython-36.pyc226900644editdlrm
workspace_test.cpython-36.pyc290550644editdlrm
_import_c_extension.cpython-36.pyc15240644editdlrm
__init__.cpython-36.pyc26070644editdlrm
Edit: /usr/local/lib64/python3.6/site-packages/caffe2/python/__pycache__/schema.cpython-36.pyc (41763B)
3 Eg5 @sdZddlZddlZddlmZddlmZddlmZddl m Z m Z ddl m Z ddlmZmZmZdd lmZdd lmZdd lmZejeZd Zd dZdVddZe dddddddgZdWej_ Gddde ddddgZ!dXe!j_ Gddde"Z#Gd d!d!e#Z$Gd"d#d#e$Z%Gd$d%d%e#Z&Gd&d'd'e#Z'dYd*d+Z(dZd,d-Z)d.d/Z*d0d1Z+d[d3d4Z,ffd5d6Z-Gd7d8d8e"Z.d\d9d:Z/d]dd?Z1d^d@dAZ2d_dBdCZ3dDdEZ4dFdGZ5d`dHdIZ6ej7ej8j9fej:ej8j;fejej8j?fej@ej8jAfejBej8jCfejDej8jEfejFej8jGfejHej8jIfejJej8jKfejLej8jMfg ZNdJdKZOdadLdMZPdbdNdOZQdPdQZRdRdSZSdTdUZTdS)ca Defines a minimal set of data types that allow to represent datasets with arbitrary nested structure, including objects of variable length, such as maps and lists. This defines a columnar storage format for such datasets on top of caffe2 tensors. In terms of capacity of representation, it can represent most of the data types supported by Parquet, ORC, DWRF file formats. See comments in operator_test/dataset_ops_test.py for an example and walkthrough on how to use schema to store and iterate through a structured in-memory dataset. N)core) workspace) BlobReference) OrderedDict namedtuple) basestring) viewitemsviewkeys viewvalues)islice)StringIO)Sequence:cCs.|r|rdj|t|S|r|S|r&|SdSdS)Nz{}{}{})formatFIELD_SEPARATOR)prefixsuffixr@/usr/local/lib64/python3.6/site-packages/caffe2/python/schema.py_join_field_name&srTcCs@t|tr|j|dSt|ttjfkr2t|dSt|dSdS)z:Clones/normalizes a field before adding it to a container.) keep_blobs)dtype)blobN) isinstanceFieldclonetypenprScalar)Zfield_or_type_or_blobrrrr_normalize_field1s    r FeatureSpecZ feature_typeZ feature_namesZ feature_idsZfeature_is_request_onlyZdesired_hash_sizeZfeature_to_indexc@seZdZUdZfZeedS)MetadataaSRepresents additional information associated with a scalar in schema. `categorical_limit` - for fields of integral type that are guaranteed to be non-negative it specifies the maximum possible value plus one. It's often used as a size of an embedding table. `expected_value` - anticipated average value of elements in the field. Usually makes sense for length fields of lists. `feature_specs` - information about the features that contained in this field. For example if field have more than 1 feature it can have list of feature names contained in this field.N)__name__ __module__ __qualname____doc__ __slots__r strrrrrr"Ks r"categorical_limitZexpected_valueZ feature_specsc@seZdZUdZd$ZeeddZddZdd Z d d Z d d Z ddZ ddZ ddZd%ddZddZddZd&ddZddZd d!Zd"d#ZdS)'rz4Represents an abstract field type in a dataset. _parent_field_offsetscCsJd|_d}g|_x(|D] }|jj||t|j7}qW|jj|dS)z:Derived classes must call this after their initialization.Nr)Nr)r*r+appendlen field_names)selfchildrenoffsetchildrrr__init__is  zField.__init__cCs |jddS)NF)r)r)r/rrr clone_schemasszField.clone_schemacCs tddS)z/Return the children field names for this field.zField is an abstract class.N)NotImplementedError)r/rrrr.vszField.field_namescCs tddS)z7Return the numpy.dtype for each of the children fields.zField is an abstract class.N)r5)r/rrr field_typeszszField.field_typescCs tddS)z4Return the Metadata for each of the children fields.zField is an abstract class.N)r5)r/rrrfield_metadata~szField.field_metadatacCs tddS)zReturn the list of blobs with contents for this Field. Values can either be all numpy.ndarray or BlobReference. If any of the fields doesn't have a blob, throws. zField is an abstract class.N)r5)r/rrr field_blobsszField.field_blobscCs tddS)zyReturn the list of all Scalar instances in the Field. The order is the same as for field_names() or field_blobs()zField is an abstract class.N)r5)r/rrr all_scalarsszField.all_scalarscCs tddS)z4Return True if every scalar of this field has blobs.zField is an abstract class.N)r5)r/rrr has_blobsszField.has_blobsTcCs tddS)z)Clone this Field along with its children.zField is an abstract class.N)r5)r/rrrrrsz Field.clonecCs||f|_dS)N)r*)r/parentZ relative_idrrr _set_parentszField._set_parentcCs|j}t||t|jS)a Returns a slice representing the range of field ids that belong to this field. This slice can be used to index a list of fields. E.g.: >>> s = Struct( >>> ('a', Scalar()), >>> ('b', Struct( >>> ('b1', Scalar()), >>> ('b2', Scalar()), >>> )), >>> ('c', Scalar()), >>> ) >>> field_data = ['da', 'db1', 'db2', 'dc'] >>> field_data[s.b.split()] ['db1', 'db2'] )_child_base_idslicer-r.)r/Zbase_idrrrr>sz Field.sliceNcCs6|j\}}|dkrdn|j|}|r2||j|7}|S)z"Get the base id of the given childNr)r*r+r=)r/Z child_indexpiposrrrr=s  zField._child_base_idcCs0|j|jko.|j|jko.|j|jkS)zEquivalance of two schemas)r.r6r7)r/otherrrr__eq__sz Field.__eq__cCs tddS)NzField is an abstract class.)r5)r/indent str_bufferrrr _pprint_implszField._pprint_implcCs&t}|jd||j}|j|S)Nr)r rFgetvalueclose)r/rEcontentsrrr__repr__s  zField.__repr__)r*r+)T)N)r#r$r%r&r'r r(r3r4r.r6r7r8r9r:rr<r>r=rCrFrJrrrrrcs"     rcseZdZUdZdZeedfdd ZddZd d Z d d Z d dZ ddZ ddZ dddZddZddZddZZS)Lista(Represents a variable-length list. Values of a list can also be complex fields such as Lists and Structs. In addition to the fields exposed by its `values` field, a List exposes an additional `lengths` field, which will contain the size of each list under the parent domain. lengths_itemsNcstt|tr$t|tstt||_nttj||_t||_|jj |d|jj |dt t |j |j|jgdS)Nr) rrrAssertionErrorr rLrint32rMr<superrKr3)r/values lengths_blob) __class__rrr3s   z List.__init__cCs|jj}dgdd|DS)NrLcSsg|]}td|qS)rR)r).0vrrr sz$List.field_names..)rMr.)r/ value_fieldsrrrr.s zList.field_namescCs|jj|jjS)N)rLr6rM)r/rrrr6szList.field_typescCs|jj|jjS)N)rLr7rM)r/rrrr7szList.field_metadatacCs|jj|jjS)N)rLr8rM)r/rrrr8szList.field_blobscCs|jj|jjS)N)rLr9rM)r/rrrr9szList.all_scalarscCs|jjo|jjS)N)rLr:rM)r/rrrr:szList.has_blobsTcCs"t|t|j|dt|j|dS)N)r)rr rMrL)r/rrrrrs z List.clonecCs||jd|d|jd|dd|jj|d|d|jd|dd|jj|d|d|jd|ddS) Nz zList( rNz lengths= )rDrEz_items= z) )writerLrFrM)r/rDrErrrrFs zList._pprint_implcCsP|jdrt|t|jtr*t|j|S|dks:|dkr@|jStd|dS)z\If the value of this list is a struct, allow to introspect directly into its fields.__valueitemszField not found in list: %s.N) startswithAttributeErrorrrMStructgetattr)r/itemrrr __getattr__s   zList.__getattr__cCs`|jtd}t|dkr6|dkr&|jS|dkrP|jSn|ddkrP|j|dStd|dS)NrNrLrRrzField not found in list: %s.)splitrr-rLrMKeyError)r/rbnamesrrr __getitem__s   zList.__getitem__)rLrM)N)T)r#r$r%r&r'r r(r3r.r6r7r8r9r:rrFrcrg __classcell__rr)rTrrKs     rKcseZdZUdZdZeedfdd ZddZdd Z d d Z d d Z ddZ ddZ dddZddZddZddZZS)ListWithEvictedzi This class is similar with List, but containing extra field evicted_values for LRU Hashing. _evicted_valuesNcsJt|tr$t|tstt||_nttj||_tt |j ||ddS)N)rS) rrrrOr rjrint64rQrir3)r/rRrSevicted_values)rTrrr3%s   zListWithEvicted.__init__cCs$|jj}dgdd|DdgS)NrLcSsg|]}td|qS)rR)r)rUrVrrrrW0sz/ListWithEvicted.field_names..rj)rMr.)r/rXrrrr.-s zListWithEvicted.field_namescCs|jj|jj|jjS)N)rLr6rMrj)r/rrrr63szListWithEvicted.field_typescCs|jj|jj|jjS)N)rLr7rMrj)r/rrrr76szListWithEvicted.field_metadatacCs|jj|jj|jjS)N)rLr8rMrj)r/rrrr89szListWithEvicted.field_blobscCs|jj|jj|jjS)N)rLr9rMrj)r/rrrr9<szListWithEvicted.all_scalarscCs|jjo|jj|jjS)N)rLr:rMrj)r/rrrr:?szListWithEvicted.has_blobsTcCs.t|t|j|dt|j|dt|j|dS)N)r)rr rMrLrj)r/rrrrrBs  zListWithEvicted.clonecCs|jd|d|jd|dd|jj|d|d|jd|dd|jj|d|d|jd|dd|jj|d|d|jd|d dS) Nz zListWithEvicted( rNz lengths= rY)rDrEz_items= z_evicted_values= z) )rZrLrFrMrj)r/rDrErrrrFIszListWithEvicted._pprint_implcCs^|jdrt||dkr |jSt|jtr8t|j|S|dksH|dkrN|jStd|dS)z\If the value of this list is a struct, allow to introspect directly into its fields.r[rjr\r]zField not found in list: %s.N)r^r_rjrrMr`ra)r/rbrrrrcTs   zListWithEvicted.__getattr__cCsn|jtd}t|dkrD|dkr&|jS|dkr4|jS|dkr^|jSn|ddkr^|j|dStd|dS)NrNrLrRrjrzField not found in list: %s.)rdrr-rLrMrjre)r/rbrfrrrrgbs   zListWithEvicted.__getitem__)rj)NN)T)r#r$r%r&r'r r(r3r.r6r7r8r9r:rrFrcrgrhrr)rTrris    ricseZdZUdZd-ZeefddZddZdd Z d d Z d d Z ddZ ddZ ddZddZd.ddZddZddZddZdd Zd!d"Zd#d$Zfd%d&Zfd'd(Zd)d*Zd+d,ZZS)/r`z?Represents a named list of fields sharing the same domain. fields_frozencs"x@|D]8}t|dkst|ds*td|ddkstdqWdd|D}t|_x||D]t\}}t|kr~|j||\}}||jkr||j|<q^t|t st|j|t rtd||j|||j|<q^Wx*t t |jD]\}\}}|j ||qWt t|j t|jd |_d S) a  fields is a list of tuples in format of (name, field). The name is a string of nested name, e.g., `a`, `a:b`, `a:b:c`. For example Struct( ('a', Scalar()), ('b:c', Scalar()), ('b:d:e', Scalar()), ('b', Struct( ('f', Scalar()), )), ) is equal to Struct( ('a', Scalar()), ('b', Struct( ('c', Scalar()), ('d', Struct(('e', Scalar()))), ('f', Scalar()), )), ) rYrzField names cannot be emptyrLz.Struct cannot contain a field named `lengths`.cSsg|]\}}|t|fqSr)r )rUnamefieldrrrrWsz#Struct.__init__..zDuplicate field name: %sTN)r-rOrrmr_struct_from_nested_namerr` ValueError enumeraterr<rQr3r rn)r/rmrproid_)rTrrr3xs*      zStruct.__init__cs>fdd|jtd}t|dks(t|d|d|fS)Ncs<|jtd}t|dkr|}n|d|}t|d|fS)NrNr)rdrr-r`) nested_namerprfZ added_field)create_internalrrrws   z8Struct._struct_from_nested_name..create_internalrNrYr)rdrr-rO)r/rvrprfr)rwrrqs  zStruct._struct_from_nested_namecCstt|jS)N)listrrm)r/rrr get_childrenszStruct.get_childrencs:g}x0t|jD]"\}|fdd|jD7}qW|S)Ncsg|]}t|qSr)r)rUf)rorrrWsz&Struct.field_names..)rrmr.)r/rfrpr)rorr.szStruct.field_namescCs,g}x"t|jD]\}}||j7}qW|S)N)rrmr6)r/typesrurprrrr6szStruct.field_typescCs,g}x"t|jD]\}}||j7}qW|S)N)rrmr7)r/metadatarurprrrr7szStruct.field_metadatacCs,g}x"t|jD]\}}||j7}qW|S)N)rrmr8)r/blobsrurprrrr8szStruct.field_blobscCs,g}x"t|jD]\}}||j7}qW|S)N)rrmr9)r/scalarsrurprrrr9szStruct.all_scalarscCstddt|jDS)Ncss|]}|jVqdS)N)r:)rUrprrr sz#Struct.has_blobs..)allr rm)r/rrrr:szStruct.has_blobsTcs$fddt|jD}t||S)Ncs g|]\}}|t|dfqS))r)r )rUkrV)rrrrWsz Struct.clone..)rrmr)r/rZnormalized_fieldsr)rrrs z Struct.clonec Csd|jtd}|jj|dd}|dkr*dSt|dkr:|Sy ||dSttfk r^dSXdS)NrNr)rdrrmgetr-re TypeError)r/rvrfrprrr_get_field_by_nested_names   z Struct._get_field_by_nested_namecCsr|jd|dxHt|jD]:\}}|jd|ddj|d|j|d|dqW|jd|ddS) Nz z Struct( rNz{}= rY)rDrEz) )rZrrmrrF)r/rDrErorprrrrFs  zStruct._pprint_implcCs|j|}|dk S)N)r)r/rbrprrr __contains__s zStruct.__contains__cCs t|jS)N)r-rm)r/rrr__len__szStruct.__len__cst|tst|tr:ttjtfdd|DSt|trZttt j|dSj |}|dkrxt d||SdS)z item can be a tuple or list of ints or strings, or a single int or string. String item is a nested field name, e.g., "a", "a:b", "a:b:c". Int item is the index of a field at the first level of the Struct. cs*g|]"}t|tr|n||fqSr)rint)rUr)keysr/rrrW sz&Struct.__getitem__..Nzfield "%s" not found) rrxtupler rmr`rnextr r rre)r/rbrpr)rr/rrgs     zStruct.__getitem__cCs t|||S)z similar to python's dictionary get method, return field of item if found (i.e. self.item is valid) or otherwise return default_value it's a syntax suger of python's builtin getattr method )ra)r/rb default_valuerrrrsz Struct.getc sH|jdrt|ytt|jd|Stk rBt|YnXdS)Nr[rm)r^r_rQr`__getattribute__re)r/rb)rTrrrc s  zStruct.__getattr__cs6t|ddr |jd r tdtt|j||dS)Nrnruz1Struct.__setattr__() is disabled after __init__())rar^rrQr` __setattr__)r/keyr\)rTrrr(szStruct.__setattr__cCst|tstSt|j}xz|jD]n\}}||kr>|||<q$||}t|toXt|tstdtt|dtt|d|||||<q$Wtt|S)a Allows to merge fields of two schema.Struct using '+' operator. If two Struct have common field names, the merge is conducted recursively. Here are examples: Example 1 s1 = Struct(('a', Scalar())) s2 = Struct(('b', Scalar())) s1 + s2 == Struct( ('a', Scalar()), ('b', Scalar()), ) Example 2 s1 = Struct( ('a', Scalar()), ('b', Struct(('c', Scalar()))), ) s2 = Struct(('b', Struct(('d', Scalar())))) s1 + s2 == Struct( ('a', Scalar()), ('b', Struct( ('c', Scalar()), ('d', Scalar()), )), ) zType of left_field, z, and type of right_field, z6, must both the Struct to allow merging of the field, ) rr`NotImplementedrryrr(rr)r/rBr0ro right_field left_fieldrrr__add__1s  *zStruct.__add__cCst|tstSt|j}x|jD]\}}||kr$||}t|t|kr|t|trp||}|jrp|||<q$|j|q$tdtt|dtt|d|q$Wt|j S)a Allows to remove common fields of two schema.Struct from self by using '-' operator. If two Struct have common field names, the removal is conducted recursively. If a child struct has no fields inside, it will be removed from its parent. Here are examples: Example 1 s1 = Struct( ('a', Scalar()), ('b', Scalar()), ) s2 = Struct(('a', Scalar())) s1 - s2 == Struct(('b', Scalar())) Example 2 s1 = Struct( ('b', Struct( ('c', Scalar()), ('d', Scalar()), )) ) s2 = Struct( ('b', Struct(('c', Scalar()))), ) s1 - s2 == Struct( ('b', Struct( ('d', Scalar()), )), ) Example 3 s1 = Struct( ('a', Scalar()), ('b', Struct( ('d', Scalar()), )) ) s2 = Struct( ('b', Struct( ('c', Scalar()) ('d', Scalar()) )), ) s1 - s2 == Struct( ('a', Scalar()), ) zType of left_field, z*, is not the same as that of right_field, z%, yet they have the same field name, ) rr`rrryrpoprr(r])r/rBr0rorrr2rrr__sub__`s 0    .zStruct.__sub__)rmrn)T)r#r$r%r&r'r r(r3rqryr.r6r7r8r9r:rrrFrrrgrrcrrrrhrr)rTrr`rs,   1    /r`cseZdZUdZd/Zeed0fdd Zd d Zd d Z d dZ ddZ ddZ ddZ ddZd1ddZddZddZeddZd d!Zd"d#Zd2d%d&Zd3d'd(Zd)d*Zd+d,Zd-d.ZZS)4ra>Represents a typed scalar or tensor of fixed shape. A Scalar is a leaf in a schema tree, translating to exactly one tensor in the dataset's underlying storage. Usually, the tensor storing the actual values of this field is a 1D tensor, representing a series of values in its domain. It is possible however to have higher rank values stored as a Scalar, as long as all entries have the same shape. E.g.: Scalar(np.float64) Scalar field of type float64. Caffe2 will expect readers and datasets to expose it as a 1D tensor of doubles (vector), where the size of the vector is determined by this fields' domain. Scalar((np.int32, 5)) Tensor field of type int32. Caffe2 will expect readers and datasets to implement it as a 2D tensor (matrix) of shape (L, 5), where L is determined by this fields' domain. Scalar((str, (10, 20))) Tensor field of type str. Caffe2 will expect readers and datasets to implement it as a 3D tensor of shape (L, 10, 20), where L is determined by this fields' domain. If the field type is unknown at construction time, call Scalar(), that will default to np.void as its dtype. It is an error to pass a structured dtype to Scalar, since it would contain more than one field. Instead, use from_dtype, which will construct a nested `Struct` field reflecting the given dtype's structure. A Scalar can also contain a blob, which represents the value of this Scalar. A blob can be either a numpy.ndarray, in which case it contain the actual contents of the Scalar, or a BlobReference, which represents a blob living in a caffe2 Workspace. If blob of different types are passed, a conversion to numpy.ndarray is attempted. _metadatar_original_dtype_blobNcs,d|_|j|||ddtt|jgdS)NT)unsafe)rsetrQrr3)r/rrr|)rTrrr3szScalar.__init__cCsdgS)Nrr)r/rrrr.szScalar.field_namescCs|jS)N)r)r/rrr field_typeszScalar.field_typecCs|jgS)N)r)r/rrrr6szScalar.field_typescCs|jgS)N)r)r/rrrr7szScalar.field_metadatacCs |jdk S)N)r)r/rrrr:szScalar.has_blobscCs|jdk std|jgS)Nz Value is not set for this field.)rrO)r/rrrr8szScalar.field_blobscCs|gS)Nr)r/rrrr9szScalar.all_scalarsTcCst|j|r|jnd|jdS)N)rrr|)rrrr)r/rrrrrs z Scalar.clonecCs|jdk std|jS)z+Gets the current blob of this Scalar field.Nz Value is not set for this field.)rrO)r/rrrrsz Scalar.getcCs|jS)zShortcut for self.get())r)r/rrr__call__szScalar.__call__cCs|jS)N)r)r/rrrr|szScalar.metadatacCs.t|tstdjt|||_|jdS)Nz!metadata must be Metadata, got {})rr"rOrrr_validate_metadata)r/r\rrr set_metadatas zScalar.set_metadatacCsL|jdkrdS|jjdk rH|jdk rHtj|jtjsHtddj|jdS)Nz6`categorical_limit` can be specified only in integral zfields but got {})rr)rrZ issubdtypeintegerrOr)r/rrrr s   zScalar._validate_metadataFcCsj|jjtjkrT|rTt|tjs,tdj||jj|jjksTtdj|jj|jj|j|j ||ddS)z.)r`rs)rrmr)rr NamedTuples rcGs td|S)zP Creates a Struct with default, sequential, field names of given types. rp)rp)r)rmrrrTuplesrrpcCs0t|tst|dkstt|ftjg|S)z9 Creates a tuple of `num_field` untyped scalars. r)rrrOrrr) num_fieldsrrrrRawTuples rcCst|tjs |}tj||f}n$||j}||jkrDtj|j|f}|jsRt|Sg}x:|jD]0\}\}}|dksztd||t||df7}q^Wt |S)aSConstructs a Caffe2 schema from the given numpy's dtype. Numpy supports scalar, array-like and structured datatypes, as long as all the shapes are fixed. This function breaks down the given dtype into a Caffe2 schema containing `Struct` and `Scalar` types. Fields containing byte offsets are not currently supported. rz+Fields with byte offsets are not supported.) _outer_shape) rrrrrrmrrO from_dtyper`)rrr struct_fieldsroZfdtyper1rrrrs   rc@sBeZdZUdZdZeedddZdd d Zd d Z d dZ dS) _SchemaNodez7This is a private class used to represent a Schema Noderor0type_strrprcCs||_g|_||_d|_dS)N)ror0rrp)r/rorrrrr3sz_SchemaNode.__init__cCsBx&|jD]}|j|kr|j|kr|SqWt||}|jj||S)N)r0rorrr,)r/rorr2rrr add_childs    z_SchemaNode.add_childc Csjddg}dddg}t|jdks*|jdk r@|jdkr:tS|jSg}x|jD]}|j|jqLWt|t|krx*|jD] }|jdkr|j}qx|j}qxWt||d|_d|_ |jSt|t|kr&x@|jD]6}|jdkr|j}q|jdkr|j}q|j}qWt |||d|_d|_ |jSg}x$|jD]}|j|j|jfq2Wt||_d|_ |jSdS) NrLrRrr)rSrKrr`) r-r0rpr`r,ror get_fieldrKrr) r/Z list_namesZ map_namesZ child_namesr2Z values_fieldZ lengths_fieldZ key_fieldrrrrrsL                 z_SchemaNode.get_fieldcCs>x|jD] }|jqWtjdtj|jtj|jdS)NzPrinting node: Name and type)r0print_recursivelyrinforor)r/r2rrrrs     z_SchemaNode.print_recursivelyN)ror0rrp)r)r) r#r$r%r&r'r r(r3rrrrrrrrs    4rcCs4|dkrdgt|}|dkr,dgt|}|dkrBdgt|}t|t|ksZtdt|t|ksrtdt|t|kstdtdd}xt||||D]\}}}}|jt}|} xhtt|D]X} || } d} d} | t|dkr|} t|||d } | j| | }| dk r | |_ |} qWqW|j S) zR Given a list of names, types, and optionally values, construct a Schema. Nz2col_names and col_types must have the same length.z5col_names and col_metadata must have the same length.z2col_names and col_blobs must have the same length.rootr`rrN)rrr|) r-rOrziprdrrangerrrpr)Z col_namesZ col_typesZ col_blobsZ col_metadatarZcol_nameZcol_typeZcol_blobcolumnscurrentr@rorrprrrrfrom_column_list&s@     rFcCst|tstdt|tr"|g}|j}|j}t|t|ksZtdt|t|fx&t||D]\}}|j||ddqfW|S)zd Create a schema that clones the given schema, but containing the given list of values. z"Argument `schema` must be a Field.z%Values must have %d elements, got %d.T)r) rrrOrr4r9r-rr)schemarRrrecordr~Zscalarr\rrrfrom_blob_listUs rcCst|tr|St|ts"t|tr^tdd|D}|rJtdd|DStdd|DSn(t|tr~tddt|DSt |SdS)Ncss0|](}|tko&t|dko&t|dtVqdS)rYrN)rr-rr)rUrzrrrrlszas_record..cSsg|]\}}|t|fqSr) as_record)rUrrVrrrrWpszas_record..cSsg|] }t|qSr)r)rUrzrrrrWrscSsg|]\}}|t|fqSr)r)rUrrVrrrrWts) rrrxrrr`rdictrr )r\Z is_field_listrrrrgs   rcsVfddt|tst|j}tdd|Ds8tfdd|D}t|||S)z Given a record containing BlobReferences, return a new record with same schema, containing numpy arrays, fetched from the current active workspace. cs,dkrtjt|Sjt|jSdS)N)rZ FetchBlobr(r}fetch)rV)wsrrrszFetchRecord..fetchcss|]}t|tVqdS)N)rr)rUrVrrrrszFetchRecord..csg|] }|qSrr)rUr\)rrrrWszFetchRecord..)rrrOr8rr) blob_recordrrr8Z field_arraysr)rrr FetchRecordys  rcsfdd}t|tst|j}tdd|Ds8tt|trJ|j}t|t|ksjtdt|x t||D]\}}|||qvWdS)z Given a Record containing blob_references and arrays, which is either a list of numpy arrays or a Record containing numpy arrays, feeds the record to the current workspace. cs@dkrtjt||n"jt|jt|j|dS)N)rZFeedBlobr(Z create_blobr}feed)brV)rrrrszFeedRecord..feedcss|]}t|tVqdS)N)rr)rUrVrrrrszFeedRecord..z(Values must contain exactly %d ndarrays.N)rrrOr8rr-r)rZarraysrrr8rrr)rr FeedRecords  rcs\t|tr*|j}|jjddd|St|ts.) rrrrrrrOr.r)rrresultZ blob_refsr)rr NewRecords   rcCs:t||}x*t|j|jD]\}}|j||qW|S)z^ Given a record of arrays, returns a record of blobs, initialized with net.Const. )rrr8ZConst)rZ array_recordrrrrrr ConstRecords  rc Cs|jst||}n|}xt|j|jD]t\}}y.t|}dgt|j}|jg|||dWq,t k rt j dj ||r|jg|dgdYq,Xq,W|S)Nr)rrzBlob {} has type error)r) r:rrr6r8data_type_for_dtyperxrZ ConstantFillrrrr)rZschema_or_recordZ enforce_typesrZ blob_typerZ data_typerrrrInitEmptyRecords rcCst|jjt|jS)N)rr.issubset)roriginal_schemarrris_schema_subsets rcCsht|tstt|tst|r4|j|jkr4dS|rL|j|jkrLdS|rd|j|jkrddSdS)NFT)rrrOr.r6r7)rrZcheck_field_namesZcheck_field_typesZcheck_field_metasrrr equal_schemassrcCs"t|}|dk rt||st|S)N)rrrO)rpreviousrrrr schema_checksrcCs6xtD]\}}|j|kr|SqWtdt|jdS)NzUnknown dtype: )_DATA_TYPE_FOR_DTYPErrr()rnp_typedtrrrrs rcCs2xtD]\}}||kr|SqWtdt|dS)NzUnknown core type: )rrr()Z core_typerrrrrdtype_for_core_typesrcCs x|jD]}|j|q WdS)N)r9r)rpr|rzrrrattach_metadata_to_scalars%sr)T)NNNNNN)NNN)rrRN)rrRNN)rp)NNN)F)NF)N)F)TTF)N)Ur&loggingZnumpyrZ caffe2.pythonrrZcaffe2.python.corer collectionsrrZ past.builtinsrZ future.utilsrr r itertoolsr sixr typingr getLoggerr#rrrr r!__new__ __defaults__r"objectrrKrir`rrrrrrrrrrrrrrrrr(ZDataTypeSTRINGZfloat16ZFLOAT16Zfloat32FLOATZfloat64ZDOUBLEboolZBOOLZint8ZINT8Zint16ZINT16rPZINT32rkZINT64Zuint8ZUINT8Zuint16ZUINT16rrrrrrrrrrrs          gSU7b   P -