FM / python

/usr/local/lib64/python3.6/site-packages/caffe2/python/

/usr/local/lib64/python3.6/site-packages/caffe2/python

Name	Size	Mode	Actions
docs/	-	0755	rm
examples/	-	0755	rm
fakelowp/	-	0755	rm
helpers/	-	0755	rm
ideep/	-	0755	rm
layers/	-	0755	rm
mint/	-	0755	rm
mkl/	-	0755	rm
modeling/	-	0755	rm
models/	-	0755	rm
onnx/	-	0755	rm
operator_test/	-	0755	rm
predictor/	-	0755	rm
rnn/	-	0755	rm
serialized_test/	-	0755	rm
test/	-	0755	rm
trt/	-	0755	rm
__pycache__/	-	0755	rm
allcompare_test.py	2255	0644	edit dl rm
attention.py	12359	0644	edit dl rm
benchmark_generator.py	4912	0644	edit dl rm
binarysize.py	5521	0644	edit dl rm
brew.py	4762	0644	edit dl rm
brew_test.py	11739	0644	edit dl rm
build.py	153	0644	edit dl rm
cached_reader.py	4394	0644	edit dl rm
caffe2_pybind11_state.cpython-36m-x86_64-linux-gnu.so	48299712	0755	edit dl rm
caffe2_pybind11_state_gpu.cpython-36m-x86_64-linux-gnu.so	49048144	0755	edit dl rm
caffe_translator.py	35227	0644	edit dl rm
caffe_translator_test.py	3553	0644	edit dl rm
checkpoint.py	32101	0644	edit dl rm
checkpoint_test.py	13405	0644	edit dl rm
cnn.py	7626	0644	edit dl rm
context.py	2841	0644	edit dl rm
context_test.py	1792	0644	edit dl rm
control.py	19309	0644	edit dl rm
control_ops_grad.py	28893	0644	edit dl rm
control_ops_grad_test.py	1752	0644	edit dl rm
control_ops_util.py	10863	0644	edit dl rm
control_test.py	12276	0644	edit dl rm
convert.py	55	0644	edit dl rm
convert_test.py	201	0644	edit dl rm
convnet_benchmarks.py	20533	0644	edit dl rm
convnet_benchmarks_test.py	839	0644	edit dl rm
core.py	119400	0644	edit dl rm
core_gradients_test.py	38022	0644	edit dl rm
core_test.py	47683	0644	edit dl rm
crf.py	13250	0644	edit dl rm
crf_predict.py	1159	0644	edit dl rm
crf_viterbi_test.py	1663	0644	edit dl rm
dataio.py	23532	0644	edit dl rm
dataio_test.py	17575	0644	edit dl rm
dataset.py	12886	0644	edit dl rm
data_parallel_model.py	83100	0644	edit dl rm
data_parallel_model_test.py	56145	0644	edit dl rm
data_workers.py	15941	0644	edit dl rm
data_workers_test.py	6561	0644	edit dl rm
db_file_reader.py	6608	0644	edit dl rm
db_test.py	1110	0644	edit dl rm
device_checker.py	5157	0644	edit dl rm
dyndep.py	1533	0644	edit dl rm
embedding_generation_benchmark.py	5256	0644	edit dl rm
experiment_util.py	3625	0644	edit dl rm
extension_loader.py	744	0644	edit dl rm
fakefp16_transform_lib.py	322	0644	edit dl rm
filler_test.py	748	0644	edit dl rm
functional.py	4415	0644	edit dl rm
functional_test.py	4204	0644	edit dl rm
fused_8bit_rowwise_conversion_ops_test.py	3945	0644	edit dl rm
gradient_checker.py	15377	0644	edit dl rm
gradient_check_test.py	20729	0644	edit dl rm
gru_cell.py	5129	0644	edit dl rm
hip_test_util.py	405	0644	edit dl rm
hsm_util.py	2259	0644	edit dl rm
hypothesis_test.py	105762	0644	edit dl rm
hypothesis_test_util.py	26853	0644	edit dl rm
ideep_test_util.py	998	0644	edit dl rm
layers_test.py	92931	0644	edit dl rm
layer_model_helper.py	29340	0644	edit dl rm
layer_model_instantiator.py	3935	0644	edit dl rm
layer_parameter_sharing_test.py	9148	0644	edit dl rm
layer_test_util.py	4875	0644	edit dl rm
lazy.py	277	0644	edit dl rm
lazy_dyndep.py	2562	0644	edit dl rm
lazy_dyndep_test.py	3914	0644	edit dl rm
lengths_reducer_fused_8bit_rowwise_ops_test.py	7575	0644	edit dl rm
lengths_reducer_rowwise_8bit_ops_test.py	5710	0644	edit dl rm
lstm_benchmark.py	10649	0644	edit dl rm
memonger.py	34041	0644	edit dl rm
memonger_test.py	36910	0644	edit dl rm
mkl_test_util.py	1142	0644	edit dl rm
model_device_test.py	4777	0644	edit dl rm
model_helper.py	23492	0644	edit dl rm
model_helper_test.py	2336	0644	edit dl rm
modifier_context.py	1772	0644	edit dl rm
muji.py	8131	0644	edit dl rm
muji_test.py	3058	0644	edit dl rm
net_builder.py	27679	0644	edit dl rm
net_builder_test.py	11382	0644	edit dl rm
net_drawer.py	14264	0644	edit dl rm
net_printer.py	12704	0644	edit dl rm
net_printer_test.py	3190	0644	edit dl rm
nomnigraph.py	4216	0644	edit dl rm
nomnigraph_test.py	15427	0644	edit dl rm
nomnigraph_transformations.py	3787	0644	edit dl rm
nomnigraph_transformations_test.py	5767	0644	edit dl rm
normalizer.py	1411	0644	edit dl rm
normalizer_context.py	1007	0644	edit dl rm
normalizer_test.py	487	0644	edit dl rm
numa_benchmark.py	2230	0644	edit dl rm
numa_test.py	1663	0644	edit dl rm
observer_test.py	5316	0644	edit dl rm
operator_fp_exceptions_test.py	1248	0644	edit dl rm
optimizer.py	78813	0644	edit dl rm
optimizer_context.py	1462	0644	edit dl rm
optimizer_test.py	30705	0644	edit dl rm
optimizer_test_util.py	9187	0644	edit dl rm
parallelize_bmuf_distributed_test.py	9908	0644	edit dl rm
parallel_workers.py	7682	0644	edit dl rm
parallel_workers_test.py	3501	0644	edit dl rm
pipeline.py	17283	0644	edit dl rm
pipeline_test.py	2542	0644	edit dl rm
predictor_constants.py	198	0644	edit dl rm
python_op_test.py	9169	0644	edit dl rm
queue_util.py	4459	0644	edit dl rm
record_queue.py	4453	0644	edit dl rm
recurrent.py	13297	0644	edit dl rm
regularizer.py	21120	0644	edit dl rm
regularizer_context.py	1013	0644	edit dl rm
regularizer_test.py	10266	0644	edit dl rm
rnn_cell.py	68233	0644	edit dl rm
schema.py	45621	0644	edit dl rm
schema_test.py	15754	0644	edit dl rm
scope.py	3623	0644	edit dl rm
scope_test.py	5249	0644	edit dl rm
session.py	7642	0644	edit dl rm
session_test.py	2078	0644	edit dl rm
sparse_to_dense_mask_test.py	6565	0644	edit dl rm
sparse_to_dense_test.py	3556	0644	edit dl rm
task.py	24274	0644	edit dl rm
task_test.py	870	0644	edit dl rm
test_util.py	3524	0644	edit dl rm
text_file_reader.py	1990	0644	edit dl rm
timeout_guard.py	4054	0644	edit dl rm
toy_regression_test.py	2822	0644	edit dl rm
transformations.py	1832	0644	edit dl rm
transformations_test.py	11960	0644	edit dl rm
tt_core.py	9349	0644	edit dl rm
tt_core_test.py	2518	0644	edit dl rm
utils.py	14181	0644	edit dl rm
utils_test.py	1399	0644	edit dl rm
visualize.py	6315	0644	edit dl rm
workspace.py	25263	0644	edit dl rm
workspace_test.py	34844	0644	edit dl rm
_import_c_extension.py	2250	0644	edit dl rm
__init__.py	3925	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/caffe2/python/schema_test.py (15754B)

from caffe2.python import core, schema
import numpy as np

import unittest
import pickle
import random

class TestField(unittest.TestCase):
    def testInitShouldSetEmptyParent(self):
        f = schema.Field([])
        self.assertTupleEqual(f._parent, (None, 0))

def testInitShouldSetFieldOffsets(self):
        f = schema.Field([
            schema.Scalar(dtype=np.int32),
            schema.Struct(
                ('field1', schema.Scalar(dtype=np.int32)),
                ('field2', schema.List(schema.Scalar(dtype=str))),
            ),
            schema.Scalar(dtype=np.int32),
            schema.Struct(
                ('field3', schema.Scalar(dtype=np.int32)),
                ('field4', schema.List(schema.Scalar(dtype=str)))
            ),
            schema.Scalar(dtype=np.int32),
        ])
        self.assertListEqual(f._field_offsets, [0, 1, 4, 5, 8, 9])

def testInitShouldSetFieldOffsetsIfNoChildren(self):
        f = schema.Field([])
        self.assertListEqual(f._field_offsets, [0])

class TestDB(unittest.TestCase):
    def testPicklable(self):
        s = schema.Struct(
            ('field1', schema.Scalar(dtype=np.int32)),
            ('field2', schema.List(schema.Scalar(dtype=str)))
        )
        s2 = pickle.loads(pickle.dumps(s))
        for r in (s, s2):
            self.assertTrue(isinstance(r.field1, schema.Scalar))
            self.assertTrue(isinstance(r.field2, schema.List))
            self.assertTrue(getattr(r, 'non_existent', None) is None)

def testListSubclassClone(self):
        class Subclass(schema.List):
            pass

s = Subclass(schema.Scalar())
        clone = s.clone()
        self.assertIsInstance(clone, Subclass)
        self.assertEqual(s, clone)
        self.assertIsNot(clone, s)

def testListWithEvictedSubclassClone(self):
        class Subclass(schema.ListWithEvicted):
            pass

s = Subclass(schema.Scalar())
        clone = s.clone()
        self.assertIsInstance(clone, Subclass)
        self.assertEqual(s, clone)
        self.assertIsNot(clone, s)

def testStructSubclassClone(self):
        class Subclass(schema.Struct):
            pass

s = Subclass(
            ('a', schema.Scalar()),
        )
        clone = s.clone()
        self.assertIsInstance(clone, Subclass)
        self.assertEqual(s, clone)
        self.assertIsNot(clone, s)

def testNormalizeField(self):
        s = schema.Struct(('field1', np.int32), ('field2', str))
        self.assertEquals(
            s,
            schema.Struct(
                ('field1', schema.Scalar(dtype=np.int32)),
                ('field2', schema.Scalar(dtype=str))
            )
        )

def testTuple(self):
        s = schema.Tuple(np.int32, str, np.float32)
        s2 = schema.Struct(
            ('field_0', schema.Scalar(dtype=np.int32)),
            ('field_1', schema.Scalar(dtype=np.str)),
            ('field_2', schema.Scalar(dtype=np.float32))
        )
        self.assertEquals(s, s2)
        self.assertEquals(s[0], schema.Scalar(dtype=np.int32))
        self.assertEquals(s[1], schema.Scalar(dtype=np.str))
        self.assertEquals(s[2], schema.Scalar(dtype=np.float32))
        self.assertEquals(
            s[2, 0],
            schema.Struct(
                ('field_2', schema.Scalar(dtype=np.float32)),
                ('field_0', schema.Scalar(dtype=np.int32)),
            )
        )
        # test iterator behavior
        for i, (v1, v2) in enumerate(zip(s, s2)):
            self.assertEquals(v1, v2)
            self.assertEquals(s[i], v1)
            self.assertEquals(s2[i], v1)

def testRawTuple(self):
        s = schema.RawTuple(2)
        self.assertEquals(
            s, schema.Struct(
                ('field_0', schema.Scalar()), ('field_1', schema.Scalar())
            )
        )
        self.assertEquals(s[0], schema.Scalar())
        self.assertEquals(s[1], schema.Scalar())

def testStructIndexing(self):
        s = schema.Struct(
            ('field1', schema.Scalar(dtype=np.int32)),
            ('field2', schema.List(schema.Scalar(dtype=str))),
            ('field3', schema.Struct()),
        )
        self.assertEquals(s['field2'], s.field2)
        self.assertEquals(s['field2'], schema.List(schema.Scalar(dtype=str)))
        self.assertEquals(s['field3'], schema.Struct())
        self.assertEquals(
            s['field2', 'field1'],
            schema.Struct(
                ('field2', schema.List(schema.Scalar(dtype=str))),
                ('field1', schema.Scalar(dtype=np.int32)),
            )
        )

def testListInStructIndexing(self):
        a = schema.List(schema.Scalar(dtype=str))
        s = schema.Struct(
            ('field1', schema.Scalar(dtype=np.int32)),
            ('field2', a)
        )
        self.assertEquals(s['field2:lengths'], a.lengths)
        self.assertEquals(s['field2:values'], a.items)
        with self.assertRaises(KeyError):
            s['fields2:items:non_existent']
        with self.assertRaises(KeyError):
            s['fields2:non_existent']

def testListWithEvictedInStructIndexing(self):
        a = schema.ListWithEvicted(schema.Scalar(dtype=str))
        s = schema.Struct(
            ('field1', schema.Scalar(dtype=np.int32)),
            ('field2', a)
        )
        self.assertEquals(s['field2:lengths'], a.lengths)
        self.assertEquals(s['field2:values'], a.items)
        self.assertEquals(s['field2:_evicted_values'], a._evicted_values)
        with self.assertRaises(KeyError):
            s['fields2:items:non_existent']
        with self.assertRaises(KeyError):
            s['fields2:non_existent']

def testMapInStructIndexing(self):
        a = schema.Map(
            schema.Scalar(dtype=np.int32),
            schema.Scalar(dtype=np.float32),
        )
        s = schema.Struct(
            ('field1', schema.Scalar(dtype=np.int32)),
            ('field2', a)
        )
        self.assertEquals(s['field2:values:keys'], a.keys)
        self.assertEquals(s['field2:values:values'], a.values)
        with self.assertRaises(KeyError):
            s['fields2:keys:non_existent']

def testPreservesMetadata(self):
        s = schema.Struct(
            ('a', schema.Scalar(np.float32)), (
                'b', schema.Scalar(
                    np.int32,
                    metadata=schema.Metadata(categorical_limit=5)
                )
            ), (
                'c', schema.List(
                    schema.Scalar(
                        np.int32,
                        metadata=schema.Metadata(categorical_limit=6)
                    )
                )
            )
        )
        # attach metadata to lengths field
        s.c.lengths.set_metadata(schema.Metadata(categorical_limit=7))

self.assertEqual(None, s.a.metadata)
        self.assertEqual(5, s.b.metadata.categorical_limit)
        self.assertEqual(6, s.c.value.metadata.categorical_limit)
        self.assertEqual(7, s.c.lengths.metadata.categorical_limit)
        sc = s.clone()
        self.assertEqual(None, sc.a.metadata)
        self.assertEqual(5, sc.b.metadata.categorical_limit)
        self.assertEqual(6, sc.c.value.metadata.categorical_limit)
        self.assertEqual(7, sc.c.lengths.metadata.categorical_limit)
        sv = schema.from_blob_list(
            s, [
                np.array([3.4]), np.array([2]), np.array([3]),
                np.array([1, 2, 3])
            ]
        )
        self.assertEqual(None, sv.a.metadata)
        self.assertEqual(5, sv.b.metadata.categorical_limit)
        self.assertEqual(6, sv.c.value.metadata.categorical_limit)
        self.assertEqual(7, sv.c.lengths.metadata.categorical_limit)

def testDupField(self):
        with self.assertRaises(ValueError):
            schema.Struct(
                ('a', schema.Scalar()),
                ('a', schema.Scalar()))

def testAssignToField(self):
        with self.assertRaises(TypeError):
            s = schema.Struct(('a', schema.Scalar()))
            s.a = schema.Scalar()

def testPreservesEmptyFields(self):
        s = schema.Struct(
            ('a', schema.Scalar(np.float32)),
            ('b', schema.Struct()),
        )
        sc = s.clone()
        self.assertIn("a", sc.fields)
        self.assertIn("b", sc.fields)
        sv = schema.from_blob_list(s, [np.array([3.4])])
        self.assertIn("a", sv.fields)
        self.assertIn("b", sv.fields)
        self.assertEqual(0, len(sv.b.fields))

def testStructSubstraction(self):
        s1 = schema.Struct(
            ('a', schema.Scalar()),
            ('b', schema.Scalar()),
            ('c', schema.Scalar()),
        )
        s2 = schema.Struct(
            ('b', schema.Scalar())
        )
        s = s1 - s2
        self.assertEqual(['a', 'c'], s.field_names())

s3 = schema.Struct(
            ('a', schema.Scalar())
        )
        s = s1 - s3
        self.assertEqual(['b', 'c'], s.field_names())

with self.assertRaises(TypeError):
            s1 - schema.Scalar()

def testStructNestedSubstraction(self):
        s1 = schema.Struct(
            ('a', schema.Scalar()),
            ('b', schema.Struct(
                ('c', schema.Scalar()),
                ('d', schema.Scalar()),
                ('e', schema.Scalar()),
                ('f', schema.Scalar()),
            )),
        )
        s2 = schema.Struct(
            ('b', schema.Struct(
                ('d', schema.Scalar()),
                ('e', schema.Scalar()),
            )),
        )
        s = s1 - s2
        self.assertEqual(['a', 'b:c', 'b:f'], s.field_names())

def testStructAddition(self):
        s1 = schema.Struct(
            ('a', schema.Scalar())
        )
        s2 = schema.Struct(
            ('b', schema.Scalar())
        )
        s = s1 + s2
        self.assertIn("a", s.fields)
        self.assertIn("b", s.fields)
        with self.assertRaises(TypeError):
            s1 + s1
        with self.assertRaises(TypeError):
            s1 + schema.Scalar()

def testStructNestedAddition(self):
        s1 = schema.Struct(
            ('a', schema.Scalar()),
            ('b', schema.Struct(
                ('c', schema.Scalar())
            )),
        )
        s2 = schema.Struct(
            ('b', schema.Struct(
                ('d', schema.Scalar())
            ))
        )
        s = s1 + s2
        self.assertEqual(['a', 'b:c', 'b:d'], s.field_names())

s3 = schema.Struct(
            ('b', schema.Scalar()),
        )
        with self.assertRaises(TypeError):
            s = s1 + s3

def testGetFieldByNestedName(self):
        st = schema.Struct(
            ('a', schema.Scalar()),
            ('b', schema.Struct(
                ('c', schema.Struct(
                    ('d', schema.Scalar()),
                )),
            )),
        )
        self.assertRaises(KeyError, st.__getitem__, '')
        self.assertRaises(KeyError, st.__getitem__, 'x')
        self.assertRaises(KeyError, st.__getitem__, 'x:y')
        self.assertRaises(KeyError, st.__getitem__, 'b:c:x')
        a = st['a']
        self.assertTrue(isinstance(a, schema.Scalar))
        bc = st['b:c']
        self.assertIn('d', bc.fields)
        bcd = st['b:c:d']
        self.assertTrue(isinstance(bcd, schema.Scalar))

def testAddFieldByNestedName(self):
        f_a = schema.Scalar(blob=core.BlobReference('blob1'))
        f_b = schema.Struct(
            ('c', schema.Struct(
                ('d', schema.Scalar(blob=core.BlobReference('blob2'))),
            )),
        )
        f_x = schema.Struct(
            ('x', schema.Scalar(blob=core.BlobReference('blob3'))),
        )

with self.assertRaises(TypeError):
            st = schema.Struct(
                ('a', f_a),
                ('b', f_b),
                ('b:c:d', f_x),
            )
        with self.assertRaises(TypeError):
            st = schema.Struct(
                ('a', f_a),
                ('b', f_b),
                ('b:c:d:e', f_x),
            )

st = schema.Struct(
            ('a', f_a),
            ('b', f_b),
            ('e:f', f_x),
        )
        self.assertEqual(['a', 'b:c:d', 'e:f:x'], st.field_names())
        self.assertEqual(['blob1', 'blob2', 'blob3'], st.field_blobs())

st = schema.Struct(
            ('a', f_a),
            ('b:c:e', f_x),
            ('b', f_b),
        )
        self.assertEqual(['a', 'b:c:e:x', 'b:c:d'], st.field_names())
        self.assertEqual(['blob1', 'blob3', 'blob2'], st.field_blobs())

st = schema.Struct(
            ('a:a1', f_a),
            ('b:b1', f_b),
            ('a', f_x),
        )
        self.assertEqual(['a:a1', 'a:x', 'b:b1:c:d'], st.field_names())
        self.assertEqual(['blob1', 'blob3', 'blob2'], st.field_blobs())

def testContains(self):
        st = schema.Struct(
            ('a', schema.Scalar()),
            ('b', schema.Struct(
                ('c', schema.Struct(
                    ('d', schema.Scalar()),
                )),
            )),
        )
        self.assertTrue('a' in st)
        self.assertTrue('b:c' in st)
        self.assertTrue('b:c:d' in st)
        self.assertFalse('' in st)
        self.assertFalse('x' in st)
        self.assertFalse('b:c:x' in st)
        self.assertFalse('b:c:d:x' in st)

def testFromEmptyColumnList(self):
        st = schema.Struct()
        columns = st.field_names()
        rec = schema.from_column_list(col_names=columns)
        self.assertEqual(rec, schema.Struct())

def testFromColumnList(self):
        st = schema.Struct(
            ('a', schema.Scalar()),
            ('b', schema.List(schema.Scalar())),
            ('c', schema.Map(schema.Scalar(), schema.Scalar()))
        )
        columns = st.field_names()
        # test that recovery works for arbitrary order
        for _ in range(10):
            some_blobs = [core.BlobReference('blob:' + x) for x in columns]
            rec = schema.from_column_list(columns, col_blobs=some_blobs)
            self.assertTrue(rec.has_blobs())
            self.assertEqual(sorted(st.field_names()), sorted(rec.field_names()))
            self.assertEqual([str(blob) for blob in rec.field_blobs()],
                             [str('blob:' + name) for name in rec.field_names()])
            random.shuffle(columns)

def testStructGet(self):
        net = core.Net('test_net')
        s1 = schema.NewRecord(net, schema.Scalar(np.float32))
        s2 = schema.NewRecord(net, schema.Scalar(np.float32))
        t = schema.Tuple(s1, s2)
        assert t.get('field_0', None) == s1
        assert t.get('field_1', None) == s2
        assert t.get('field_2', None) is None

def testScalarForVoidType(self):
        s0_good = schema.Scalar((None, (2, )))
        with self.assertRaises(TypeError):
            s0_bad = schema.Scalar((np.void, (2, )))

s1_good = schema.Scalar(np.void)
        s2_good = schema.Scalar(None)
        assert s1_good == s2_good

def testScalarShape(self):
        s0 = schema.Scalar(np.int32)
        self.assertEqual(s0.field_type().shape, ())

s1_good = schema.Scalar((np.int32, 5))
        self.assertEqual(s1_good.field_type().shape, (5, ))

with self.assertRaises(ValueError):
            s1_bad = schema.Scalar((np.int32, -1))

s1_hard = schema.Scalar((np.int32, 1))
        self.assertEqual(s1_hard.field_type().shape, (1, ))

s2 = schema.Scalar((np.int32, (2, 3)))
        self.assertEqual(s2.field_type().shape, (2, 3))

def testDtypeForCoreType(self):
        dtype = schema.dtype_for_core_type(core.DataType.FLOAT16)
        self.assertEqual(dtype, np.float16)

with self.assertRaises(TypeError):
            schema.dtype_for_core_type(100)

from caffe2.python import core, schema import numpy as np import unittest import pickle import random class TestField(unittest.TestCase): def testInitShouldSetEmptyParent(self): f = schema.Field([]) self.assertTupleEqual(f._parent, (None, 0)) def testInitShouldSetFieldOffsets(self): f = schema.Field([ schema.Scalar(dtype=np.int32), schema.Struct( ('field1', schema.Scalar(dtype=np.int32)), ('field2', schema.List(schema.Scalar(dtype=str))), ), schema.Scalar(dtype=np.int32), schema.Struct( ('field3', schema.Scalar(dtype=np.int32)), ('field4', schema.List(schema.Scalar(dtype=str))) ), schema.Scalar(dtype=np.int32), ]) self.assertListEqual(f._field_offsets, [0, 1, 4, 5, 8, 9]) def testInitShouldSetFieldOffsetsIfNoChildren(self): f = schema.Field([]) self.assertListEqual(f._field_offsets, [0]) class TestDB(unittest.TestCase): def testPicklable(self): s = schema.Struct( ('field1', schema.Scalar(dtype=np.int32)), ('field2', schema.List(schema.Scalar(dtype=str))) ) s2 = pickle.loads(pickle.dumps(s)) for r in (s, s2): self.assertTrue(isinstance(r.field1, schema.Scalar)) self.assertTrue(isinstance(r.field2, schema.List)) self.assertTrue(getattr(r, 'non_existent', None) is None) def testListSubclassClone(self): class Subclass(schema.List): pass s = Subclass(schema.Scalar()) clone = s.clone() self.assertIsInstance(clone, Subclass) self.assertEqual(s, clone) self.assertIsNot(clone, s) def testListWithEvictedSubclassClone(self): class Subclass(schema.ListWithEvicted): pass s = Subclass(schema.Scalar()) clone = s.clone() self.assertIsInstance(clone, Subclass) self.assertEqual(s, clone) self.assertIsNot(clone, s) def testStructSubclassClone(self): class Subclass(schema.Struct): pass s = Subclass( ('a', schema.Scalar()), ) clone = s.clone() self.assertIsInstance(clone, Subclass) self.assertEqual(s, clone) self.assertIsNot(clone, s) def testNormalizeField(self): s = schema.Struct(('field1', np.int32), ('field2', str)) self.assertEquals( s, schema.Struct( ('field1', schema.Scalar(dtype=np.int32)), ('field2', schema.Scalar(dtype=str)) ) ) def testTuple(self): s = schema.Tuple(np.int32, str, np.float32) s2 = schema.Struct( ('field_0', schema.Scalar(dtype=np.int32)), ('field_1', schema.Scalar(dtype=np.str)), ('field_2', schema.Scalar(dtype=np.float32)) ) self.assertEquals(s, s2) self.assertEquals(s[0], schema.Scalar(dtype=np.int32)) self.assertEquals(s[1], schema.Scalar(dtype=np.str)) self.assertEquals(s[2], schema.Scalar(dtype=np.float32)) self.assertEquals( s[2, 0], schema.Struct( ('field_2', schema.Scalar(dtype=np.float32)), ('field_0', schema.Scalar(dtype=np.int32)), ) ) # test iterator behavior for i, (v1, v2) in enumerate(zip(s, s2)): self.assertEquals(v1, v2) self.assertEquals(s[i], v1) self.assertEquals(s2[i], v1) def testRawTuple(self): s = schema.RawTuple(2) self.assertEquals( s, schema.Struct( ('field_0', schema.Scalar()), ('field_1', schema.Scalar()) ) ) self.assertEquals(s[0], schema.Scalar()) self.assertEquals(s[1], schema.Scalar()) def testStructIndexing(self): s = schema.Struct( ('field1', schema.Scalar(dtype=np.int32)), ('field2', schema.List(schema.Scalar(dtype=str))), ('field3', schema.Struct()), ) self.assertEquals(s['field2'], s.field2) self.assertEquals(s['field2'], schema.List(schema.Scalar(dtype=str))) self.assertEquals(s['field3'], schema.Struct()) self.assertEquals( s['field2', 'field1'], schema.Struct( ('field2', schema.List(schema.Scalar(dtype=str))), ('field1', schema.Scalar(dtype=np.int32)), ) ) def testListInStructIndexing(self): a = schema.List(schema.Scalar(dtype=str)) s = schema.Struct( ('field1', schema.Scalar(dtype=np.int32)), ('field2', a) ) self.assertEquals(s['field2:lengths'], a.lengths) self.assertEquals(s['field2:values'], a.items) with self.assertRaises(KeyError): s['fields2:items:non_existent'] with self.assertRaises(KeyError): s['fields2:non_existent'] def testListWithEvictedInStructIndexing(self): a = schema.ListWithEvicted(schema.Scalar(dtype=str)) s = schema.Struct( ('field1', schema.Scalar(dtype=np.int32)), ('field2', a) ) self.assertEquals(s['field2:lengths'], a.lengths) self.assertEquals(s['field2:values'], a.items) self.assertEquals(s['field2:_evicted_values'], a._evicted_values) with self.assertRaises(KeyError): s['fields2:items:non_existent'] with self.assertRaises(KeyError): s['fields2:non_existent'] def testMapInStructIndexing(self): a = schema.Map( schema.Scalar(dtype=np.int32), schema.Scalar(dtype=np.float32), ) s = schema.Struct( ('field1', schema.Scalar(dtype=np.int32)), ('field2', a) ) self.assertEquals(s['field2:values:keys'], a.keys) self.assertEquals(s['field2:values:values'], a.values) with self.assertRaises(KeyError): s['fields2:keys:non_existent'] def testPreservesMetadata(self): s = schema.Struct( ('a', schema.Scalar(np.float32)), ( 'b', schema.Scalar( np.int32, metadata=schema.Metadata(categorical_limit=5) ) ), ( 'c', schema.List( schema.Scalar( np.int32, metadata=schema.Metadata(categorical_limit=6) ) ) ) ) # attach metadata to lengths field s.c.lengths.set_metadata(schema.Metadata(categorical_limit=7)) self.assertEqual(None, s.a.metadata) self.assertEqual(5, s.b.metadata.categorical_limit) self.assertEqual(6, s.c.value.metadata.categorical_limit) self.assertEqual(7, s.c.lengths.metadata.categorical_limit) sc = s.clone() self.assertEqual(None, sc.a.metadata) self.assertEqual(5, sc.b.metadata.categorical_limit) self.assertEqual(6, sc.c.value.metadata.categorical_limit) self.assertEqual(7, sc.c.lengths.metadata.categorical_limit) sv = schema.from_blob_list( s, [ np.array([3.4]), np.array([2]), np.array([3]), np.array([1, 2, 3]) ] ) self.assertEqual(None, sv.a.metadata) self.assertEqual(5, sv.b.metadata.categorical_limit) self.assertEqual(6, sv.c.value.metadata.categorical_limit) self.assertEqual(7, sv.c.lengths.metadata.categorical_limit) def testDupField(self): with self.assertRaises(ValueError): schema.Struct( ('a', schema.Scalar()), ('a', schema.Scalar())) def testAssignToField(self): with self.assertRaises(TypeError): s = schema.Struct(('a', schema.Scalar())) s.a = schema.Scalar() def testPreservesEmptyFields(self): s = schema.Struct( ('a', schema.Scalar(np.float32)), ('b', schema.Struct()), ) sc = s.clone() self.assertIn("a", sc.fields) self.assertIn("b", sc.fields) sv = schema.from_blob_list(s, [np.array([3.4])]) self.assertIn("a", sv.fields) self.assertIn("b", sv.fields) self.assertEqual(0, len(sv.b.fields)) def testStructSubstraction(self): s1 = schema.Struct( ('a', schema.Scalar()), ('b', schema.Scalar()), ('c', schema.Scalar()), ) s2 = schema.Struct( ('b', schema.Scalar()) ) s = s1 - s2 self.assertEqual(['a', 'c'], s.field_names()) s3 = schema.Struct( ('a', schema.Scalar()) ) s = s1 - s3 self.assertEqual(['b', 'c'], s.field_names()) with self.assertRaises(TypeError): s1 - schema.Scalar() def testStructNestedSubstraction(self): s1 = schema.Struct( ('a', schema.Scalar()), ('b', schema.Struct( ('c', schema.Scalar()), ('d', schema.Scalar()), ('e', schema.Scalar()), ('f', schema.Scalar()), )), ) s2 = schema.Struct( ('b', schema.Struct( ('d', schema.Scalar()), ('e', schema.Scalar()), )), ) s = s1 - s2 self.assertEqual(['a', 'b:c', 'b:f'], s.field_names()) def testStructAddition(self): s1 = schema.Struct( ('a', schema.Scalar()) ) s2 = schema.Struct( ('b', schema.Scalar()) ) s = s1 + s2 self.assertIn("a", s.fields) self.assertIn("b", s.fields) with self.assertRaises(TypeError): s1 + s1 with self.assertRaises(TypeError): s1 + schema.Scalar() def testStructNestedAddition(self): s1 = schema.Struct( ('a', schema.Scalar()), ('b', schema.Struct( ('c', schema.Scalar()) )), ) s2 = schema.Struct( ('b', schema.Struct( ('d', schema.Scalar()) )) ) s = s1 + s2 self.assertEqual(['a', 'b:c', 'b:d'], s.field_names()) s3 = schema.Struct( ('b', schema.Scalar()), ) with self.assertRaises(TypeError): s = s1 + s3 def testGetFieldByNestedName(self): st = schema.Struct( ('a', schema.Scalar()), ('b', schema.Struct( ('c', schema.Struct( ('d', schema.Scalar()), )), )), ) self.assertRaises(KeyError, st.__getitem__, '') self.assertRaises(KeyError, st.__getitem__, 'x') self.assertRaises(KeyError, st.__getitem__, 'x:y') self.assertRaises(KeyError, st.__getitem__, 'b:c:x') a = st['a'] self.assertTrue(isinstance(a, schema.Scalar)) bc = st['b:c'] self.assertIn('d', bc.fields) bcd = st['b:c:d'] self.assertTrue(isinstance(bcd, schema.Scalar)) def testAddFieldByNestedName(self): f_a = schema.Scalar(blob=core.BlobReference('blob1')) f_b = schema.Struct( ('c', schema.Struct( ('d', schema.Scalar(blob=core.BlobReference('blob2'))), )), ) f_x = schema.Struct( ('x', schema.Scalar(blob=core.BlobReference('blob3'))), ) with self.assertRaises(TypeError): st = schema.Struct( ('a', f_a), ('b', f_b), ('b:c:d', f_x), ) with self.assertRaises(TypeError): st = schema.Struct( ('a', f_a), ('b', f_b), ('b:c:d:e', f_x), ) st = schema.Struct( ('a', f_a), ('b', f_b), ('e:f', f_x), ) self.assertEqual(['a', 'b:c:d', 'e:f:x'], st.field_names()) self.assertEqual(['blob1', 'blob2', 'blob3'], st.field_blobs()) st = schema.Struct( ('a', f_a), ('b:c:e', f_x), ('b', f_b), ) self.assertEqual(['a', 'b:c:e:x', 'b:c:d'], st.field_names()) self.assertEqual(['blob1', 'blob3', 'blob2'], st.field_blobs()) st = schema.Struct( ('a:a1', f_a), ('b:b1', f_b), ('a', f_x), ) self.assertEqual(['a:a1', 'a:x', 'b:b1:c:d'], st.field_names()) self.assertEqual(['blob1', 'blob3', 'blob2'], st.field_blobs()) def testContains(self): st = schema.Struct( ('a', schema.Scalar()), ('b', schema.Struct( ('c', schema.Struct( ('d', schema.Scalar()), )), )), ) self.assertTrue('a' in st) self.assertTrue('b:c' in st) self.assertTrue('b:c:d' in st) self.assertFalse('' in st) self.assertFalse('x' in st) self.assertFalse('b:c:x' in st) self.assertFalse('b:c:d:x' in st) def testFromEmptyColumnList(self): st = schema.Struct() columns = st.field_names() rec = schema.from_column_list(col_names=columns) self.assertEqual(rec, schema.Struct()) def testFromColumnList(self): st = schema.Struct( ('a', schema.Scalar()), ('b', schema.List(schema.Scalar())), ('c', schema.Map(schema.Scalar(), schema.Scalar())) ) columns = st.field_names() # test that recovery works for arbitrary order for _ in range(10): some_blobs = [core.BlobReference('blob:' + x) for x in columns] rec = schema.from_column_list(columns, col_blobs=some_blobs) self.assertTrue(rec.has_blobs()) self.assertEqual(sorted(st.field_names()), sorted(rec.field_names())) self.assertEqual([str(blob) for blob in rec.field_blobs()], [str('blob:' + name) for name in rec.field_names()]) random.shuffle(columns) def testStructGet(self): net = core.Net('test_net') s1 = schema.NewRecord(net, schema.Scalar(np.float32)) s2 = schema.NewRecord(net, schema.Scalar(np.float32)) t = schema.Tuple(s1, s2) assert t.get('field_0', None) == s1 assert t.get('field_1', None) == s2 assert t.get('field_2', None) is None def testScalarForVoidType(self): s0_good = schema.Scalar((None, (2, ))) with self.assertRaises(TypeError): s0_bad = schema.Scalar((np.void, (2, ))) s1_good = schema.Scalar(np.void) s2_good = schema.Scalar(None) assert s1_good == s2_good def testScalarShape(self): s0 = schema.Scalar(np.int32) self.assertEqual(s0.field_type().shape, ()) s1_good = schema.Scalar((np.int32, 5)) self.assertEqual(s1_good.field_type().shape, (5, )) with self.assertRaises(ValueError): s1_bad = schema.Scalar((np.int32, -1)) s1_hard = schema.Scalar((np.int32, 1)) self.assertEqual(s1_hard.field_type().shape, (1, )) s2 = schema.Scalar((np.int32, (2, 3))) self.assertEqual(s2.field_type().shape, (2, 3)) def testDtypeForCoreType(self): dtype = schema.dtype_for_core_type(core.DataType.FLOAT16) self.assertEqual(dtype, np.float16) with self.assertRaises(TypeError): schema.dtype_for_core_type(100)