FM / operator

/usr/local/lib64/python3.6/site-packages/caffe2/python/operator_test/

/usr/local/lib64/python3.6/site-packages/caffe2/python/operator_test

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
activation_ops_test.py	9691	0644	edit dl rm
adadelta_test.py	7932	0644	edit dl rm
adagrad_test.py	7586	0644	edit dl rm
adagrad_test_helper.py	5181	0644	edit dl rm
adam_test.py	21559	0644	edit dl rm
affine_channel_op_test.py	3784	0644	edit dl rm
alias_with_name_test.py	938	0644	edit dl rm
apmeter_test.py	2738	0644	edit dl rm
arg_ops_test.py	1917	0644	edit dl rm
assert_test.py	797	0644	edit dl rm
async_net_barrier_test.py	946	0644	edit dl rm
atomic_ops_test.py	4104	0644	edit dl rm
basic_rnn_test.py	4720	0644	edit dl rm
batch_box_cox_test.py	5080	0644	edit dl rm
batch_bucketize_op_test.py	3730	0644	edit dl rm
batch_moments_op_test.py	2795	0644	edit dl rm
batch_sparse_to_dense_op_test.py	4199	0644	edit dl rm
bbox_transform_test.py	12258	0644	edit dl rm
bisect_percentile_op_test.py	6227	0644	edit dl rm
blobs_queue_db_test.py	3240	0644	edit dl rm
boolean_mask_test.py	16389	0644	edit dl rm
boolean_unmask_test.py	1711	0644	edit dl rm
box_with_nms_limit_op_test.py	8750	0644	edit dl rm
bucketize_op_test.py	930	0644	edit dl rm
cast_op_test.py	1600	0644	edit dl rm
ceil_op_test.py	888	0644	edit dl rm
channel_backprop_stats_op_test.py	2131	0644	edit dl rm
channel_shuffle_test.py	1794	0644	edit dl rm
channel_stats_op_test.py	2639	0644	edit dl rm
checkpoint_test.py	1500	0644	edit dl rm
clip_op_test.py	1984	0644	edit dl rm
clip_tensor_op_test.py	2076	0644	edit dl rm
collect_and_distribute_fpn_rpn_proposals_op_test.py	11269	0644	edit dl rm
concat_op_cost_test.py	2858	0644	edit dl rm
concat_split_op_test.py	7266	0644	edit dl rm
conditional_test.py	995	0644	edit dl rm
conftest.py	1446	0644	edit dl rm
conv_test.py	32473	0644	edit dl rm
conv_transpose_test.py	15945	0644	edit dl rm
copy_ops_test.py	7374	0644	edit dl rm
copy_rows_to_tensor_op_test.py	2526	0644	edit dl rm
cosine_embedding_criterion_op_test.py	1953	0644	edit dl rm
counter_ops_test.py	3348	0644	edit dl rm
crf_test.py	5315	0644	edit dl rm
cross_entropy_ops_test.py	10085	0644	edit dl rm
ctc_beam_search_decoder_op_test.py	5197	0644	edit dl rm
ctc_greedy_decoder_op_test.py	4743	0644	edit dl rm
cudnn_recurrent_test.py	5817	0644	edit dl rm
dataset_ops_test.py	23847	0644	edit dl rm
data_couple_op_test.py	858	0644	edit dl rm
decay_adagrad_test.py	2694	0644	edit dl rm
deform_conv_test.py	19276	0644	edit dl rm
dense_vector_to_id_list_op_test.py	2044	0644	edit dl rm
depthwise_3x3_conv_test.py	1863	0644	edit dl rm
detectron_keypoints.py	7973	0644	edit dl rm
distance_op_test.py	4351	0644	edit dl rm
dropout_op_test.py	2971	0644	edit dl rm
duplicate_operands_test.py	734	0644	edit dl rm
elementwise_linear_op_test.py	1382	0644	edit dl rm
elementwise_logical_ops_test.py	4617	0644	edit dl rm
elementwise_ops_test.py	33354	0644	edit dl rm
elementwise_op_broadcast_test.py	17466	0644	edit dl rm
emptysample_ops_test.py	1977	0644	edit dl rm
enforce_finite_op_test.py	1286	0644	edit dl rm
ensure_clipped_test.py	1505	0644	edit dl rm
ensure_cpu_output_op_test.py	1244	0644	edit dl rm
erf_op_test.py	749	0644	edit dl rm
expand_op_test.py	2109	0644	edit dl rm
fc_operator_test.py	3720	0644	edit dl rm
feature_maps_ops_test.py	21492	0644	edit dl rm
filler_ops_test.py	8476	0644	edit dl rm
find_op_test.py	1316	0644	edit dl rm
flatten_op_test.py	922	0644	edit dl rm
flexible_top_k_test.py	2609	0644	edit dl rm
floor_op_test.py	894	0644	edit dl rm
fused_nbit_rowwise_conversion_ops_test.py	14077	0644	edit dl rm
fused_nbit_rowwise_test_helper.py	2693	0644	edit dl rm
gather_ops_test.py	9216	0644	edit dl rm
gather_ranges_op_test.py	9125	0644	edit dl rm
given_tensor_byte_string_to_uint8_fill_op_test.py	1392	0644	edit dl rm
given_tensor_fill_op_test.py	1503	0644	edit dl rm
glu_op_test.py	1212	0644	edit dl rm
group_conv_test.py	2870	0644	edit dl rm
group_norm_op_test.py	5252	0644	edit dl rm
gru_test.py	12932	0644	edit dl rm
heatmap_max_keypoint_op_test.py	4770	0644	edit dl rm
histogram_test.py	3097	0644	edit dl rm
hsm_test.py	9456	0644	edit dl rm
hyperbolic_ops_test.py	1472	0644	edit dl rm
im2col_col2im_test.py	4311	0644	edit dl rm
image_input_op_test.py	17345	0644	edit dl rm
index_hash_ops_test.py	2885	0644	edit dl rm
index_ops_test.py	4597	0644	edit dl rm
instance_norm_test.py	9917	0644	edit dl rm
integral_image_ops_test.py	3419	0644	edit dl rm
jsd_ops_test.py	1044	0644	edit dl rm
key_split_ops_test.py	1289	0644	edit dl rm
lars_test.py	1354	0644	edit dl rm
layer_norm_op_test.py	14983	0644	edit dl rm
leaky_relu_test.py	5639	0644	edit dl rm
learning_rate_adaption_op_test.py	2837	0644	edit dl rm
learning_rate_op_test.py	8652	0644	edit dl rm
lengths_pad_op_test.py	1625	0644	edit dl rm
lengths_reducer_fused_nbit_rowwise_ops_test.py	15495	0644	edit dl rm
lengths_tile_op_test.py	1332	0644	edit dl rm
lengths_top_k_ops_test.py	2371	0644	edit dl rm
length_split_op_test.py	4868	0644	edit dl rm
listwise_l2r_operator_test.py	8740	0644	edit dl rm
load_save_test.py	33241	0644	edit dl rm
locally_connected_op_test.py	7761	0644	edit dl rm
loss_ops_test.py	902	0644	edit dl rm
lpnorm_op_test.py	2725	0644	edit dl rm
map_ops_test.py	2249	0644	edit dl rm
margin_ranking_criterion_op_test.py	1816	0644	edit dl rm
math_ops_test.py	1603	0644	edit dl rm
matmul_op_test.py	10096	0644	edit dl rm
mean_op_test.py	1469	0644	edit dl rm
merge_id_lists_op_test.py	2989	0644	edit dl rm
mkl_conv_op_test.py	1547	0644	edit dl rm
mkl_packed_fc_op_test.py	2647	0644	edit dl rm
mod_op_test.py	1459	0644	edit dl rm
moments_op_test.py	1722	0644	edit dl rm
momentum_sgd_test.py	6480	0644	edit dl rm
mpi_test.py	8154	0644	edit dl rm
mul_gradient_benchmark.py	1509	0644	edit dl rm
negate_gradient_op_test.py	1518	0644	edit dl rm
ngram_ops_test.py	2327	0644	edit dl rm
normalize_op_test.py	1679	0644	edit dl rm
numpy_tile_op_test.py	1924	0644	edit dl rm
one_hot_ops_test.py	7478	0644	edit dl rm
onnx_while_test.py	3070	0644	edit dl rm
order_switch_test.py	1306	0644	edit dl rm
pack_ops_test.py	12634	0644	edit dl rm
pack_rnn_sequence_op_test.py	2891	0644	edit dl rm
pad_test.py	1377	0644	edit dl rm
partition_ops_test.py	6838	0644	edit dl rm
percentile_op_test.py	4427	0644	edit dl rm
piecewise_linear_transform_test.py	6187	0644	edit dl rm
pooling_test.py	16508	0644	edit dl rm
prepend_dim_test.py	1505	0644	edit dl rm
python_op_test.py	1312	0644	edit dl rm
quantile_test.py	3276	0644	edit dl rm
rand_quantization_op_speed_test.py	3128	0644	edit dl rm
rank_loss_operator_test.py	5752	0644	edit dl rm
rebatching_queue_test.py	9047	0644	edit dl rm
record_queue_test.py	3125	0644	edit dl rm
recurrent_network_test.py	14048	0644	edit dl rm
recurrent_net_executor_test.py	10922	0644	edit dl rm
reduce_ops_test.py	17341	0644	edit dl rm
reduction_ops_test.py	4664	0644	edit dl rm
reshape_ops_test.py	8211	0644	edit dl rm
resize_op_test.py	9417	0644	edit dl rm
rmac_regions_op_test.py	3178	0644	edit dl rm
rms_norm_op_test.py	1325	0644	edit dl rm
rnn_cell_test.py	59707	0644	edit dl rm
roi_align_rotated_op_test.py	7567	0644	edit dl rm
rowwise_counter_test.py	2205	0644	edit dl rm
scale_op_test.py	2177	0644	edit dl rm
segment_ops_test.py	25745	0644	edit dl rm
self_binning_histogram_test.py	12915	0644	edit dl rm
selu_op_test.py	3232	0644	edit dl rm
sequence_ops_test.py	16000	0644	edit dl rm
shape_inference_test.py	25708	0644	edit dl rm
sinusoid_position_encoding_op_test.py	2308	0644	edit dl rm
softmax_ops_test.py	23685	0644	edit dl rm
softplus_op_test.py	516	0644	edit dl rm
sparse_dropout_with_replacement_op_test.py	2885	0644	edit dl rm
sparse_gradient_checker_test.py	1294	0644	edit dl rm
sparse_itemwise_dropout_with_replacement_op_test.py	2913	0644	edit dl rm
sparse_lengths_sum_benchmark.py	4159	0644	edit dl rm
sparse_lp_regularizer_test.py	2553	0644	edit dl rm
sparse_normalize_test.py	3136	0644	edit dl rm
sparse_ops_test.py	3469	0644	edit dl rm
sparse_to_dense_mask_op_test.py	3693	0644	edit dl rm
spatial_bn_op_test.py	20182	0644	edit dl rm
specialized_segment_ops_test.py	11775	0644	edit dl rm
split_op_cost_test.py	8645	0644	edit dl rm
square_root_divide_op_test.py	2179	0644	edit dl rm
stats_ops_test.py	1789	0644	edit dl rm
stats_put_ops_test.py	6596	0644	edit dl rm
storm_test.py	6507	0644	edit dl rm
string_ops_test.py	4154	0644	edit dl rm
text_file_reader_test.py	2517	0644	edit dl rm
thresholded_relu_op_test.py	2323	0644	edit dl rm
tile_op_test.py	3887	0644	edit dl rm
top_k_test.py	9113	0644	edit dl rm
torch_integration_test.py	39941	0644	edit dl rm
transpose_op_test.py	2722	0644	edit dl rm
trigonometric_op_test.py	1715	0644	edit dl rm
unique_ops_test.py	2255	0644	edit dl rm
unique_uniform_fill_op_test.py	1335	0644	edit dl rm
unsafe_coalesce_test.py	2940	0644	edit dl rm
upsample_op_test.py	7308	0644	edit dl rm
utility_ops_test.py	15054	0644	edit dl rm
video_input_op_test.py	10503	0644	edit dl rm
weighted_multi_sample_test.py	1997	0644	edit dl rm
weighted_sample_test.py	2739	0644	edit dl rm
weighted_sum_test.py	3052	0644	edit dl rm
weight_scale_test.py	2057	0644	edit dl rm
wngrad_test.py	8279	0644	edit dl rm
__init__.py	0	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/caffe2/python/operator_test/dataset_ops_test.py (23847B)

import functools
import operator
import string

import hypothesis.strategies as st
import numpy as np
import numpy.testing as npt
from caffe2.python import core, dataset, workspace
from caffe2.python.dataset import Const
from caffe2.python.schema import (
    FeedRecord,
    FetchRecord,
    Field,
    List,
    Map,
    NewRecord,
    Scalar,
    Struct,
    from_blob_list,
)
from caffe2.python.test_util import TestCase
from hypothesis import given

def _assert_arrays_equal(actual, ref, err_msg):
    if ref.dtype.kind in ("S", "O", "U"):
        np.testing.assert_array_equal(actual, ref, err_msg=err_msg)
    else:
        np.testing.assert_allclose(actual, ref, atol=1e-4, rtol=1e-4, err_msg=err_msg)

def _assert_records_equal(actual, ref):
    assert isinstance(actual, Field)
    assert isinstance(ref, Field)
    b1 = actual.field_blobs()
    b2 = ref.field_blobs()
    assert len(b1) == len(b2), "Records have different lengths: %d vs. %d" % (
        len(b1),
        len(b2),
    )
    for name, d1, d2 in zip(ref.field_names(), b1, b2):
        _assert_arrays_equal(d1, d2, err_msg="Mismatch in field %s." % name)

@st.composite
def _sparse_features_map(draw, num_records, **kwargs):
    sparse_maps_lengths = draw(
        st.lists(
            st.integers(min_value=1, max_value=10),
            min_size=num_records,
            max_size=num_records,
        )
    )

sparse_maps_total_length = sum(sparse_maps_lengths)

sparse_keys = draw(
        st.lists(
            st.integers(min_value=1, max_value=100),
            min_size=sparse_maps_total_length,
            max_size=sparse_maps_total_length,
            unique=True,
        )
    )

sparse_values_lengths = draw(
        st.lists(
            st.integers(min_value=1, max_value=10),
            min_size=sparse_maps_total_length,
            max_size=sparse_maps_total_length,
        )
    )

total_sparse_values_lengths = sum(sparse_values_lengths)

sparse_values = draw(
        # max_value is max int64
        st.lists(
            st.integers(min_value=1, max_value=9223372036854775807),
            min_size=total_sparse_values_lengths,
            max_size=total_sparse_values_lengths,
        )
    )

return [
        sparse_maps_lengths,
        sparse_keys,
        sparse_values_lengths,
        sparse_values,
    ]

@st.composite
def _dense_features_map(draw, num_records, **kwargs):
    float_lengths = draw(
        st.lists(
            st.integers(min_value=1, max_value=10),
            min_size=num_records,
            max_size=num_records,
        )
    )

total_length = sum(float_lengths)

float_keys = draw(
        st.lists(
            st.integers(min_value=1, max_value=100),
            min_size=total_length,
            max_size=total_length,
            unique=True,
        )
    )

float_values = draw(
        st.lists(st.floats(), min_size=total_length, max_size=total_length)
    )

return [float_lengths, float_keys, float_values]

@st.composite
def _dataset(draw, min_elements=3, max_elements=10, **kwargs):
    schema = Struct(
        # Dense Features Map
        ("floats", Map(Scalar(np.int32), Scalar(np.float32))),
        # Sparse Features Map
        (
            "int_lists",
            Map(
                Scalar(np.int32),
                List(Scalar(np.int64)),
            ),
        ),
        # Complex Type
        ("text", Scalar(str)),
    )

num_records = draw(st.integers(min_value=min_elements, max_value=max_elements))

raw_dense_features_map_contents = draw(_dense_features_map(num_records))

raw_sparse_features_map_contents = draw(_sparse_features_map(num_records))

raw_text_contents = [
        draw(
            st.lists(
                st.text(alphabet=string.ascii_lowercase),
                min_size=num_records,
                max_size=num_records,
            )
        )
    ]

# Concatenate all raw contents to a single one
    contents_raw = (
        raw_dense_features_map_contents
        + raw_sparse_features_map_contents
        + raw_text_contents
    )

contents = from_blob_list(schema, contents_raw)

return (schema, contents, num_records)

class TestDatasetOps(TestCase):
    @given(_dataset())
    def test_pack_unpack(self, input):
        """
        Tests if packing and unpacking of the whole dataset is an identity.
        """
        (schema, contents, num_records) = input

dataset_fields = schema.field_names()

for pack_to_single_shared_ptr in (True, False):
            net = core.Net("pack_unpack_net")
            batch = NewRecord(net, contents)
            FeedRecord(batch, contents)

packed = net.PackRecords(
                batch.field_blobs(),
                1,
                fields=dataset_fields,
                pack_to_single_shared_ptr=pack_to_single_shared_ptr,
            )

unpacked = packed.UnPackRecords(
                [], len(dataset_fields), fields=dataset_fields
            )

workspace.RunNetOnce(net)

for initial_tensor, unpacked_tensor in zip(batch.field_blobs(), unpacked):
                npt.assert_array_equal(
                    workspace.FetchBlob(initial_tensor),
                    workspace.FetchBlob(unpacked_tensor),
                )

def test_dataset_ops(self):
        """
        1. Defining the schema of our dataset.

This example schema could represent, for example, a search query log.
        """
        schema = Struct(
            # fixed size vector, which will be stored as a matrix when batched
            ("dense", Scalar((np.float32, 3))),
            # could represent a feature map from feature ID to float value
            ("floats", Map(Scalar(np.int32), Scalar(np.float32))),
            # could represent a multi-valued categorical feature map
            (
                "int_lists",
                Map(
                    Scalar(np.int32),
                    List(Scalar(np.int64)),
                ),
            ),
            # could represent a multi-valued, weighted categorical feature map
            (
                "id_score_pairs",
                Map(
                    Scalar(np.int32),
                    Map(
                        Scalar(np.int64),
                        Scalar(np.float32),
                        keys_name="ids",
                        values_name="scores",
                    ),
                ),
            ),
            # additional scalar information
            (
                "metadata",
                Struct(
                    ("user_id", Scalar(np.int64)),
                    ("user_embed", Scalar((np.float32, 2))),
                    ("query", Scalar(str)),
                ),
            ),
        )
        """
        This is what the flattened fields for this schema look like, along
        with its type. Each one of these fields will be stored, read and
        written as a tensor.
        """
        expected_fields = [
            ("dense", (np.float32, 3)),
            ("floats:lengths", np.int32),
            ("floats:values:keys", np.int32),
            ("floats:values:values", np.float32),
            ("int_lists:lengths", np.int32),
            ("int_lists:values:keys", np.int32),
            ("int_lists:values:values:lengths", np.int32),
            ("int_lists:values:values:values", np.int64),
            ("id_score_pairs:lengths", np.int32),
            ("id_score_pairs:values:keys", np.int32),
            ("id_score_pairs:values:values:lengths", np.int32),
            ("id_score_pairs:values:values:values:ids", np.int64),
            ("id_score_pairs:values:values:values:scores", np.float32),
            ("metadata:user_id", np.int64),
            ("metadata:user_embed", (np.float32, 2)),
            ("metadata:query", str),
        ]
        zipped = zip(expected_fields, schema.field_names(), schema.field_types())
        for (ref_name, ref_type), name, dtype in zipped:
            self.assertEquals(ref_name, name)
            self.assertEquals(np.dtype(ref_type), dtype)
        """
        2. The contents of our dataset.

Contents as defined below could represent, for example, a log of
        search queries along with dense, sparse features and metadata.
        The dataset below has 3 top-level entries.
        """
        contents_raw = [
            # dense
            [[1.1, 1.2, 1.3], [2.1, 2.2, 2.3], [3.1, 3.2, 3.3]],
            # floats
            [1, 2, 3],  # len
            [11, 21, 22, 31, 32, 33],  # key
            [1.1, 2.1, 2.2, 3.1, 3.2, 3.3],  # value
            # int lists
            [2, 0, 1],  # len
            [11, 12, 31],  # key
            [2, 4, 3],  # value:len
            [111, 112, 121, 122, 123, 124, 311, 312, 313],  # value:value
            # id score pairs
            [1, 2, 2],  # len
            [11, 21, 22, 31, 32],  # key
            [1, 1, 2, 2, 3],  # value:len
            [111, 211, 221, 222, 311, 312, 321, 322, 323],  # value:ids
            [11.1, 21.1, 22.1, 22.2, 31.1, 31.2, 32.1, 32.2, 32.3],  # val:score
            # metadata
            [123, 234, 456],  # user_id
            [[0.2, 0.8], [0.5, 0.5], [0.7, 0.3]],  # user_embed
            ["dog posts", "friends who like to", "posts about ca"],  # query
        ]
        # convert the above content to ndarrays, checking against the schema
        contents = from_blob_list(schema, contents_raw)
        """
        3. Creating and appending to the dataset.
        We first create an empty dataset with the given schema.
        Then, a Writer is used to append these entries to the dataset.
        """
        ds = dataset.Dataset(schema)
        net = core.Net("init")
        with core.NameScope("init"):
            ds.init_empty(net)

content_blobs = NewRecord(net, contents)
            FeedRecord(content_blobs, contents)
            writer = ds.writer(init_net=net)
            writer.write_record(net, content_blobs)
        workspace.RunNetOnce(net)
        """
        4. Iterating through the dataset contents.

If we were to iterate through the top level entries of our dataset,
        this is what we should expect to see:
        """
        entries_raw = [
            (
                [[1.1, 1.2, 1.3]],  # dense
                [1],
                [11],
                [1.1],  # floats
                [2],
                [11, 12],
                [2, 4],
                [111, 112, 121, 122, 123, 124],  # intlst
                [1],
                [11],
                [1],
                [111],
                [11.1],  # id score pairs
                [123],
                [[0.2, 0.8]],
                ["dog posts"],  # metadata
            ),
            (
                [[2.1, 2.2, 2.3]],  # dense
                [2],
                [21, 22],
                [2.1, 2.2],  # floats
                [0],
                [],
                [],
                [],  # int list
                [2],
                [21, 22],
                [1, 2],
                [211, 221, 222],
                [21.1, 22.1, 22.2],
                [234],
                [[0.5, 0.5]],
                ["friends who like to"],  # metadata
            ),
            (
                [[3.1, 3.2, 3.3]],  # dense
                [3],
                [31, 32, 33],
                [3.1, 3.2, 3.3],  # floats
                [1],
                [31],
                [3],
                [311, 312, 313],  # int lst
                [2],
                [31, 32],
                [2, 3],
                [311, 312, 321, 322, 323],
                [31.1, 31.2, 32.1, 32.2, 32.3],  # id score list
                [456],
                [[0.7, 0.3]],
                ["posts about ca"],  # metadata
            ),
            # after the end of the dataset, we will keep getting empty vectors
            ([],) * 16,
            ([],) * 16,
        ]
        entries = [from_blob_list(schema, e) for e in entries_raw]
        """
        Let's go ahead and create the reading nets.
        We will run `read` net multiple times and assert that we are reading the
        entries the way we stated above.
        """
        read_init_net = core.Net("read_init")
        read_next_net = core.Net("read_next")
        reader = ds.reader(read_init_net)
        should_continue, batch = reader.read_record(read_next_net)

workspace.RunNetOnce(read_init_net)
        workspace.CreateNet(read_next_net, True)

for entry in entries:
            workspace.RunNet(str(read_next_net))
            actual = FetchRecord(batch)
            _assert_records_equal(actual, entry)
        """
        5. Reading/writing in a single plan

If all of operations on the data are expressible as Caffe2 operators,
        we don't need to load the data to python, iterating through the dataset
        in a single Plan.

Where we will process the dataset a little and store it in a second
        dataset. We can reuse the same Reader since it supports reset.
        """
        reset_net = core.Net("reset_net")
        reader.reset(reset_net)
        read_step, batch = reader.execution_step()
        """ We will add the line number * 1000 to the feature ids. """
        process_net = core.Net("process")
        line_no = Const(process_net, 0, dtype=np.int32)
        const_one = Const(process_net, 1000, dtype=np.int32)
        process_net.Add([line_no, const_one], [line_no])
        field = batch.floats.keys.get()
        process_net.Print(field, [])
        process_net.Add([field, line_no], field, broadcast=1, axis=0)
        """ Lets create a second dataset and append to it. """
        ds2 = dataset.Dataset(schema, name="dataset2")
        ds2.init_empty(reset_net)
        writer = ds2.writer(reset_net)
        writer.write_record(process_net, batch)
        # commit is not necessary for DatasetWriter but will add it for
        # generality of the example
        commit_net = core.Net("commit")
        writer.commit(commit_net)
        """ Time to create and run a plan which will do the processing """
        plan = core.Plan("process")
        plan.AddStep(core.execution_step("reset", reset_net))
        plan.AddStep(read_step.AddNet(process_net))
        plan.AddStep(core.execution_step("commit", commit_net))
        workspace.RunPlan(plan)
        """
        Now we should have dataset2 populated.
        """
        ds2_data = FetchRecord(ds2.content())
        field = ds2_data.floats.keys
        field.set(blob=field.get() - [1000, 2000, 2000, 3000, 3000, 3000])
        _assert_records_equal(contents, ds2_data)
        """
        6. Slicing a dataset

You can create a new schema from pieces of another schema and reuse
        the same data.
        """
        subschema = Struct(("top_level", schema.int_lists.values))
        int_list_contents = contents.int_lists.values.field_names()
        self.assertEquals(len(subschema.field_names()), len(int_list_contents))
        """
        7. Random Access a dataset

"""
        read_init_net = core.Net("read_init")
        read_next_net = core.Net("read_next")

idx = np.array([2, 1, 0])
        indices_blob = Const(read_init_net, idx, name="indices")
        reader = ds.random_reader(read_init_net, indices_blob)
        reader.computeoffset(read_init_net)

should_stop, batch = reader.read_record(read_next_net)

workspace.CreateNet(read_init_net, True)
        workspace.RunNetOnce(read_init_net)

workspace.CreateNet(read_next_net, True)

for i in range(len(entries)):
            k = idx[i] if i in idx else i
            entry = entries[k]
            workspace.RunNet(str(read_next_net))
            actual = FetchRecord(batch)
            _assert_records_equal(actual, entry)
        workspace.RunNet(str(read_next_net))
        self.assertEquals(True, workspace.FetchBlob(should_stop))
        """
        8. Random Access a dataset with loop_over = true

"""
        read_init_net = core.Net("read_init")
        read_next_net = core.Net("read_next")

idx = np.array([2, 1, 0])
        indices_blob = Const(read_init_net, idx, name="indices")
        reader = ds.random_reader(read_init_net, indices_blob, loop_over=True)
        reader.computeoffset(read_init_net)

should_stop, batch = reader.read_record(read_next_net)

workspace.CreateNet(read_init_net, True)
        workspace.RunNetOnce(read_init_net)

workspace.CreateNet(read_next_net, True)

for _ in range(len(entries) * 3):
            workspace.RunNet(str(read_next_net))
            self.assertEquals(False, workspace.FetchBlob(should_stop))
        """
        9. Sort and shuffle a dataset

This sort the dataset using the score of a certain column,
        and then shuffle within each chunk of size batch_size * shuffle_size
        before shuffling the chunks.

"""
        read_init_net = core.Net("read_init")
        read_next_net = core.Net("read_next")

reader = ds.random_reader(read_init_net)
        reader.sort_and_shuffle(read_init_net, "int_lists:lengths", 1, 2)
        reader.computeoffset(read_init_net)

should_continue, batch = reader.read_record(read_next_net)

workspace.CreateNet(read_init_net, True)
        workspace.RunNetOnce(read_init_net)

workspace.CreateNet(read_next_net, True)

expected_idx = np.array([2, 1, 0])
        for i in range(len(entries)):
            k = expected_idx[i] if i in expected_idx else i
            entry = entries[k]
            workspace.RunNet(str(read_next_net))
            actual = FetchRecord(batch)
            _assert_records_equal(actual, entry)

"""
        Trim a dataset
        """
        trim_net = core.Net("trim_ds")
        ds.trim(trim_net, multiple_of=2)
        workspace.RunNetOnce(trim_net)
        trimmed = FetchRecord(ds.content())
        EXPECTED_SIZES = [2, 2, 3, 3, 2, 2, 2, 6, 2, 3, 3, 4, 4, 2, 2, 2]
        actual_sizes = [d.shape[0] for d in trimmed.field_blobs()]
        self.assertEquals(EXPECTED_SIZES, actual_sizes)

def test_last_n_window_ops(self):
        collect_net = core.Net("collect_net")
        collect_net.GivenTensorFill(
            [],
            "input",
            shape=[3, 2],
            values=[1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
        )
        input_array = np.array(list(range(1, 7)), dtype=np.float32).reshape(3, 2)

workspace.CreateBlob("output")
        workspace.FeedBlob("next", np.array(0, dtype=np.int32))
        collect_net.LastNWindowCollector(
            ["output", "next", "input"],
            ["output", "next"],
            num_to_collect=7,
        )
        plan = core.Plan("collect_data")
        plan.AddStep(core.execution_step("collect_data", [collect_net], num_iter=1))
        workspace.RunPlan(plan)
        reference_result = workspace.FetchBlob("output")
        npt.assert_array_equal(input_array, reference_result)

plan = core.Plan("collect_data")
        plan.AddStep(core.execution_step("collect_data", [collect_net], num_iter=2))
        workspace.RunPlan(plan)
        reference_result = workspace.FetchBlob("output")
        npt.assert_array_equal(input_array[[1, 2, 2, 0, 1, 2, 0]], reference_result)

plan = core.Plan("collect_data")
        plan.AddStep(core.execution_step("collect_data", [collect_net], num_iter=3))
        workspace.RunPlan(plan)
        reference_result = workspace.FetchBlob("output")
        npt.assert_array_equal(input_array[[2, 0, 1, 2, 2, 0, 1]], reference_result)

def test_last_n_window_ops_shape_inference(self):
        collect_net = core.Net("collect_net")
        collect_net.GivenTensorFill(
            [],
            "input",
            shape=[3, 2],
            values=[1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
        )

workspace.CreateBlob("output")
        workspace.FeedBlob("next", np.array(0, dtype=np.int32))
        collect_net.LastNWindowCollector(
            ["output", "next", "input"],
            ["output", "next"],
            num_to_collect=7,
        )
        (shapes, types) = workspace.InferShapesAndTypes([collect_net])
        workspace.RunNetOnce(collect_net)

self.assertTrue(
            np.array_equal(
                shapes["output"], np.array([7, workspace.blobs["output"].shape[1]])
            )
        )

def test_last_n_window_ops_shape_inference_4d_input(self):
        input_shape = [3, 2, 4, 5]
        collect_net = core.Net("collect_net")
        collect_net.GivenTensorFill(
            [],
            "input",
            shape=input_shape,
            values=[
                float(val) for val in range(functools.reduce(operator.mul, input_shape))
            ],
        )

workspace.CreateBlob("output")
        workspace.FeedBlob("next", np.array(0, dtype=np.int32))
        collect_net.LastNWindowCollector(
            ["output", "next", "input"],
            ["output", "next"],
            num_to_collect=7,
        )
        (shapes, types) = workspace.InferShapesAndTypes([collect_net])
        workspace.RunNetOnce(collect_net)

self.assertTrue(
            np.array_equal(
                shapes["output"], np.array([7, *list(workspace.blobs["output"].shape[1:])])
            )
        )

def test_collect_tensor_ops(self):
        init_net = core.Net("init_net")
        blobs = ["blob_1", "blob_2", "blob_3"]
        bvec_map = {}
        ONE = init_net.ConstantFill([], "ONE", shape=[1, 2], value=1)
        for b in blobs:
            init_net.ConstantFill([], [b], shape=[1, 2], value=0)
            bvec_map[b] = b + "_vec"
            init_net.CreateTensorVector([], [bvec_map[b]])

reader_net = core.Net("reader_net")
        for b in blobs:
            reader_net.Add([b, ONE], [b])

collect_net = core.Net("collect_net")
        num_to_collect = 1000
        max_example_to_cover = 100000
        bvec = [bvec_map[b] for b in blobs]
        collect_net.CollectTensor(
            bvec + blobs,
            bvec,
            num_to_collect=num_to_collect,
        )

print("Collect Net Proto: {}".format(collect_net.Proto()))

plan = core.Plan("collect_data")
        plan.AddStep(core.execution_step("collect_init", init_net))
        plan.AddStep(
            core.execution_step(
                "collect_data", [reader_net, collect_net], num_iter=max_example_to_cover
            )
        )
        workspace.RunPlan(plan)

# concat the collected tensors
        concat_net = core.Net("concat_net")
        bconcated_map = {}
        bsize_map = {}
        for b in blobs:
            bconcated_map[b] = b + "_concated"
            bsize_map[b] = b + "_size"
            concat_net.ConcatTensorVector([bvec_map[b]], [bconcated_map[b]])
            concat_net.TensorVectorSize([bvec_map[b]], [bsize_map[b]])

workspace.RunNetOnce(concat_net)

# check data
        reference_result = workspace.FetchBlob(bconcated_map[blobs[0]])
        self.assertEqual(
            reference_result.shape, (min(num_to_collect, max_example_to_cover), 2)
        )
        size = workspace.FetchBlob(bsize_map[blobs[0]])
        self.assertEqual(tuple(), size.shape)
        self.assertEqual(min(num_to_collect, max_example_to_cover), size.item())

hist, _ = np.histogram(
            reference_result[:, 0], bins=10, range=(1, max_example_to_cover)
        )
        print("Sample histogram: {}".format(hist))

self.assertTrue(all(hist > 0.6 * (num_to_collect / 10)))
        for i in range(1, len(blobs)):
            result = workspace.FetchBlob(bconcated_map[blobs[i]])
            self.assertEqual(reference_result.tolist(), result.tolist())

if __name__ == "__main__":
    import unittest

unittest.main()

cmd: