FM / layers

/usr/local/lib64/python3.6/site-packages/caffe2/python/layers/

/usr/local/lib64/python3.6/site-packages/caffe2/python/layers

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
adaptive_weight.py	5687	0644	edit dl rm
add_bias.py	1396	0644	edit dl rm
arc_cosine_feature_map.py	7345	0644	edit dl rm
batch_huber_loss.py	3523	0644	edit dl rm
batch_lr_loss.py	11577	0644	edit dl rm
batch_mse_loss.py	2333	0644	edit dl rm
batch_normalization.py	3823	0644	edit dl rm
batch_sigmoid_cross_entropy_loss.py	1483	0644	edit dl rm
batch_softmax_loss.py	4580	0644	edit dl rm
blob_weighted_sum.py	2219	0644	edit dl rm
bpr_loss.py	1499	0644	edit dl rm
bucket_weighted.py	2355	0644	edit dl rm
build_index.py	1937	0644	edit dl rm
concat.py	4849	0644	edit dl rm
constant_weight.py	1208	0644	edit dl rm
conv.py	5050	0644	edit dl rm
dropout.py	1410	0644	edit dl rm
fc.py	9296	0644	edit dl rm
fc_without_bias.py	1954	0644	edit dl rm
fc_with_bootstrap.py	12788	0644	edit dl rm
feature_sparse_to_dense.py	14361	0644	edit dl rm
functional.py	4875	0644	edit dl rm
gather_record.py	3260	0644	edit dl rm
homotopy_weight.py	4306	0644	edit dl rm
label_smooth.py	3507	0644	edit dl rm
last_n_window_collector.py	2392	0644	edit dl rm
layers.py	17412	0644	edit dl rm
layer_normalization.py	4291	0644	edit dl rm
margin_rank_loss.py	1951	0644	edit dl rm
merge_id_lists.py	1500	0644	edit dl rm
pairwise_similarity.py	3549	0644	edit dl rm
position_weighted.py	2066	0644	edit dl rm
random_fourier_features.py	3187	0644	edit dl rm
reservoir_sampling.py	3013	0644	edit dl rm
sampling_train.py	2210	0644	edit dl rm
sampling_trainable_mixin.py	1366	0644	edit dl rm
select_record_by_context.py	2381	0644	edit dl rm
semi_random_features.py	5809	0644	edit dl rm
sparse_dropout_with_replacement.py	3943	0644	edit dl rm
sparse_feature_hash.py	4618	0644	edit dl rm
sparse_itemwise_dropout_with_replacement.py	3944	0644	edit dl rm
sparse_lookup.py	22170	0644	edit dl rm
split.py	2257	0644	edit dl rm
tags.py	4114	0644	edit dl rm
uniform_sampling.py	2779	0644	edit dl rm
__init__.py	943	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/caffe2/python/layers/layer_normalization.py (4291B)

from caffe2.python import schema
from caffe2.python.layers.layers import ModelLayer

import numpy as np

class LayerNormalization(ModelLayer):
    def __init__(
        self,
        model,
        input_record,
        name='layer_normalization',
        scale_optim=None,
        bias_optim=None,
        epsilon=1e-4,
        axis=1,
        use_layer_norm_op=True,
        scale_init_value=1.0,
        **kwargs
    ):
        super(LayerNormalization, self).__init__(
            model, name, input_record, **kwargs)

assert isinstance(input_record, schema.Scalar), (
            "Incorrect input type: {}".format(input_record))

self.input_shape = input_record.field_type().shape
        self.axis = axis

assert len(self.input_shape) >= 1, (
            "This layer supports only >= 2D tensors")
        input_dims = self.input_shape[0]

self.output_schema = schema.Scalar(
            (np.float32, self.input_shape),
            self.get_next_blob_reference('output')
        )

self.scale = self.create_param(param_name='scale',
                                       shape=[input_dims],
                                       initializer=('ConstantFill', {'value': scale_init_value}),
                                       optimizer=scale_optim)
        self.bias = self.create_param(param_name='bias',
                                       shape=[input_dims],
                                       initializer=('ConstantFill', {'value': 0.0}),
                                       optimizer=bias_optim)
        self.use_layer_norm_op = use_layer_norm_op

if self.use_layer_norm_op:
            self.epsilon = epsilon
        else:
            assert len(self.input_shape) == 1, (
                "When using alternative implementation, "
                "input data can only be 2D"
            )
            self.epsilon = model.maybe_add_global_constant(
                "%s_epsilon" % self.name, float(epsilon)
            )

def add_ops_with_layer_norm_op(self, net):
        input_blob = self.input_record.field_blobs()
        ln_output = self.output_schema.field_blobs()

output_blobs = [net.NextScopedBlob('ln_output'), net.NextScopedBlob('ln_mean'),
                        net.NextScopedBlob('ln_stdev')]

normalized, mean, stdev = net.LayerNorm(input_blob,
            output_blobs,
            axis=self.axis,
            epsilon=self.epsilon)

scaled = net.Mul(
            [normalized, self.scale],
            [net.NextScopedBlob('ln_scaled')],
            broadcast=1,
            axis=self.axis,
        )

net.Add(
            [scaled, self.bias],
            ln_output,
            broadcast=1,
            axis=self.axis,
        )

def add_ops_without_layer_norm_op(self, net):
        # two issues here:
        #  1. use multiple ops to replace the function of LayerNorm
        #  2. do not use legacy broadcast
        ln_output = net.NextScopedBlob("ln_output")
        ln_mean = net.NextScopedBlob("ln_mean")
        ln_stdev = net.NextScopedBlob("ln_stdev")
        ln_mean_arr = net.NextScopedBlob("ln_mean_arr")
        net.ReduceBackMean(self.input_record.field_blobs(), [ln_mean_arr])
        net.ExpandDims([ln_mean_arr], [ln_mean], dims=[1])
        ln_centered = net.NextScopedBlob("ln_centered")
        net.Sub(self.input_record.field_blobs() + [ln_mean], [ln_centered])
        ln_sqr = net.NextScopedBlob("ln_sqr")
        net.Sqr([ln_centered], [ln_sqr])
        ln_sqr_mean = net.NextScopedBlob("ln_sqr_mean")
        net.ReduceBackMean([ln_sqr], [ln_sqr_mean])
        ln_var = net.NextScopedBlob("ln_var")
        net.Add([ln_sqr_mean, self.epsilon], ln_var)
        ln_std_arr = net.NextScopedBlob("ln_std_arr")
        net.Pow([ln_var], [ln_std_arr], exponent=0.5)
        net.ExpandDims([ln_std_arr], [ln_stdev], dims=[1])
        net.Div([ln_centered, ln_stdev], [ln_output])
        ln_scaled = net.NextScopedBlob("ln_scaled")
        net.Mul([ln_output, self.scale], [ln_scaled])
        net.Add([ln_scaled, self.bias], self.output_schema.field_blobs())

def add_ops(self, net):
        if self.use_layer_norm_op:
            self.add_ops_with_layer_norm_op(net)
        else:
            self.add_ops_without_layer_norm_op(net)

from caffe2.python import schema from caffe2.python.layers.layers import ModelLayer import numpy as np class LayerNormalization(ModelLayer): def __init__( self, model, input_record, name='layer_normalization', scale_optim=None, bias_optim=None, epsilon=1e-4, axis=1, use_layer_norm_op=True, scale_init_value=1.0, **kwargs ): super(LayerNormalization, self).__init__( model, name, input_record, **kwargs) assert isinstance(input_record, schema.Scalar), ( "Incorrect input type: {}".format(input_record)) self.input_shape = input_record.field_type().shape self.axis = axis assert len(self.input_shape) >= 1, ( "This layer supports only >= 2D tensors") input_dims = self.input_shape[0] self.output_schema = schema.Scalar( (np.float32, self.input_shape), self.get_next_blob_reference('output') ) self.scale = self.create_param(param_name='scale', shape=[input_dims], initializer=('ConstantFill', {'value': scale_init_value}), optimizer=scale_optim) self.bias = self.create_param(param_name='bias', shape=[input_dims], initializer=('ConstantFill', {'value': 0.0}), optimizer=bias_optim) self.use_layer_norm_op = use_layer_norm_op if self.use_layer_norm_op: self.epsilon = epsilon else: assert len(self.input_shape) == 1, ( "When using alternative implementation, " "input data can only be 2D" ) self.epsilon = model.maybe_add_global_constant( "%s_epsilon" % self.name, float(epsilon) ) def add_ops_with_layer_norm_op(self, net): input_blob = self.input_record.field_blobs() ln_output = self.output_schema.field_blobs() output_blobs = [net.NextScopedBlob('ln_output'), net.NextScopedBlob('ln_mean'), net.NextScopedBlob('ln_stdev')] normalized, mean, stdev = net.LayerNorm(input_blob, output_blobs, axis=self.axis, epsilon=self.epsilon) scaled = net.Mul( [normalized, self.scale], [net.NextScopedBlob('ln_scaled')], broadcast=1, axis=self.axis, ) net.Add( [scaled, self.bias], ln_output, broadcast=1, axis=self.axis, ) def add_ops_without_layer_norm_op(self, net): # two issues here: # 1. use multiple ops to replace the function of LayerNorm # 2. do not use legacy broadcast ln_output = net.NextScopedBlob("ln_output") ln_mean = net.NextScopedBlob("ln_mean") ln_stdev = net.NextScopedBlob("ln_stdev") ln_mean_arr = net.NextScopedBlob("ln_mean_arr") net.ReduceBackMean(self.input_record.field_blobs(), [ln_mean_arr]) net.ExpandDims([ln_mean_arr], [ln_mean], dims=[1]) ln_centered = net.NextScopedBlob("ln_centered") net.Sub(self.input_record.field_blobs() + [ln_mean], [ln_centered]) ln_sqr = net.NextScopedBlob("ln_sqr") net.Sqr([ln_centered], [ln_sqr]) ln_sqr_mean = net.NextScopedBlob("ln_sqr_mean") net.ReduceBackMean([ln_sqr], [ln_sqr_mean]) ln_var = net.NextScopedBlob("ln_var") net.Add([ln_sqr_mean, self.epsilon], ln_var) ln_std_arr = net.NextScopedBlob("ln_std_arr") net.Pow([ln_var], [ln_std_arr], exponent=0.5) net.ExpandDims([ln_std_arr], [ln_stdev], dims=[1]) net.Div([ln_centered, ln_stdev], [ln_output]) ln_scaled = net.NextScopedBlob("ln_scaled") net.Mul([ln_output, self.scale], [ln_scaled]) net.Add([ln_scaled, self.bias], self.output_schema.field_blobs()) def add_ops(self, net): if self.use_layer_norm_op: self.add_ops_with_layer_norm_op(net) else: self.add_ops_without_layer_norm_op(net)