FM / layers

/usr/local/lib64/python3.6/site-packages/caffe2/python/layers/

/usr/local/lib64/python3.6/site-packages/caffe2/python/layers

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
adaptive_weight.py	5687	0644	edit dl rm
add_bias.py	1396	0644	edit dl rm
arc_cosine_feature_map.py	7345	0644	edit dl rm
batch_huber_loss.py	3523	0644	edit dl rm
batch_lr_loss.py	11577	0644	edit dl rm
batch_mse_loss.py	2333	0644	edit dl rm
batch_normalization.py	3823	0644	edit dl rm
batch_sigmoid_cross_entropy_loss.py	1483	0644	edit dl rm
batch_softmax_loss.py	4580	0644	edit dl rm
blob_weighted_sum.py	2219	0644	edit dl rm
bpr_loss.py	1499	0644	edit dl rm
bucket_weighted.py	2355	0644	edit dl rm
build_index.py	1937	0644	edit dl rm
concat.py	4849	0644	edit dl rm
constant_weight.py	1208	0644	edit dl rm
conv.py	5050	0644	edit dl rm
dropout.py	1410	0644	edit dl rm
fc.py	9296	0644	edit dl rm
fc_without_bias.py	1954	0644	edit dl rm
fc_with_bootstrap.py	12788	0644	edit dl rm
feature_sparse_to_dense.py	14361	0644	edit dl rm
functional.py	4875	0644	edit dl rm
gather_record.py	3260	0644	edit dl rm
homotopy_weight.py	4306	0644	edit dl rm
label_smooth.py	3507	0644	edit dl rm
last_n_window_collector.py	2392	0644	edit dl rm
layers.py	17412	0644	edit dl rm
layer_normalization.py	4291	0644	edit dl rm
margin_rank_loss.py	1951	0644	edit dl rm
merge_id_lists.py	1500	0644	edit dl rm
pairwise_similarity.py	3549	0644	edit dl rm
position_weighted.py	2066	0644	edit dl rm
random_fourier_features.py	3187	0644	edit dl rm
reservoir_sampling.py	3013	0644	edit dl rm
sampling_train.py	2210	0644	edit dl rm
sampling_trainable_mixin.py	1366	0644	edit dl rm
select_record_by_context.py	2381	0644	edit dl rm
semi_random_features.py	5809	0644	edit dl rm
sparse_dropout_with_replacement.py	3943	0644	edit dl rm
sparse_feature_hash.py	4618	0644	edit dl rm
sparse_itemwise_dropout_with_replacement.py	3944	0644	edit dl rm
sparse_lookup.py	22170	0644	edit dl rm
split.py	2257	0644	edit dl rm
tags.py	4114	0644	edit dl rm
uniform_sampling.py	2779	0644	edit dl rm
__init__.py	943	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/caffe2/python/layers/homotopy_weight.py (4306B)

# @package homotopy_weight
# Module caffe2.fb.python.layers.homotopy_weight

from caffe2.python import core, schema
from caffe2.python.layers.layers import ModelLayer
import numpy as np
import logging
logger = logging.getLogger(__name__)
'''
Homotopy Weighting between two weights x, y by doing:
    alpha x + beta y
where alpha is a decreasing scalar parameter ranging from [min, max] (default,
[0, 1]), and alpha + beta = max + min, which means that beta is increasing in
the range [min, max];

Homotopy methods first solves an "easy" problem (one to which the solution is
well known), and is gradually transformed into the target problem
'''

class HomotopyWeight(ModelLayer):
    def __init__(
        self,
        model,
        input_record,
        name='homotopy_weight',
        min_weight=0.,
        max_weight=1.,
        half_life=1e6,
        quad_life=3e6,
        atomic_iter=None,
        **kwargs
    ):
        super(HomotopyWeight,
              self).__init__(model, name, input_record, **kwargs)
        self.output_schema = schema.Scalar(
            np.float32, self.get_next_blob_reference('homotopy_weight')
        )
        data = self.input_record.field_blobs()
        assert len(data) == 2
        self.x = data[0]
        self.y = data[1]
        # TODO: currently model building does not have access to iter counter or
        # learning rate; it's added at optimization time;
        self.use_external_iter = (atomic_iter is not None)
        self.atomic_iter = (
            atomic_iter if self.use_external_iter else self.create_atomic_iter()
        )
        # to map lr to [min, max]; alpha = scale * lr + offset
        assert max_weight > min_weight
        self.scale = float(max_weight - min_weight)
        self.offset = self.model.add_global_constant(
            '%s_offset_1dfloat' % self.name, float(min_weight)
        )
        self.gamma, self.power = self.solve_inv_lr_params(half_life, quad_life)

def solve_inv_lr_params(self, half_life, quad_life):
        # ensure that the gamma, power is solvable
        assert half_life > 0
        # convex monotonically decreasing
        assert quad_life > 2 * half_life
        t = float(quad_life) / float(half_life)
        x = t * (1.0 + np.sqrt(2.0)) / 2.0 - np.sqrt(2.0)
        gamma = (x - 1.0) / float(half_life)
        power = np.log(2.0) / np.log(x)
        logger.info(
            'homotopy_weighting: found lr param: gamma=%g, power=%g' %
            (gamma, power)
        )
        return gamma, power

def create_atomic_iter(self):
        self.mutex = self.create_param(
            param_name=('%s_mutex' % self.name),
            shape=None,
            initializer=('CreateMutex', ),
            optimizer=self.model.NoOptim,
        )
        self.atomic_iter = self.create_param(
            param_name=('%s_atomic_iter' % self.name),
            shape=[1],
            initializer=(
                'ConstantFill', {
                    'value': 0,
                    'dtype': core.DataType.INT64
                }
            ),
            optimizer=self.model.NoOptim,
        )
        return self.atomic_iter

def update_weight(self, net):
        alpha = net.NextScopedBlob('alpha')
        beta = net.NextScopedBlob('beta')
        lr = net.NextScopedBlob('lr')
        comp_lr = net.NextScopedBlob('complementary_lr')
        scaled_lr = net.NextScopedBlob('scaled_lr')
        scaled_comp_lr = net.NextScopedBlob('scaled_complementary_lr')
        if not self.use_external_iter:
            net.AtomicIter([self.mutex, self.atomic_iter], [self.atomic_iter])
        net.LearningRate(
            [self.atomic_iter],
            [lr],
            policy='inv',
            gamma=self.gamma,
            power=self.power,
            base_lr=1.0,
        )
        net.Sub([self.model.global_constants['ONE'], lr], [comp_lr])
        net.Scale([lr], [scaled_lr], scale=self.scale)
        net.Scale([comp_lr], [scaled_comp_lr], scale=self.scale)
        net.Add([scaled_lr, self.offset], [alpha])
        net.Add([scaled_comp_lr, self.offset], [beta])
        return alpha, beta

def add_ops(self, net):
        alpha, beta = self.update_weight(net)
        # alpha x + beta y
        net.WeightedSum([self.x, alpha, self.y, beta], self.output_schema())

cmd: