FM / utils

/usr/local/lib64/python3.6/site-packages/torch/include/torch/csrc/utils/

/usr/local/lib64/python3.6/site-packages/torch/include/torch/csrc/utils

Name	Size	Mode	Actions
auto_gil.h	1034	0644	edit dl rm
byte_order.h	2484	0644	edit dl rm
crash_handler.h	1146	0644	edit dl rm
cuda_enabled.h	154	0644	edit dl rm
cuda_lazy_init.h	974	0644	edit dl rm
disable_torch_function.h	863	0644	edit dl rm
disallow_copy.h	103	0644	edit dl rm
init.h	324	0644	edit dl rm
invalid_arguments.h	302	0644	edit dl rm
memory.h	1173	0644	edit dl rm
numpy_stub.h	399	0644	edit dl rm
object_ptr.h	1329	0644	edit dl rm
out_types.h	294	0644	edit dl rm
pybind.h	7966	0644	edit dl rm
pycfunction_helpers.h	209	0644	edit dl rm
python_arg_parser.h	30753	0644	edit dl rm
python_compat.h	2904	0644	edit dl rm
python_dispatch.h	174	0644	edit dl rm
python_numbers.h	5064	0644	edit dl rm
python_scalars.h	2928	0644	edit dl rm
python_strings.h	4592	0644	edit dl rm
python_stub.h	56	0644	edit dl rm
python_tuples.h	684	0644	edit dl rm
six.h	1425	0644	edit dl rm
structseq.h	153	0644	edit dl rm
tensor_apply.h	431	0644	edit dl rm
tensor_dtypes.h	244	0644	edit dl rm
tensor_flatten.h	2780	0644	edit dl rm
tensor_layouts.h	107	0644	edit dl rm
tensor_list.h	196	0644	edit dl rm
tensor_memoryformats.h	113	0644	edit dl rm
tensor_new.h	1807	0644	edit dl rm
tensor_numpy.h	542	0644	edit dl rm
tensor_qschemes.h	186	0644	edit dl rm
tensor_types.h	491	0644	edit dl rm
throughput_benchmark-inl.h	5210	0644	edit dl rm
throughput_benchmark.h	6868	0644	edit dl rm
variadic.h	4394	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/include/torch/csrc/utils/throughput_benchmark.h (6868B)

#pragma once

#include <ATen/core/ivalue.h>
#include <torch/csrc/jit/api/module.h>
#include <pybind11/pybind11.h>

#include <torch/csrc/jit/python/pybind_utils.h>

#include <iostream>
#include <memory>
#include <string>
#include <vector>

namespace py = pybind11;

namespace torch {
namespace throughput_benchmark {

/**
 * The struct is used to provide results of a benchmark to the caller
 * In the future all additional statics should be added here.
 */
struct BenchmarkExecutionStats {
  float latency_avg_ms{-1};
  int64_t num_iters{-1};
};

std::ostream& operator<<(std::ostream& os, const BenchmarkExecutionStats& value);

/**
 * Use this struct in order to configure a throughput benchmark run.
 * This struct should include parameters related to threading, batching, number
 * of iterations, warm-up, etc. More configs can be added as needed.
 * General rule here is that only things that c++ must(!) to be aware of should
 * be here. If we can keep other parts in python, we should keep them there.
 * This is typical for things that are not perf critical and don't affect
 * execution statistics benchmark returns.
 */
struct BenchmarkConfig {
 public:
  // Calling threads are those threads that are calling into a module in
  // parallel.
  int num_calling_threads{1};
  // Worker threads are not supported yet. This is just an example that we plan
  // to support some sort of multi-threaded forward calls. We may change this
  // setting in the future to support different intra and inter op parallelizm
  // which is not available in PyTorch yet
  int num_worker_threads{1};
  // Warmup iters are used to make sure we run a module a few times before
  // actually measuring things. This way we avoid cold caches and any other
  // similar problems
  int num_warmup_iters{1};
  // Number of iterations the benchmark should run with. This number is separate
  // from the warmup iterations
  int64_t num_iters{100};
  // If set autograd profiler will be enabled. I.e. this variable would be created
  // before the main benchmark loop (but after the warmup):
  // RecordProfile guard(profiler_output_path);
  std::string profiler_output_path{""};
};

namespace detail {

/**
 * A helper class to abstract out different models we test throughput of
 */
template <class Input, class Output, class Model>
class BenchmarkHelper {
public:
  BenchmarkHelper();
  // NOLINTNEXTLINE(modernize-pass-by-value)
  explicit BenchmarkHelper(Model model): model_(model), initialized_(true) {}

// This method to be used in benchmark() method
  // Note that there is no result. This way we don't have to call this under GIL
  // even when running in the nn.Module mode. Otherwise destructor of the result
  // would race with Python
  void runOnce(Input&&) const;
  // This method is to be used when calling from Python dirrectly
  Output runOnce(py::args&&, py::kwargs&&) const;
  // Aggregate input in the format Model expects in order to avoid further
  // conversions at the benchmark time
  void addInput(py::args&&, py::kwargs&&);
  void addInput(Input&&);
  BenchmarkExecutionStats benchmark(const BenchmarkConfig& config) const;

bool initialized() const { return initialized_; }

// Destructor doesn't require the GIL because it is going to be executed on
  // the PyThon thread
  std::vector<Input> inputs_;
  Model model_;
  bool initialized_{false};
};

struct C10_HIDDEN ModuleInput {
  ModuleInput(ModuleInput&& other) = default;

ModuleInput(const ModuleInput&) = delete;
  ModuleInput& operator=(ModuleInput& other) = delete;
  ModuleInput& operator=(ModuleInput&& other) = delete;

ModuleInput(py::args&& args, py::kwargs&& kwargs)
      : args(std::move(args)), kwargs(std::move(kwargs)) {}

py::args args;
  py::kwargs kwargs;
};
typedef py::object ModuleOutput;
typedef std::vector<at::IValue> ScriptModuleInput;
typedef at::IValue ScriptModuleOutput;

template<class Input>
Input cloneInput(const Input& input);

typedef BenchmarkHelper<
    ScriptModuleInput,
    at::IValue,
    jit::Module>
    ScriptModuleBenchmark;
template <>
inline BenchmarkHelper<ScriptModuleInput, at::IValue, jit::Module>::BenchmarkHelper()
  : model_("Module", std::make_shared<jit::CompilationUnit>()),
    initialized_(false) {}
typedef BenchmarkHelper<ModuleInput, py::object, py::object> ModuleBenchmark;
template <>
inline BenchmarkHelper<ModuleInput, py::object, py::object>::BenchmarkHelper()
  : initialized_(false) {}

template <>
void ScriptModuleBenchmark::runOnce(
    ScriptModuleInput&& input) const;

template <>
ScriptModuleOutput ScriptModuleBenchmark::runOnce(
    py::args&& args,
    py::kwargs&& kwargs) const;

template <>
void ModuleBenchmark::runOnce(ModuleInput&& input) const;

template <>
ModuleOutput ModuleBenchmark::runOnce(py::args&& args, py::kwargs&& kwargs)
    const;

template <>
void ScriptModuleBenchmark::addInput(py::args&& args, py::kwargs&& kwargs);
template <>
void ScriptModuleBenchmark::addInput(ScriptModuleInput&& input);

template <>
void ModuleBenchmark::addInput(py::args&& args, py::kwargs&& kwargs);

} // namespace detail

/**
 * This class is a small c++ component responsible for executing a PyTorch
 * module under an inference server like load. It can emulate multiple calling
 * threads to a single module provided. In the future we plan to enhance this
 * component to support inter and intra-op parallelism as well as multiple
 * models running in a single process.
 *
 * For current available configurations refer to the BenchmkarConfig
 * documentation
 *
 * The class supports working with either nn.Module or ScriptModule.
 * Under the hood it just dispatches to corresponding specialization of
 * class BenchmarkHelper<Input, Output, Model>
 */
class C10_HIDDEN ThroughputBenchmark {
 public:
  explicit ThroughputBenchmark(jit::Module module);
  explicit ThroughputBenchmark(py::object module);

// Add one more input example. This input example should be in the exact
  // format the module under test expects. It is responsibility of the module to
  // perform any such format checks, the benchmark doesn't perform any
  // validation of its own
  void addInput(py::args args, py::kwargs kwargs);

// Equivalent to just running the model dirrectly on the given input
  py::object runOnce(py::args&& args, py::kwargs&& kwargs);

// The main method of the class allows to perform a multi-threaded benchmark
  // It returns BenchmarkExecutionStats object with a lot of useful statistics
  // about runtime execution. We can enhance this class in the future to provide
  // more information to the user
  BenchmarkExecutionStats benchmark(const BenchmarkConfig& config) const;

private:
  detail::ScriptModuleBenchmark script_module_;
  detail::ModuleBenchmark module_;
};
} // namespace throughput benchmark
} // namepsace torch

#include <torch/csrc/utils/throughput_benchmark-inl.h>

cmd: