FM / utils

/usr/local/lib64/python3.6/site-packages/torch/include/torch/csrc/utils/

/usr/local/lib64/python3.6/site-packages/torch/include/torch/csrc/utils

Name	Size	Mode	Actions
auto_gil.h	1034	0644	edit dl rm
byte_order.h	2484	0644	edit dl rm
crash_handler.h	1146	0644	edit dl rm
cuda_enabled.h	154	0644	edit dl rm
cuda_lazy_init.h	974	0644	edit dl rm
disable_torch_function.h	863	0644	edit dl rm
disallow_copy.h	103	0644	edit dl rm
init.h	324	0644	edit dl rm
invalid_arguments.h	302	0644	edit dl rm
memory.h	1173	0644	edit dl rm
numpy_stub.h	399	0644	edit dl rm
object_ptr.h	1329	0644	edit dl rm
out_types.h	294	0644	edit dl rm
pybind.h	7966	0644	edit dl rm
pycfunction_helpers.h	209	0644	edit dl rm
python_arg_parser.h	30753	0644	edit dl rm
python_compat.h	2904	0644	edit dl rm
python_dispatch.h	174	0644	edit dl rm
python_numbers.h	5064	0644	edit dl rm
python_scalars.h	2928	0644	edit dl rm
python_strings.h	4592	0644	edit dl rm
python_stub.h	56	0644	edit dl rm
python_tuples.h	684	0644	edit dl rm
six.h	1425	0644	edit dl rm
structseq.h	153	0644	edit dl rm
tensor_apply.h	431	0644	edit dl rm
tensor_dtypes.h	244	0644	edit dl rm
tensor_flatten.h	2780	0644	edit dl rm
tensor_layouts.h	107	0644	edit dl rm
tensor_list.h	196	0644	edit dl rm
tensor_memoryformats.h	113	0644	edit dl rm
tensor_new.h	1807	0644	edit dl rm
tensor_numpy.h	542	0644	edit dl rm
tensor_qschemes.h	186	0644	edit dl rm
tensor_types.h	491	0644	edit dl rm
throughput_benchmark-inl.h	5210	0644	edit dl rm
throughput_benchmark.h	6868	0644	edit dl rm
variadic.h	4394	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/include/torch/csrc/utils/throughput_benchmark-inl.h (5210B)

#pragma once

#include <random>
#include <thread>

#include <torch/csrc/autograd/profiler.h>
#include <torch/csrc/jit/python/pybind_utils.h>
#include <torch/csrc/utils/pybind.h>

#include <aten/src/ATen/Parallel.h>
#include <c10/util/irange.h>

namespace torch {
namespace throughput_benchmark {
namespace detail {

template <class Input, class Output, class Model>
BenchmarkExecutionStats BenchmarkHelper<Input, Output, Model>::benchmark(
    const BenchmarkConfig& config) const {
  CHECK(initialized_);
  TORCH_CHECK(
      config.num_worker_threads == 1,
      "Only parallelization by callers is supported");

LOG(INFO) << at::get_parallel_info();

// We pre-generate inputs here for each of the threads. This allows us to
  // safely move inputs out for each of the threads independently and thus avoid
  // overhead from the benchmark runner itself
  std::vector<std::vector<Input>> thread_inputs(config.num_calling_threads);
  std::vector<size_t> input_iters(config.num_calling_threads);
  {
    std::random_device seeder;
    std::mt19937 engine(seeder());
    TORCH_CHECK(
        !inputs_.empty(),
        "Please provide benchmark inputs."
        "Did you forget to call add_input()? ");
    std::uniform_int_distribution<int> dist(0, inputs_.size() - 1);

for (const auto thread_id : c10::irange(config.num_calling_threads)) {
      // Just in case we generate num_iters inputs for each of the threads
      // This was if one thread does all the work we will be fine
      for (int i = 0; i < config.num_iters + config.num_warmup_iters; ++i) {
        thread_inputs[thread_id].push_back(cloneInput(inputs_[dist(engine)]));
      }
      input_iters[thread_id] = 0;
    }
  }

std::mutex m;
  std::condition_variable worker_main_cv;
  std::condition_variable main_worker_cv;
  // TODO: add GUARDED_BY once it is available
  int64_t initialized{0};
  int64_t finished{0};
  bool start{false};
  std::atomic<int64_t> num_attempted_iters{0};
  std::vector<std::thread> callers;

callers.reserve(config.num_calling_threads);
  for (const auto thread_id : c10::irange(config.num_calling_threads)) {
    callers.emplace_back([&, thread_id]() {
      // We use conditional variable as a barrier to make sure each thread
      // performs required warmeup iterations before we start measuring
      for (const auto j : c10::irange(config.num_warmup_iters)) {
        runOnce(std::move(thread_inputs[thread_id][input_iters[thread_id]]));
        ++input_iters[thread_id];
      }
      {
        std::unique_lock<std::mutex> lock(m);
        ++initialized;
        worker_main_cv.notify_one();
        // NOLINTNEXTLINE(bugprone-infinite-loop)
        while (!start) {
          main_worker_cv.wait(lock);
        }
      }
      LOG(INFO) << "Starting forward thread " << thread_id;
      while (num_attempted_iters.fetch_add(1) < config.num_iters) {
        runOnce(std::move(thread_inputs[thread_id][input_iters[thread_id]]));
        ++input_iters[thread_id];
      }

{
        std::unique_lock<std::mutex> lock(m);
        ++finished;
        worker_main_cv.notify_one();
        LOG(INFO) << "Shutting down forward thread " << thread_id
                  << ". Total number of finished threads: " << finished;
      }

});
  }

using Clock = std::chrono::high_resolution_clock;
  using RecordProfile = torch::autograd::profiler::RecordProfile;
  using TimePoint = std::chrono::time_point<Clock>;
  TimePoint start_time;

std::unique_ptr<RecordProfile> profiler_guard;
  {
    std::unique_lock<std::mutex> lock(m);
    while (initialized != config.num_calling_threads) {
      worker_main_cv.wait(lock);
    }
    if (!config.profiler_output_path.empty()) {
      LOG(INFO) << "Using Autograd profiler. Trace will be saved to "
                << config.profiler_output_path;
      profiler_guard = std::make_unique<RecordProfile>(config.profiler_output_path);
    }
    LOG(INFO) << "Starting threads";
    start = true;
    start_time = Clock::now();
  }

main_worker_cv.notify_all();
  {
    std::unique_lock<std::mutex> lock(m);
    worker_main_cv.wait(
        lock, [&]() { return finished == config.num_calling_threads; });
  }
  auto end_time = std::chrono::high_resolution_clock::now();
  profiler_guard.reset();
  LOG(INFO) << "Finished benchmark";

BenchmarkExecutionStats stats;
  // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions)
  float total_time_ms = std::chrono::duration_cast<std::chrono::nanoseconds>(
                            end_time - start_time)
                            .count() /
      1000.0 / 1000.0;
  // We use config.num_iters instead of num_attempted_iters as it is
  // repsesatative of the real work done. Last attempted iteration on each
  // calling threads doesn't represent the real work (i.e. running the model)
  stats.latency_avg_ms =
      // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions)
      total_time_ms * config.num_calling_threads / config.num_iters;
  stats.num_iters = config.num_iters;

for (auto& t : callers) {
    t.join();
  }
  return stats;
}

} // namespace detail
} // namespace throughput_benchmark
} // namespace torch

cmd: