From 8065dd4345c97c5dc3eb2d30fc5d921f0392dbe6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Constantin=20F=C3=BCrst?= <c@fuersten.info>
Date: Mon, 4 Dec 2023 11:48:29 +0100
Subject: [PATCH] collect entire duration vector and dont condense the
 information down

---
 benchmarks/benchmark.hpp         | 25 +------------------------
 benchmarks/task-data.hpp         | 12 +++---------
 benchmarks/task-description.json |  2 +-
 3 files changed, 5 insertions(+), 34 deletions(-)

diff --git a/benchmarks/benchmark.hpp b/benchmarks/benchmark.hpp
index 3afc34e..4752716 100644
--- a/benchmarks/benchmark.hpp
+++ b/benchmarks/benchmark.hpp
@@ -15,32 +15,16 @@
 #include "statuscode-tostring.hpp"
 #include "task-data.hpp"
 
-double avg(const std::vector<uint64_t>& v) {
-    return static_cast<long double>(std::accumulate(v.begin(), v.end(), 0)) / static_cast<long double>(v.size());
-}
-
-double stdev(const std::vector<uint64_t>& v, const double mean) {
-    std::vector<double> diff(v.size());
-    std::transform(v.begin(), v.end(), diff.begin(), [mean](double x) { return x - mean; });
-    const double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
-    const double stdev = std::sqrt(sq_sum / static_cast<double>(v.size()));
-    return stdev;
-}
-
 #define LOG_CODE_INFO "Location: " << __FILE__ << "@" << __LINE__ << "::" << __FUNCTION__ << std::endl
 #define LOG_ERR { pthread_t t = pthread_self(); std::cerr << "--- BEGIN ERROR MSG ---" << std::endl << "Physical: [Node " << args->numa_node << " | Thread " << t << "]" << std::endl; } std::cerr << LOG_CODE_INFO
 #define CHECK_STATUS(status,msg) { if (status != dml::status_code::ok) { LOG_ERR << "Status Code: " << StatusCodeToString(status) << std::endl << msg << std::endl; args->status = status; return nullptr; }}
 
-#define ADD_TIMING_MESSUREMENT { if (i >= 5) { submission_durations.emplace_back(std::chrono::duration_cast<std::chrono::microseconds>(se - st).count()); completion_durations.emplace_back(std::chrono::duration_cast<std::chrono::microseconds>(et - se).count()); combined_durations.emplace_back(std::chrono::duration_cast<std::chrono::microseconds>(et - st).count());}}
+#define ADD_TIMING_MESSUREMENT { if (i >= 5) { args->submit_duration.emplace_back(std::chrono::duration_cast<std::chrono::microseconds>(se - st).count()); args->complete_duration.emplace_back(std::chrono::duration_cast<std::chrono::microseconds>(et - se).count()); args->combined_duration.emplace_back(std::chrono::duration_cast<std::chrono::microseconds>(et - st).count());}}
 
 template <typename path>
 void* thread_function(void* argp) {
     TaskData* args = reinterpret_cast<TaskData*>(argp);
 
-    std::vector<uint64_t> submission_durations;
-    std::vector<uint64_t> completion_durations;
-    std::vector<uint64_t> combined_durations;
-
     // set numa node and core affinity of the current thread
     numa_run_on_node(args->numa_node);
     
@@ -151,13 +135,6 @@ void* thread_function(void* argp) {
     numa_free(src, args->size);
     numa_free(dst, args->size);
 
-    args->combined_duration = avg(combined_durations);
-    args->complete_duration = avg(completion_durations);
-    args->submit_duration = avg(submission_durations);
-    args->combined_duration_stdev = stdev(combined_durations, args->combined_duration);
-    args->complete_duration_stdev = stdev(completion_durations, args->complete_duration);
-    args->submit_duration_stdev = stdev(submission_durations, args->submit_duration);
-
     return nullptr;
 }
 
diff --git a/benchmarks/task-data.hpp b/benchmarks/task-data.hpp
index 6550343..a78f631 100644
--- a/benchmarks/task-data.hpp
+++ b/benchmarks/task-data.hpp
@@ -21,12 +21,9 @@ struct TaskData {
     // thread output
     dml::status_code status;
     // average run duration in microseconds
-    double combined_duration;
-    double submit_duration;
-    double complete_duration;
-    double combined_duration_stdev;
-    double submit_duration_stdev;
-    double complete_duration_stdev;
+    std::vector<double> combined_duration;
+    std::vector<double> submit_duration;
+    std::vector<double> complete_duration;
     // completed iterations
     uint32_t rep_completed;
     // set by execution
@@ -45,9 +42,6 @@ inline void to_json(nlohmann::json& j, const TaskData& a) {
     j["report"]["time"]["completion_avg"] = a.complete_duration;
     j["report"]["time"]["submission_avg"] = a.submit_duration;
     j["report"]["time"]["combined_avg"] = a.combined_duration;
-    j["report"]["time"]["completion_stdev"] = a.complete_duration_stdev;
-    j["report"]["time"]["submission_stdev"] = a.submit_duration_stdev;
-    j["report"]["time"]["combined_stdev"] = a.combined_duration_stdev;
     j["report"]["iterations_completed"] = a.rep_completed;
     j["report"]["status"] = StatusCodeToString(a.status);
 }
diff --git a/benchmarks/task-description.json b/benchmarks/task-description.json
index 3c8086c..6c4f577 100644
--- a/benchmarks/task-description.json
+++ b/benchmarks/task-description.json
@@ -5,7 +5,7 @@
     {
       "task": {
         "size": 4096,
-        "iterations": 1000,
+        "iterations": 100,
         "batching": {
           "batch_submit": false,
           "batch_size": 0