This contains my bachelors thesis and associated tex files, code snippets and maybe more. Topic: Data Movement in Heterogeneous Memories with Intel Data Streaming Accelerator
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

280 lines
9.6 KiB

  1. #pragma once
  2. #include <iostream>
  3. #include <unordered_map>
  4. #include <shared_mutex>
  5. #include <mutex>
  6. #include <memory>
  7. #include <sched.h>
  8. #include <numa.h>
  9. #include <numaif.h>
  10. #include <dml/dml.hpp>
  11. #include "cache-data.hpp"
  12. namespace dsacache {
  13. // singleton which holds the cache workers
  14. // and is the place where work will be submited
  15. class Cache {
  16. public:
  17. // cache policy is defined as a type here to allow flexible usage of the cacher
  18. // given a numa destination node (where the data will be needed), the numa source
  19. // node (current location of the data) and the data size, this function should
  20. // return optimal cache placement
  21. // dst node and returned value can differ if the system, for example, has HBM
  22. // attached accessible directly to node n under a different node id m
  23. typedef int (CachePolicy)(const int numa_dst_node, const int numa_src_node, const size_t data_size);
  24. // copy policy specifies the copy-executing nodes for a given task
  25. // which allows flexibility in assignment for optimizing raw throughput
  26. // or choosing a conservative usage policy
  27. typedef std::vector<int> (CopyPolicy)(const int numa_dst_node, const int numa_src_node);
  28. private:
  29. // mutex for accessing the cache state map
  30. std::shared_mutex cache_mutex_;
  31. // map from [dst-numa-node,map2]
  32. // map2 from [data-ptr,cache-structure]
  33. std::unordered_map<uint8_t, std::unordered_map<uint8_t*, CacheData>> cache_state_;
  34. CachePolicy* cache_policy_function_ = nullptr;
  35. CopyPolicy* copy_policy_function_ = nullptr;
  36. dml::handler<dml::mem_copy_operation, std::allocator<uint8_t>> ExecuteCopy(
  37. const uint8_t* src, uint8_t* dst, const size_t size, const int node
  38. ) const;
  39. void SubmitTask(CacheData* task, const int dst_node, const int src_node);
  40. void GetCacheNode(uint8_t* src, const size_t size, int* OUT_DST_NODE, int* OUT_SRC_NODE) const;
  41. std::unique_ptr<CacheData> GetFromCache(uint8_t* src, const size_t size, const int dst_node);
  42. public:
  43. void Init(CachePolicy* cache_policy_function, CopyPolicy* copy_policy_function);
  44. // function to perform data access through the cache
  45. std::unique_ptr<CacheData> Access(uint8_t* data, const size_t size);
  46. void Flush(const int node = -1);
  47. };
  48. }
  49. inline void dsacache::Cache::Init(CachePolicy* cache_policy_function, CopyPolicy* copy_policy_function) {
  50. cache_policy_function_ = cache_policy_function;
  51. copy_policy_function_ = copy_policy_function;
  52. // initialize numa library
  53. numa_available();
  54. const int nodes_max = numa_num_configured_nodes();
  55. const bitmask* valid_nodes = numa_get_mems_allowed();
  56. for (int node = 0; node < nodes_max; node++) {
  57. if (numa_bitmask_isbitset(valid_nodes, node)) {
  58. cache_state_.insert({node,{}});
  59. }
  60. }
  61. std::cout << "[-] Cache Initialized" << std::endl;
  62. }
  63. inline std::unique_ptr<dsacache::CacheData> dsacache::Cache::Access(uint8_t* data, const size_t size) {
  64. // get destination numa node for the cache
  65. int dst_node = -1;
  66. int src_node = -1;
  67. GetCacheNode(data, size, &dst_node, &src_node);
  68. // check whether the data is already cached
  69. std::unique_ptr<CacheData> task = GetFromCache(data, size, dst_node);
  70. if (task != nullptr) {
  71. return std::move(task);
  72. }
  73. // at this point the requested data is not present in cache
  74. // and we create a caching task for it
  75. task = std::make_unique<CacheData>(data, size);
  76. {
  77. std::unique_lock<std::shared_mutex> lock(cache_mutex_);
  78. const auto state = cache_state_[dst_node].emplace(task->src_, *task);
  79. // if state.second is false then no insertion took place
  80. // which means that concurrently whith this thread
  81. // some other thread must have accessed the same
  82. // resource in which case we return the other
  83. // threads data cache structure
  84. if (!state.second) {
  85. std::cout << "[!] Found another cache instance for 0x" << std::hex << (uint64_t)task->src_ << std::dec << std::endl;
  86. return std::move(std::make_unique<CacheData>(state.first->second));
  87. }
  88. }
  89. SubmitTask(task.get(), dst_node, src_node);
  90. return std::move(task);
  91. }
  92. inline void dsacache::Cache::SubmitTask(CacheData* task, const int dst_node, const int src_node) {
  93. std::cout << "[+] Allocating " << task->size_ << "B on node " << dst_node << " for " << std::hex << (uint64_t)task->src_ << std::dec << std::endl;
  94. // allocate data on this node and flush the unused parts of the
  95. // cache if the operation fails and retry once
  96. // TODO: smarter flush strategy could keep some stuff cached
  97. uint8_t* dst = reinterpret_cast<uint8_t*>(numa_alloc_onnode(task->size_, dst_node));
  98. if (dst == nullptr) {
  99. std::cout << "[!] First allocation try failed for " << task->size_ << "B on node " << dst_node << std::endl;
  100. // allocation on dst_node failed so we flush the cache for this
  101. // node hoping to free enough currently unused entries to make
  102. // the second allocation attempt successful
  103. Flush(dst_node);
  104. dst = reinterpret_cast<uint8_t*>(numa_alloc_onnode(task->size_, dst_node));
  105. if (dst == nullptr) {
  106. std::cout << "[x] Second allocation try failed for " << task->size_ << "B on node " << dst_node << std::endl;
  107. return;
  108. }
  109. }
  110. task->incomplete_cache_ = dst;
  111. // querry copy policy function for the nodes to use for the copy
  112. const std::vector<int> executing_nodes = copy_policy_function_(dst_node, src_node);
  113. const size_t task_count = executing_nodes.size();
  114. // each task will copy one fair part of the total size
  115. // and in case the total size is not a factor of the
  116. // given task count the last node must copy the remainder
  117. const size_t size = task->size_ / task_count;
  118. const size_t last_size = size + task->size_ % task_count;
  119. std::cout << "[-] Splitting Copy into " << task_count << " tasks of " << size << "B 0x" << std::hex << (uint64_t)task->src_ << std::dec << std::endl;
  120. // save the current numa node mask to restore later
  121. // as executing the copy task will place this thread
  122. // on a different node
  123. bitmask* nodemask = numa_get_run_node_mask();
  124. for (uint32_t i = 0; i < task_count; i++) {
  125. const size_t local_size = i + 1 == task_count ? size : last_size;
  126. const size_t local_offset = i * size;
  127. const uint8_t* local_src = task->src_ + local_offset;
  128. uint8_t* local_dst = dst + local_offset;
  129. task->handlers_->emplace_back(ExecuteCopy(local_src, local_dst, local_size, executing_nodes[i]));
  130. }
  131. // restore the previous nodemask
  132. numa_run_on_node_mask(nodemask);
  133. }
  134. inline dml::handler<dml::mem_copy_operation, std::allocator<uint8_t>> dsacache::Cache::ExecuteCopy(
  135. const uint8_t* src, uint8_t* dst, const size_t size, const int node
  136. ) const {
  137. dml::const_data_view srcv = dml::make_view(src, size);
  138. dml::data_view dstv = dml::make_view(dst, size);
  139. numa_run_on_node(node);
  140. return dml::submit<dml::automatic>(dml::mem_copy.block_on_fault(), srcv, dstv);
  141. }
  142. void dsacache::Cache::GetCacheNode(uint8_t* src, const size_t size, int* OUT_DST_NODE, int* OUT_SRC_NODE) const {
  143. // obtain numa node of current thread to determine where the data is needed
  144. const int current_cpu = sched_getcpu();
  145. const int current_node = numa_node_of_cpu(current_cpu);
  146. // obtain node that the given data pointer is allocated on
  147. *OUT_SRC_NODE = -1;
  148. get_mempolicy(OUT_SRC_NODE, NULL, 0, (void*)src, MPOL_F_NODE | MPOL_F_ADDR);
  149. // querry cache policy function for the destination numa node
  150. *OUT_DST_NODE = cache_policy_function_(current_node, *OUT_SRC_NODE, size);
  151. }
  152. inline void dsacache::Cache::Flush(const int node) {
  153. std::cout << "[-] Flushing Cache for " << (node == -1 ? "all nodes" : "node " + std::to_string(node)) << std::endl;
  154. const auto FlushNode = [](std::unordered_map<uint8_t*,CacheData>& map) {
  155. auto it = map.begin();
  156. while (it != map.end()) {
  157. if (it->second.Active() == false) {
  158. map.erase(it);
  159. it = map.begin();
  160. }
  161. else {
  162. it++;
  163. }
  164. }
  165. };
  166. {
  167. std::unique_lock<std::shared_mutex> lock(cache_mutex_);
  168. if (node == -1) {
  169. for (auto& nc : cache_state_) {
  170. FlushNode(nc.second);
  171. }
  172. }
  173. else {
  174. FlushNode(cache_state_[node]);
  175. }
  176. }
  177. }
  178. std::unique_ptr<dsacache::CacheData> dsacache::Cache::GetFromCache(uint8_t* src, const size_t size, const int dst_node) {
  179. // the best situation is if this data is already cached
  180. // which we check in an unnamed block in which the cache
  181. // is locked for reading to prevent another thread
  182. // from marking the element we may find as unused and
  183. // clearing it
  184. std::shared_lock<std::shared_mutex> lock(cache_mutex_);
  185. const auto search = cache_state_[dst_node].find(src);
  186. if (search != cache_state_[dst_node].end()) {
  187. if (search->second.size_ == size) {
  188. search->second.active_->store(true);
  189. std::cout << "[+] Found Cached version for 0x" << std::hex << (uint64_t)src << std::dec << std::endl;
  190. return std::move(std::make_unique<CacheData>(search->second));
  191. }
  192. else {
  193. std::cout << "[!] Found Cached version with size missmatch for 0x" << std::hex << (uint64_t)src << std::dec << std::endl;
  194. cache_state_[dst_node].erase(search);
  195. }
  196. }
  197. return nullptr;
  198. }