From e28327b74af8fdab2df76733eff3ba560dcbc2e6 Mon Sep 17 00:00:00 2001
Date: Wed, 8 Dec 2021 16:07:54 +0800
Subject: [PATCH] G1 GC NUMA feature preferentially selects the nearest NUMA
 node for memory allocation

---
 src/hotspot/os/aix/os_aix.cpp                 |  4 +++
 src/hotspot/os/bsd/os_bsd.cpp                 |  4 ++-
 src/hotspot/os/linux/os_linux.cpp             |  4 +++
 src/hotspot/os/windows/os_windows.cpp         |  4 +++
 src/hotspot/share/gc/g1/g1NUMA.cpp            | 31 ++++++++++++++++++-
 src/hotspot/share/gc/g1/g1NUMA.hpp            |  9 ++++++
 .../share/gc/g1/heapRegionSet.inline.hpp      | 24 +++++++++++++-
 src/hotspot/share/runtime/os.hpp              |  1 +
 8 files changed, 78 insertions(+), 3 deletions(-)

diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index 1d2ce086b..5b5c6c449 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -279,6 +279,10 @@ bool os::have_special_privileges() {
   return privileges;
 }
 
+uint os::numa_distance(uint node_index, uint node_index_other) {
+  return 0;
+}
+
 // Helper function, emulates disclaim64 using multiple 32bit disclaims
 // because we cannot use disclaim64() on AS/400 and old AIX releases.
 static bool my_disclaim64(char* addr, size_t size) {
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index d71d44d1d..ba3d1568c 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -191,7 +191,9 @@ bool os::have_special_privileges() {
   return privileges;
 }
 
-
+uint os::numa_distance(uint node_index, uint node_index_other) {
+  return 0;
+}
 
 // Cpu architecture string
 #if   defined(ZERO)
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index b4c89b518..fc3800edd 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -2862,6 +2862,10 @@ int os::numa_get_group_id_for_address(const void* address) {
   return id;
 }
 
+uint os::numa_distance(uint node_index, uint node_index_other) {
+  return Linux::numa_distance(node_index, node_index_other);
+}
+
 int os::Linux::get_existing_num_nodes() {
   int node;
   int highest_node_number = Linux::numa_max_node();
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 4f9d54029..fe52ffb22 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -278,6 +278,10 @@ void os::run_periodic_checks() {
   return;
 }
 
+uint os::numa_distance(uint node_index, uint node_index_other) {
+  return 0;
+}
+
 // previous UnhandledExceptionFilter, if there is one
 static LPTOP_LEVEL_EXCEPTION_FILTER prev_uef_handler = NULL;
 
diff --git a/src/hotspot/share/gc/g1/g1NUMA.cpp b/src/hotspot/share/gc/g1/g1NUMA.cpp
index 95d9d8c15..2c2431a39 100644
--- a/src/hotspot/share/gc/g1/g1NUMA.cpp
+++ b/src/hotspot/share/gc/g1/g1NUMA.cpp
@@ -70,9 +70,20 @@ uint G1NUMA::index_of_node_id(int node_id) const {
   return node_index;
 }
 
+// Returns numa distance
+const uint G1NUMA::calc_numa_node_distance(uint node_index, uint other_node_index) const {
+  if (node_index >= _num_active_node_ids || other_node_index >= _num_active_node_ids) {
+    return UINT_MAX;
+  }
+  return _numa_node_distance[node_index][other_node_index];
+}
+bool G1NUMA::use_nearest_node() const {
+   return _use_nearest;
+}
+
 G1NUMA::G1NUMA() :
   _node_id_to_index_map(NULL), _len_node_id_to_index_map(0),
-  _node_ids(NULL), _num_active_node_ids(0),
+  _node_ids(NULL), _num_active_node_ids(0), _use_nearest(false),
   _region_size(0), _page_size(0), _stats(NULL) {
 }
 
@@ -104,10 +115,24 @@ void G1NUMA::initialize(bool use_numa) {
   for (uint i = 0; i < _num_active_node_ids; i++) {
     max_node_id = MAX2(max_node_id, _node_ids[i]);
   }
+  if (_num_active_node_ids > 2) {
+    _use_nearest = true;
+  }
 
   // Create a mapping between node_id and index.
   _len_node_id_to_index_map = max_node_id + 1;
   _node_id_to_index_map = NEW_C_HEAP_ARRAY(uint, _len_node_id_to_index_map, mtGC);
+  _numa_node_distance =  NEW_C_HEAP_ARRAY(uint*, _num_active_node_ids * _num_active_node_ids, mtGC);
+   // Set node disctance
+  for (uint node_i = 0; node_i < _num_active_node_ids; node_i++) {
+    _numa_node_distance[node_i] = NEW_C_HEAP_ARRAY(uint, _num_active_node_ids*_num_active_node_ids, mtGC);
+  }
+  for (uint node_i = 0; node_i < _num_active_node_ids; node_i++) {
+    for (uint node_j = 0; node_j < _num_active_node_ids; node_j++) {
+      uint distance = os::numa_distance(node_i, node_j);
+      _numa_node_distance[node_i][node_j] = distance;
+    }
+  }
 
   // Set all indices with unknown node id.
   for (int i = 0; i < _len_node_id_to_index_map; i++) {
@@ -126,6 +151,10 @@ G1NUMA::~G1NUMA() {
   delete _stats;
   FREE_C_HEAP_ARRAY(int, _node_id_to_index_map);
   FREE_C_HEAP_ARRAY(int, _node_ids);
+  for (uint node_i = 0; node_i < _num_active_node_ids; node_i++) {
+    FREE_C_HEAP_ARRAY(uint, _numa_node_distance[node_i]);
+  }
+  FREE_C_HEAP_ARRAY(uint*, _numa_node_distance);
 }
 
 void G1NUMA::set_region_info(size_t region_size, size_t page_size) {
diff --git a/src/hotspot/share/gc/g1/g1NUMA.hpp b/src/hotspot/share/gc/g1/g1NUMA.hpp
index 83117b8ec..24ae9712d 100644
--- a/src/hotspot/share/gc/g1/g1NUMA.hpp
+++ b/src/hotspot/share/gc/g1/g1NUMA.hpp
@@ -45,6 +45,10 @@ class G1NUMA: public CHeapObj<mtGC> {
   int* _node_ids;
   // Total number of node ids.
   uint _num_active_node_ids;
+  // numa node distance
+  uint** _numa_node_distance;
+  // Use nearest numa node
+  bool _use_nearest;
 
   // HeapRegion size
   size_t _region_size;
@@ -84,6 +88,8 @@ public:
   // Returns active memory node count.
   uint num_active_nodes() const;
 
+  bool use_nearest_node() const;
+
   bool is_enabled() const;
 
   int numa_id(int index) const;
@@ -91,6 +97,9 @@ public:
   // Returns memory node ids
   const int* node_ids() const;
 
+  // Returns numa distance
+  const uint calc_numa_node_distance(uint node_index, uint other_node_index) const;
+
   // Returns node index of current calling thread.
   uint index_of_current_thread() const;
 
diff --git a/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp b/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp
index 1f509ff3e..67b3ade50 100644
--- a/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp
+++ b/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp
@@ -183,26 +183,48 @@ inline HeapRegion* FreeRegionList::remove_region_with_node_index(bool from_head,
 
   // Find the region to use, searching from _head or _tail as requested.
   size_t cur_depth = 0;
+  bool exist_region = false;
+  uint numa_distance_min = UINT_MAX;
+  HeapRegion* remote_node_region = NULL;
+
   if (from_head) {
     for (cur = _head;
          cur != NULL && cur_depth < max_search_depth;
          cur = cur->next(), ++cur_depth) {
       if (requested_node_index == cur->node_index()) {
+        exist_region = true;
         break;
       }
+      if (G1NUMA::numa()->use_nearest_node()) {
+         uint distance = G1NUMA::numa()->calc_numa_node_distance(requested_node_index, cur->node_index());
+         if (distance < numa_distance_min) {
+           remote_node_region = cur;
+           numa_distance_min = distance;
+         }
+      }
     }
   } else {
     for (cur = _tail;
          cur != NULL && cur_depth < max_search_depth;
          cur = cur->prev(), ++cur_depth) {
       if (requested_node_index == cur->node_index()) {
+        exist_region = true;
         break;
       }
+      if (G1NUMA::numa()->use_nearest_node()) {
+          uint distance = G1NUMA::numa()->calc_numa_node_distance(requested_node_index, cur->node_index());
+          if (distance < numa_distance_min) {
+            remote_node_region = cur;
+            numa_distance_min = distance;
+          }
+      }
     }
   }
 
   // Didn't find a region to use.
-  if (cur == NULL || cur_depth >= max_search_depth) {
+  if (G1NUMA::numa()->use_nearest_node() && !exist_region && remote_node_region != NULL) {
+    cur = remote_node_region;
+  } else if (cur == NULL || cur_depth >= max_search_depth) {
     return NULL;
   }
 
diff --git a/src/hotspot/share/runtime/os.hpp b/src/hotspot/share/runtime/os.hpp
index 7eaaa9db9..34dcebd9e 100644
--- a/src/hotspot/share/runtime/os.hpp
+++ b/src/hotspot/share/runtime/os.hpp
@@ -418,6 +418,7 @@ class os: AllStatic {
   static bool   numa_topology_changed();
   static int    numa_get_group_id();
   static int    numa_get_group_id_for_address(const void* address);
+  static uint   numa_distance(uint node_index, uint node_index_other);
 
   // Page manipulation
   struct page_info {
-- 
2.23.0