Date: Thu, 8 Jun 2023 20:43:33 +0800
Subject: 8072070: Improve interpreter stack banging

Bug url: https://bugs.openjdk.org/browse/JDK-8072070
---
 .../vm/templateInterpreter_aarch64.cpp        | 49 +++++++++++---
 .../cpu/ppc/vm/templateInterpreter_ppc.cpp    | 20 ++++++
 .../sparc/vm/templateInterpreter_sparc.cpp    | 22 +++++++
 .../cpu/x86/vm/templateInterpreter_x86.cpp    | 64 +++++++++++++++++++
 .../src/share/vm/interpreter/interpreter.cpp  | 25 +-------
 hotspot/src/share/vm/runtime/os.cpp           |  7 +-
 hotspot/src/share/vm/runtime/thread.cpp       |  4 +-
 hotspot/src/share/vm/runtime/thread.hpp       | 20 ++++++
 8 files changed, 174 insertions(+), 37 deletions(-)

diff --git a/hotspot/src/cpu/aarch64/vm/templateInterpreter_aarch64.cpp b/hotspot/src/cpu/aarch64/vm/templateInterpreter_aarch64.cpp
index f356fbf81..a5a91e5f3 100644
--- a/hotspot/src/cpu/aarch64/vm/templateInterpreter_aarch64.cpp
+++ b/hotspot/src/cpu/aarch64/vm/templateInterpreter_aarch64.cpp
@@ -1087,17 +1087,46 @@ address InterpreterGenerator::generate_Dgemv_dgemv_entry() {
 }
 
 void InterpreterGenerator::bang_stack_shadow_pages(bool native_call) {
-  // Bang each page in the shadow zone. We can't assume it's been done for
-  // an interpreter frame with greater than a page of locals, so each page
-  // needs to be checked.  Only true for non-native.
-  if (UseStackBanging) {
-    const int start_page = native_call ? StackShadowPages : 1;
-    const int page_size = os::vm_page_size();
-    for (int pages = start_page; pages <= StackShadowPages ; pages++) {
-      __ sub(rscratch2, sp, pages*page_size);
-      __ str(zr, Address(rscratch2));
-    }
+  // See more discussion in stackOverflow.hpp.
+
+  const int shadow_zone_size = (int)(JavaThread::stack_shadow_zone_size());
+  const int page_size = os::vm_page_size();
+  const int n_shadow_pages = shadow_zone_size / page_size;
+
+#ifdef ASSERT
+  Label L_good_limit;
+  __ ldr(rscratch1, Address(rthread, JavaThread::shadow_zone_safe_limit_offset()));
+  __ cbnz(rscratch1, L_good_limit);
+  __ stop("shadow zone safe limit is not initialized");
+  __ bind(L_good_limit);
+
+  Label L_good_watermark;
+  __ ldr(rscratch1, Address(rthread, JavaThread::shadow_zone_growth_watermark_offset()));
+  __ cbnz(rscratch1, L_good_watermark);
+  __ stop("shadow zone growth watermark is not initialized");
+  __ bind(L_good_watermark);
+#endif
+
+  Label L_done;
+
+  __ ldr(rscratch1, Address(rthread, JavaThread::shadow_zone_growth_watermark_offset()));
+  __ cmp(sp, rscratch1);
+  __ br(Assembler::HI, L_done);
+
+  for (int p = 1; p <= n_shadow_pages; p++) {
+    __ sub(rscratch2, sp, p*page_size);
+    __ str(zr, Address(rscratch2));
   }
+
+  // Record the new watermark, but only if the update is above the safe limit.
+  // Otherwise, the next time around the check above would pass the safe limit.
+  __ ldr(rscratch1, Address(rthread, JavaThread::shadow_zone_safe_limit_offset()));
+  __ cmp(sp, rscratch1);
+  __ br(Assembler::LS, L_done);
+  __ mov(rscratch1, sp);
+  __ str(rscratch1, Address(rthread, JavaThread::shadow_zone_growth_watermark_offset()));
+
+  __ bind(L_done);
 }
 
 
diff --git a/hotspot/src/cpu/ppc/vm/templateInterpreter_ppc.cpp b/hotspot/src/cpu/ppc/vm/templateInterpreter_ppc.cpp
index 42dd8f2a9..0fb934166 100644
--- a/hotspot/src/cpu/ppc/vm/templateInterpreter_ppc.cpp
+++ b/hotspot/src/cpu/ppc/vm/templateInterpreter_ppc.cpp
@@ -2030,5 +2030,25 @@ void TemplateInterpreterGenerator::stop_interpreter_at() {
   __ bind(L);
 }
 
+void AbstractInterpreterGenerator::bang_stack_shadow_pages(bool native_call) {
+  // Quick & dirty stack overflow checking: bang the stack & handle trap.
+  // Note that we do the banging after the frame is setup, since the exception
+  // handling code expects to find a valid interpreter frame on the stack.
+  // Doing the banging earlier fails if the caller frame is not an interpreter
+  // frame.
+  // (Also, the exception throwing code expects to unlock any synchronized
+  // method receiever, so do the banging after locking the receiver.)
+
+  // Bang each page in the shadow zone. We can't assume it's been done for
+  // an interpreter frame with greater than a page of locals, so each page
+  // needs to be checked.  Only true for non-native.
+  if (UseStackBanging) {
+    const int start_page = native_call ? StackShadowPages : 1;
+    const int page_size = os::vm_page_size();
+    for (int pages = start_page; pages <= StackShadowPages ; pages++) {
+      __ bang_stack_with_offset(pages*page_size);
+    }
+  }
+}
 #endif // !PRODUCT
 #endif // !CC_INTERP
diff --git a/hotspot/src/cpu/sparc/vm/templateInterpreter_sparc.cpp b/hotspot/src/cpu/sparc/vm/templateInterpreter_sparc.cpp
index 83d40496c..540f9b287 100644
--- a/hotspot/src/cpu/sparc/vm/templateInterpreter_sparc.cpp
+++ b/hotspot/src/cpu/sparc/vm/templateInterpreter_sparc.cpp
@@ -2113,5 +2113,27 @@ void TemplateInterpreterGenerator::stop_interpreter_at() {
   __ cmp(G3_scratch, G4_scratch);
   __ breakpoint_trap(Assembler::equal, Assembler::icc);
 }
+
+void AbstractInterpreterGenerator::bang_stack_shadow_pages(bool native_call) {
+  // Quick & dirty stack overflow checking: bang the stack & handle trap.
+  // Note that we do the banging after the frame is setup, since the exception
+  // handling code expects to find a valid interpreter frame on the stack.
+  // Doing the banging earlier fails if the caller frame is not an interpreter
+  // frame.
+  // (Also, the exception throwing code expects to unlock any synchronized
+  // method receiever, so do the banging after locking the receiver.)
+
+  // Bang each page in the shadow zone. We can't assume it's been done for
+  // an interpreter frame with greater than a page of locals, so each page
+  // needs to be checked.  Only true for non-native.
+  if (UseStackBanging) {
+    const int start_page = native_call ? StackShadowPages : 1;
+    const int page_size = os::vm_page_size();
+    for (int pages = start_page; pages <= StackShadowPages ; pages++) {
+      __ bang_stack_with_offset(pages*page_size);
+    }
+  }
+}
+
 #endif // not PRODUCT
 #endif // !CC_INTERP
diff --git a/hotspot/src/cpu/x86/vm/templateInterpreter_x86.cpp b/hotspot/src/cpu/x86/vm/templateInterpreter_x86.cpp
index e470aa62d..fda2908c3 100644
--- a/hotspot/src/cpu/x86/vm/templateInterpreter_x86.cpp
+++ b/hotspot/src/cpu/x86/vm/templateInterpreter_x86.cpp
@@ -29,6 +29,8 @@
 
 #ifndef CC_INTERP
 
+# define __ _masm->
+
 // asm based interpreter deoptimization helpers
 int AbstractInterpreter::size_activation(int max_stack,
                                          int temps,
@@ -121,4 +123,66 @@ void AbstractInterpreter::layout_activation(Method* method,
     method->constants()->cache();
 }
 
+void AbstractInterpreterGenerator::bang_stack_shadow_pages(bool native_call) {
+  // See more discussion in stackOverflow.hpp.
+
+  // Note that we do the banging after the frame is setup, since the exception
+  // handling code expects to find a valid interpreter frame on the stack.
+  // Doing the banging earlier fails if the caller frame is not an interpreter
+  // frame.
+  // (Also, the exception throwing code expects to unlock any synchronized
+  // method receiver, so do the banging after locking the receiver.)
+
+  const int shadow_zone_size = (int)(JavaThread::stack_shadow_zone_size());
+  const int page_size = os::vm_page_size();
+  const int n_shadow_pages = shadow_zone_size / page_size;
+
+  const Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
+ #ifndef _LP64
+  __ push(thread);
+  __ get_thread(thread);
+#endif
+
+#ifdef ASSERT
+  Label L_good_limit;
+  __ cmpptr(Address(thread, JavaThread::shadow_zone_safe_limit_offset()), (int32_t)NULL_WORD);
+  __ jcc(Assembler::notEqual, L_good_limit);
+  __ stop("shadow zone safe limit is not initialized");
+  __ bind(L_good_limit);
+
+  Label L_good_watermark;
+  __ cmpptr(Address(thread, JavaThread::shadow_zone_growth_watermark_offset()), (int32_t)NULL_WORD);
+  __ jcc(Assembler::notEqual, L_good_watermark);
+  __ stop("shadow zone growth watermark is not initialized");
+  __ bind(L_good_watermark);
+#endif
+
+  Label L_done;
+
+  __ cmpptr(rsp, Address(thread, JavaThread::shadow_zone_growth_watermark_offset()));
+  __ jcc(Assembler::above, L_done);
+
+  for (int p = 1; p <= n_shadow_pages; p++) {
+    __ bang_stack_with_offset(p*page_size);
+  }
+
+  // Record a new watermark, unless the update is above the safe limit.
+  // Otherwise, the next time around a check above would pass the safe limit.
+  __ cmpptr(rsp, Address(thread, JavaThread::shadow_zone_safe_limit_offset()));
+  __ jccb(Assembler::belowEqual, L_done);
+#ifdef _LP64
+  __ movptr(rscratch1, rsp);
+  __ andptr(rscratch1, ~(page_size - 1));
+  __ movptr(Address(thread, JavaThread::shadow_zone_growth_watermark_offset()), rscratch1);
+#else
+  __ movptr(Address(thread, JavaThread::shadow_zone_growth_watermark_offset()), rsp);
+#endif
+
+  __ bind(L_done);
+
+#ifndef _LP64
+  __ pop(thread);
+#endif
+}
+
 #endif // CC_INTERP
diff --git a/hotspot/src/share/vm/interpreter/interpreter.cpp b/hotspot/src/share/vm/interpreter/interpreter.cpp
index bfcb1bea2..d5d94f34c 100644
--- a/hotspot/src/share/vm/interpreter/interpreter.cpp
+++ b/hotspot/src/share/vm/interpreter/interpreter.cpp
@@ -44,7 +44,9 @@
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
 #include "runtime/timer.hpp"
-
+#include "runtime/vframeArray.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/macros.hpp"
 # define __ _masm->
 
 
@@ -474,27 +476,6 @@ bool AbstractInterpreter::bytecode_should_reexecute(Bytecodes::Code code) {
   }
 }
 
-void AbstractInterpreterGenerator::bang_stack_shadow_pages(bool native_call) {
-  // Quick & dirty stack overflow checking: bang the stack & handle trap.
-  // Note that we do the banging after the frame is setup, since the exception
-  // handling code expects to find a valid interpreter frame on the stack.
-  // Doing the banging earlier fails if the caller frame is not an interpreter
-  // frame.
-  // (Also, the exception throwing code expects to unlock any synchronized
-  // method receiever, so do the banging after locking the receiver.)
-
-  // Bang each page in the shadow zone. We can't assume it's been done for
-  // an interpreter frame with greater than a page of locals, so each page
-  // needs to be checked.  Only true for non-native.
-  if (UseStackBanging) {
-    const int start_page = native_call ? StackShadowPages : 1;
-    const int page_size = os::vm_page_size();
-    for (int pages = start_page; pages <= StackShadowPages ; pages++) {
-      __ bang_stack_with_offset(pages*page_size);
-    }
-  }
-}
-
 void AbstractInterpreterGenerator::initialize_method_handle_entries() {
   // method handle entry kinds are generated later in MethodHandlesAdapterGenerator::generate:
   for (int i = Interpreter::method_handle_invoke_FIRST; i <= Interpreter::method_handle_invoke_LAST; i++) {
diff --git a/hotspot/src/share/vm/runtime/os.cpp b/hotspot/src/share/vm/runtime/os.cpp
index b4f83a3cb..84841b76e 100644
--- a/hotspot/src/share/vm/runtime/os.cpp
+++ b/hotspot/src/share/vm/runtime/os.cpp
@@ -1371,11 +1371,10 @@ bool os::stack_shadow_pages_available(Thread *thread, methodHandle method) {
   // respectively.
   const int framesize_in_bytes =
     Interpreter::size_top_interpreter_activation(method()) * wordSize;
-  int reserved_area = ((StackShadowPages + StackRedPages + StackYellowPages)
-                      * vm_page_size()) + framesize_in_bytes;
+
   // The very lower end of the stack
-  address stack_limit = thread->stack_base() - thread->stack_size();
-  return (sp > (stack_limit + reserved_area));
+  address stack_limit = ((JavaThread*)thread)->shadow_zone_safe_limit();
+  return (sp > (stack_limit + framesize_in_bytes));
 }
 
 size_t os::page_size_for_region(size_t region_size, size_t min_pages, bool must_be_aligned) {
diff --git a/hotspot/src/share/vm/runtime/thread.cpp b/hotspot/src/share/vm/runtime/thread.cpp
index d111aff96..2be226463 100644
--- a/hotspot/src/share/vm/runtime/thread.cpp
+++ b/hotspot/src/share/vm/runtime/thread.cpp
@@ -326,7 +326,7 @@ void Thread::record_stack_base_and_size() {
   set_stack_base(os::current_stack_base());
   set_stack_size(os::current_stack_size());
   if (is_Java_thread()) {
-    ((JavaThread*) this)->set_stack_overflow_limit();
+    ((JavaThread*) this)->set_shadow_zone_limits();
   }
   // CR 7190089: on Solaris, primordial thread's stack is adjusted
   // in initialize_thread(). Without the adjustment, stack size is
@@ -1488,6 +1488,8 @@ void JavaThread::initialize() {
   _suspend_equivalent = false;
   _in_deopt_handler = 0;
   _doing_unsafe_access = false;
+  _shadow_zone_safe_limit = NULL;
+  _shadow_zone_growth_watermark = NULL;
   _stack_guard_state = stack_guard_unused;
   (void)const_cast<oop&>(_exception_oop = oop(NULL));
   _exception_pc  = 0;
diff --git a/hotspot/src/share/vm/runtime/thread.hpp b/hotspot/src/share/vm/runtime/thread.hpp
index 0e126964d..f1f0cdc58 100644
--- a/hotspot/src/share/vm/runtime/thread.hpp
+++ b/hotspot/src/share/vm/runtime/thread.hpp
@@ -561,6 +561,7 @@ protected:
   void    set_stack_base(address base) { _stack_base = base; }
   size_t  stack_size() const           { return _stack_size; }
   void    set_stack_size(size_t size)  { _stack_size = size; }
+  address stack_end()  const           { return stack_base() - stack_size(); }
   void    record_stack_base_and_size();
 
   bool    on_local_stack(address adr) const {
@@ -925,6 +926,9 @@ class JavaThread: public Thread {
   // We load it from here to simplify the stack overflow check in assembly.
   address          _stack_overflow_limit;
 
+  address          _shadow_zone_safe_limit;
+  address          _shadow_zone_growth_watermark;
+
   // Compiler exception handling (NOTE: The _exception_oop is *NOT* the same as _pending_exception. It is
   // used to temp. parsing values into and out of the runtime system during exception handling for compiled
   // code)
@@ -1313,6 +1317,14 @@ class JavaThread: public Thread {
   bool in_stack_red_zone(address a)
     { return (a <= stack_red_zone_base()) && (a >= (address)((intptr_t)stack_base() - stack_size())); }
 
+  static size_t stack_shadow_zone_size()
+    { return StackShadowPages * os::vm_page_size(); }
+
+  address shadow_zone_safe_limit() const {
+    assert(_shadow_zone_safe_limit != NULL, "Don't call this before the field is initialized.");
+    return _shadow_zone_safe_limit;
+  }
+
   void create_stack_guard_pages();
   void remove_stack_guard_pages();
 
@@ -1344,6 +1356,12 @@ class JavaThread: public Thread {
                               StackRedPages) * os::vm_page_size());
   }
 
+ void set_shadow_zone_limits() {
+  _shadow_zone_safe_limit =
+     Thread::stack_end() + JavaThread::stack_red_zone_size() + JavaThread::stack_yellow_zone_size() + JavaThread::stack_shadow_zone_size();
+  _shadow_zone_growth_watermark = JavaThread::stack_base();
+ }
+
   // Misc. accessors/mutators
   void set_do_not_unlock(void)                   { _do_not_unlock_if_synchronized = true; }
   void clr_do_not_unlock(void)                   { _do_not_unlock_if_synchronized = false; }
@@ -1381,6 +1399,8 @@ class JavaThread: public Thread {
   static ByteSize stack_overflow_limit_offset()  { return byte_offset_of(JavaThread, _stack_overflow_limit); }
   static ByteSize is_method_handle_return_offset() { return byte_offset_of(JavaThread, _is_method_handle_return); }
   static ByteSize stack_guard_state_offset()     { return byte_offset_of(JavaThread, _stack_guard_state   ); }
+  static ByteSize shadow_zone_safe_limit_offset() { return byte_offset_of(JavaThread, _shadow_zone_safe_limit);}
+  static ByteSize shadow_zone_growth_watermark_offset() { return byte_offset_of(JavaThread, _shadow_zone_growth_watermark);}
   static ByteSize suspend_flags_offset()         { return byte_offset_of(JavaThread, _suspend_flags       ); }
 
   static ByteSize do_not_unlock_if_synchronized_offset() { return byte_offset_of(JavaThread, _do_not_unlock_if_synchronized); }
-- 
2.22.0