concurrent thread-stack processing for interpreter

src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp

@@ -41,7 +41,22 @@
 #define CHECK_BAILOUT() { if (ce->compilation()->bailed_out()) return; }
 
 void C1SafepointPollStub::emit_code(LIR_Assembler* ce) {
-  ShouldNotReachHere();
+  __ bind(_entry);
+  address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point();
+
+  // Determine saved exception pc using pc relative address computation.
+  {
+    Label next_pc;
+    __ z_larl(Z_R1_scratch, next_pc);
+    __ bind(next_pc);
+  }
+
+  int current_offset = __ offset();
+  __ add2reg(Z_R1_scratch, safepoint_offset() - current_offset);
+  __ z_stg(Z_R1_scratch, Address(Z_thread, JavaThread::saved_exception_pc_offset()));
+
+  __ load_const_optimized(Z_R1_scratch, (intptr_t)stub);
+  __ z_br(Z_R1_scratch);
 }
 
 void RangeCheckStub::emit_code(LIR_Assembler* ce) {

src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp

@@ -1214,8 +1214,6 @@ void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
          (result->is_single_fpu() && result->as_float_reg() == Z_F0) ||
          (result->is_double_fpu() && result->as_double_reg() == Z_F0), "convention");
 
-  __ z_lg(Z_R1_scratch, Address(Z_thread, JavaThread::polling_page_offset()));
-
   // Pop the frame before the safepoint code.
   __ pop_frame_restore_retPC(initial_frame_size_in_bytes());
 
@@ -1225,8 +1223,9 @@ void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 
   // We need to mark the code position where the load from the safepoint
   // polling page was emitted as relocInfo::poll_return_type here.
+  code_stub->set_safepoint_offset(__ offset());
   __ relocate(relocInfo::poll_return_type);
-  __ load_from_polling_page(Z_R1_scratch);
+  __ safepoint_poll(*code_stub->entry(), Z_R0_scratch, true /* at_return */, true /* in_nmethod */);
 
   __ z_br(Z_R14); // Return to caller.
 }

src/hotspot/cpu/s390/downcallLinker_s390.cpp

@@ -253,7 +253,7 @@ void DowncallLinker::StubGenerator::generate() {
       __ z_fence(); // Order state change wrt. safepoint poll.
     }
 
-    __ safepoint_poll(L_safepoint_poll_slow_path, tmp);
+    __ safepoint_poll(L_safepoint_poll_slow_path, tmp, true /* at_return */, false /* in_nmethod */);
 
     __ load_and_test_int(tmp, Address(Z_thread, JavaThread::suspend_flags_offset()));
     __ z_brne(L_safepoint_poll_slow_path);

src/hotspot/cpu/s390/frame_s390.hpp

@@ -560,4 +560,7 @@
 
   static jint interpreter_frame_expression_stack_direction() { return -1; }
 
+  // returns the sending frame, without applying any barriers
+  inline frame sender_raw(RegisterMap* map) const;
+
 #endif // CPU_S390_FRAME_S390_HPP

src/hotspot/cpu/s390/frame_s390.inline.hpp

@@ -336,7 +336,7 @@ inline void frame::set_offset_unextended_sp(int value) {
 //------------------------------------------------------------------------------
 // frame::sender
 
-inline frame frame::sender(RegisterMap* map) const {
+inline frame frame::sender_raw(RegisterMap* map) const {
   // Default is we don't have to follow them. The sender_for_xxx will
   // update it accordingly.
   map->set_include_argument_oops(false);
@@ -353,6 +353,16 @@ inline frame frame::sender(RegisterMap* map) const {
   return frame(sender_sp(), sender_pc());
 }
 
+inline frame frame::sender(RegisterMap* map) const {
+  frame result = sender_raw(map);
+
+  if (map->process_frames()) {
+    StackWatermarkSet::on_iteration(map->thread(), result);
+  }
+
+  return result;
+}
+
 inline frame frame::sender_for_compiled_frame(RegisterMap *map) const {
   assert(map != nullptr, "map must be set");
 

src/hotspot/cpu/s390/interp_masm_s390.cpp

@@ -951,6 +951,7 @@ void InterpreterMacroAssembler::narrow(Register result, Register ret_type) {
 
 // remove activation
 //
+// Apply stack watermark barrier.
 // Unlock the receiver if this is a synchronized method.
 // Unlock any Java monitors from synchronized blocks.
 // Remove the activation from the stack.
@@ -970,6 +971,22 @@ void InterpreterMacroAssembler::remove_activation(TosState state,
   BLOCK_COMMENT("remove_activation {");
   unlock_if_synchronized_method(state, throw_monitor_exception, install_monitor_exception);
 
+  // The below poll is for the stack watermark barrier. It allows fixing up frames lazily,
+  // that would normally not be safe to use. Such bad returns into unsafe territory of
+  // the stack, will call InterpreterRuntime::at_unwind.
+  Label slow_path, fast_path;
+  safepoint_poll(slow_path, Z_R0_scratch, true /* at_return */, false /* in_nmethod */);
+  branch_optimized(Assembler::bcondAlways, fast_path);
+  bind (slow_path);
+  push(state);
+  // TODO: I have followed ppc, but double check this
+  set_last_Java_frame(Z_SP, noreg);
+  call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), Z_thread);
+  reset_last_Java_frame();
+  pop(state);
+  align(32);
+  bind(fast_path);
+
   // Save result (push state before jvmti call and pop it afterwards) and notify jvmti.
   notify_method_exit(false, state, notify_jvmti ? NotifyJVMTI : SkipNotifyJVMTI);
 

src/hotspot/cpu/s390/macroAssembler_s390.cpp

@@ -2763,11 +2763,26 @@ uint MacroAssembler::get_poll_register(address instr_loc) {
   return 0;
 }
 
-void MacroAssembler::safepoint_poll(Label& slow_path, Register temp_reg) {
-  const Address poll_byte_addr(Z_thread, in_bytes(JavaThread::polling_word_offset()) + 7 /* Big Endian */);
-  // Armed page has poll_bit set.
-  z_tm(poll_byte_addr, SafepointMechanism::poll_bit());
-  z_brnaz(slow_path);
+void MacroAssembler::safepoint_poll(Label& slow_path, Register tmp_reg, bool at_return, bool in_nmethod) {
+  const Address poll_byte_addr(Z_thread, in_bytes(JavaThread::polling_word_offset()));
+
+  if (at_return) {
+    if (in_nmethod) {
+      z_clg(Z_SP, poll_byte_addr);
+      branch_optimized(Assembler::bcondHigh, slow_path);
+    } else {
+      // Frame still on stack, need to get fp.
+      Register fp = tmp_reg;
+      z_lg(fp, _z_abi(callers_sp), Z_SP);
+      z_clg(fp, poll_byte_addr);
+      branch_optimized(Assembler::bcondHigh, slow_path);
+    }
+  } else {
+    // Armed page has poll_bit set.
+    z_lg(tmp_reg, poll_byte_addr);
+    z_tmll(tmp_reg, SafepointMechanism::poll_bit());
+    branch_optimized(Assembler::bcondNotAllZero, slow_path);
+  }
 }
 
 // Don't rely on register locking, always use Z_R1 as scratch register instead.

src/hotspot/cpu/s390/macroAssembler_s390.hpp

@@ -642,7 +642,7 @@ class MacroAssembler: public Assembler {
   static uint get_poll_register(address instr_loc);
 
   // Check if safepoint requested and if so branch
-  void safepoint_poll(Label& slow_path, Register temp_reg);
+  void safepoint_poll(Label& slow_path, Register temp_reg, bool is_return, bool in_nmethod);
 
   // Stack overflow checking
   void bang_stack_with_offset(int offset);

src/hotspot/cpu/s390/sharedRuntime_s390.cpp

@@ -1889,7 +1889,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
       __ z_fence();
     }
 
-    __ safepoint_poll(sync, Z_R1);
+    __ safepoint_poll(sync, Z_R1, true /* at_return */, false /* in_nmethod */);
 
     __ load_and_test_int(Z_R0, Address(Z_thread, JavaThread::suspend_flags_offset()));
     __ z_bre(no_block);

src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp

@@ -1562,7 +1562,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
   // Check for safepoint operation in progress and/or pending suspend requests.
   {
     Label Continue, do_safepoint;
-    __ safepoint_poll(do_safepoint, Z_R1);
+    __ safepoint_poll(do_safepoint, Z_R1, true /* at_return */, false /* in_nmethod */);
     // Check for suspend.
     __ load_and_test_int(Z_R0/*suspend_flags*/, thread_(suspend_flags));
     __ z_bre(Continue); // 0 -> no flag set -> not suspended
@@ -1839,7 +1839,8 @@ address TemplateInterpreterGenerator::generate_CRC32_update_entry() {
   Label    slow_path;
 
   // If we need a safepoint check, generate full interpreter entry.
-  __ safepoint_poll(slow_path, Z_R1);
+  // TODO:on x86 they are passing at_return true
+  __ safepoint_poll(slow_path, Z_R1, false /* at_return */, false /* in_nemthod */);
 
   BLOCK_COMMENT("CRC32_update {");
 
@@ -1888,7 +1889,7 @@ address TemplateInterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractI
   Label    slow_path;
 
   // If we need a safepoint check, generate full interpreter entry.
-  __ safepoint_poll(slow_path, Z_R1);
+  __ safepoint_poll(slow_path, Z_R1, false /* at_return */, false /* in_nmethod */);
 
   // We don't generate local frame and don't align stack because
   // we call stub code and there is no safepoint on this path.

src/hotspot/cpu/s390/vm_version_s390.hpp

@@ -427,6 +427,8 @@ class VM_Version: public Abstract_VM_Version {
 
   constexpr static bool supports_recursive_fast_locking() { return true; }
 
+  constexpr static bool supports_stack_watermark_barrier() { return true; }
+
   // CPU feature query functions
   static const char* get_model_string()       { return _model_string; }
   static bool has_StoreFacilityListExtended() { return  (_features[0] & StoreFacilityListExtendedMask) == StoreFacilityListExtendedMask; }