diff --git a/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp b/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp index f1272ee1a2219..2cecb942dbf99 100644 --- a/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp +++ b/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp @@ -41,7 +41,22 @@ #define CHECK_BAILOUT() { if (ce->compilation()->bailed_out()) return; } void C1SafepointPollStub::emit_code(LIR_Assembler* ce) { - ShouldNotReachHere(); + __ bind(_entry); + address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point(); + + // Determine saved exception pc using pc relative address computation. + { + Label next_pc; + __ z_larl(Z_R1_scratch, next_pc); + __ bind(next_pc); + } + + int current_offset = __ offset(); + __ add2reg(Z_R1_scratch, safepoint_offset() - current_offset); + __ z_stg(Z_R1_scratch, Address(Z_thread, JavaThread::saved_exception_pc_offset())); + + __ load_const_optimized(Z_R1_scratch, (intptr_t)stub); + __ z_br(Z_R1_scratch); } void RangeCheckStub::emit_code(LIR_Assembler* ce) { diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp index e1d8d062c238c..29bfb89652af1 100644 --- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp @@ -1214,8 +1214,6 @@ void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) { (result->is_single_fpu() && result->as_float_reg() == Z_F0) || (result->is_double_fpu() && result->as_double_reg() == Z_F0), "convention"); - __ z_lg(Z_R1_scratch, Address(Z_thread, JavaThread::polling_page_offset())); - // Pop the frame before the safepoint code. __ pop_frame_restore_retPC(initial_frame_size_in_bytes()); @@ -1225,8 +1223,9 @@ void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) { // We need to mark the code position where the load from the safepoint // polling page was emitted as relocInfo::poll_return_type here. + code_stub->set_safepoint_offset(__ offset()); __ relocate(relocInfo::poll_return_type); - __ load_from_polling_page(Z_R1_scratch); + __ safepoint_poll(*code_stub->entry(), Z_R0_scratch, true /* at_return */, true /* in_nmethod */); __ z_br(Z_R14); // Return to caller. } diff --git a/src/hotspot/cpu/s390/downcallLinker_s390.cpp b/src/hotspot/cpu/s390/downcallLinker_s390.cpp index f1c41d05b5cf7..1e441b9e0cc3a 100644 --- a/src/hotspot/cpu/s390/downcallLinker_s390.cpp +++ b/src/hotspot/cpu/s390/downcallLinker_s390.cpp @@ -253,7 +253,7 @@ void DowncallLinker::StubGenerator::generate() { __ z_fence(); // Order state change wrt. safepoint poll. } - __ safepoint_poll(L_safepoint_poll_slow_path, tmp); + __ safepoint_poll(L_safepoint_poll_slow_path, tmp, true /* at_return */, false /* in_nmethod */); __ load_and_test_int(tmp, Address(Z_thread, JavaThread::suspend_flags_offset())); __ z_brne(L_safepoint_poll_slow_path); diff --git a/src/hotspot/cpu/s390/frame_s390.hpp b/src/hotspot/cpu/s390/frame_s390.hpp index bcdeec43e1a25..4f58864dd4ac8 100644 --- a/src/hotspot/cpu/s390/frame_s390.hpp +++ b/src/hotspot/cpu/s390/frame_s390.hpp @@ -560,4 +560,7 @@ static jint interpreter_frame_expression_stack_direction() { return -1; } + // returns the sending frame, without applying any barriers + inline frame sender_raw(RegisterMap* map) const; + #endif // CPU_S390_FRAME_S390_HPP diff --git a/src/hotspot/cpu/s390/frame_s390.inline.hpp b/src/hotspot/cpu/s390/frame_s390.inline.hpp index 6fcd36c57d17a..2848afbc7cd88 100644 --- a/src/hotspot/cpu/s390/frame_s390.inline.hpp +++ b/src/hotspot/cpu/s390/frame_s390.inline.hpp @@ -336,7 +336,7 @@ inline void frame::set_offset_unextended_sp(int value) { //------------------------------------------------------------------------------ // frame::sender -inline frame frame::sender(RegisterMap* map) const { +inline frame frame::sender_raw(RegisterMap* map) const { // Default is we don't have to follow them. The sender_for_xxx will // update it accordingly. map->set_include_argument_oops(false); @@ -353,6 +353,16 @@ inline frame frame::sender(RegisterMap* map) const { return frame(sender_sp(), sender_pc()); } +inline frame frame::sender(RegisterMap* map) const { + frame result = sender_raw(map); + + if (map->process_frames()) { + StackWatermarkSet::on_iteration(map->thread(), result); + } + + return result; +} + inline frame frame::sender_for_compiled_frame(RegisterMap *map) const { assert(map != nullptr, "map must be set"); diff --git a/src/hotspot/cpu/s390/interp_masm_s390.cpp b/src/hotspot/cpu/s390/interp_masm_s390.cpp index d5239898dd75d..72f6593d0327d 100644 --- a/src/hotspot/cpu/s390/interp_masm_s390.cpp +++ b/src/hotspot/cpu/s390/interp_masm_s390.cpp @@ -951,6 +951,7 @@ void InterpreterMacroAssembler::narrow(Register result, Register ret_type) { // remove activation // +// Apply stack watermark barrier. // Unlock the receiver if this is a synchronized method. // Unlock any Java monitors from synchronized blocks. // Remove the activation from the stack. @@ -970,6 +971,22 @@ void InterpreterMacroAssembler::remove_activation(TosState state, BLOCK_COMMENT("remove_activation {"); unlock_if_synchronized_method(state, throw_monitor_exception, install_monitor_exception); + // The below poll is for the stack watermark barrier. It allows fixing up frames lazily, + // that would normally not be safe to use. Such bad returns into unsafe territory of + // the stack, will call InterpreterRuntime::at_unwind. + Label slow_path, fast_path; + safepoint_poll(slow_path, Z_R0_scratch, true /* at_return */, false /* in_nmethod */); + branch_optimized(Assembler::bcondAlways, fast_path); + bind (slow_path); + push(state); + // TODO: I have followed ppc, but double check this + set_last_Java_frame(Z_SP, noreg); + call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), Z_thread); + reset_last_Java_frame(); + pop(state); + align(32); + bind(fast_path); + // Save result (push state before jvmti call and pop it afterwards) and notify jvmti. notify_method_exit(false, state, notify_jvmti ? NotifyJVMTI : SkipNotifyJVMTI); diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp index de3608e74ba6e..d9d7833535508 100644 --- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp @@ -2763,11 +2763,26 @@ uint MacroAssembler::get_poll_register(address instr_loc) { return 0; } -void MacroAssembler::safepoint_poll(Label& slow_path, Register temp_reg) { - const Address poll_byte_addr(Z_thread, in_bytes(JavaThread::polling_word_offset()) + 7 /* Big Endian */); - // Armed page has poll_bit set. - z_tm(poll_byte_addr, SafepointMechanism::poll_bit()); - z_brnaz(slow_path); +void MacroAssembler::safepoint_poll(Label& slow_path, Register tmp_reg, bool at_return, bool in_nmethod) { + const Address poll_byte_addr(Z_thread, in_bytes(JavaThread::polling_word_offset())); + + if (at_return) { + if (in_nmethod) { + z_clg(Z_SP, poll_byte_addr); + branch_optimized(Assembler::bcondHigh, slow_path); + } else { + // Frame still on stack, need to get fp. + Register fp = tmp_reg; + z_lg(fp, _z_abi(callers_sp), Z_SP); + z_clg(fp, poll_byte_addr); + branch_optimized(Assembler::bcondHigh, slow_path); + } + } else { + // Armed page has poll_bit set. + z_lg(tmp_reg, poll_byte_addr); + z_tmll(tmp_reg, SafepointMechanism::poll_bit()); + branch_optimized(Assembler::bcondNotAllZero, slow_path); + } } // Don't rely on register locking, always use Z_R1 as scratch register instead. diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp index 32e484d4790f2..31b1598f74d04 100644 --- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp +++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp @@ -642,7 +642,7 @@ class MacroAssembler: public Assembler { static uint get_poll_register(address instr_loc); // Check if safepoint requested and if so branch - void safepoint_poll(Label& slow_path, Register temp_reg); + void safepoint_poll(Label& slow_path, Register temp_reg, bool is_return, bool in_nmethod); // Stack overflow checking void bang_stack_with_offset(int offset); diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp index 00a830a80cd93..9c44c2ebd61ae 100644 --- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp +++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp @@ -1889,7 +1889,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, __ z_fence(); } - __ safepoint_poll(sync, Z_R1); + __ safepoint_poll(sync, Z_R1, true /* at_return */, false /* in_nmethod */); __ load_and_test_int(Z_R0, Address(Z_thread, JavaThread::suspend_flags_offset())); __ z_bre(no_block); diff --git a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp index 2da21f08bbcf1..e57b77c2a9dfa 100644 --- a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp +++ b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp @@ -1562,7 +1562,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) { // Check for safepoint operation in progress and/or pending suspend requests. { Label Continue, do_safepoint; - __ safepoint_poll(do_safepoint, Z_R1); + __ safepoint_poll(do_safepoint, Z_R1, true /* at_return */, false /* in_nmethod */); // Check for suspend. __ load_and_test_int(Z_R0/*suspend_flags*/, thread_(suspend_flags)); __ z_bre(Continue); // 0 -> no flag set -> not suspended @@ -1839,7 +1839,8 @@ address TemplateInterpreterGenerator::generate_CRC32_update_entry() { Label slow_path; // If we need a safepoint check, generate full interpreter entry. - __ safepoint_poll(slow_path, Z_R1); + // TODO:on x86 they are passing at_return true + __ safepoint_poll(slow_path, Z_R1, false /* at_return */, false /* in_nemthod */); BLOCK_COMMENT("CRC32_update {"); @@ -1888,7 +1889,7 @@ address TemplateInterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractI Label slow_path; // If we need a safepoint check, generate full interpreter entry. - __ safepoint_poll(slow_path, Z_R1); + __ safepoint_poll(slow_path, Z_R1, false /* at_return */, false /* in_nmethod */); // We don't generate local frame and don't align stack because // we call stub code and there is no safepoint on this path. diff --git a/src/hotspot/cpu/s390/vm_version_s390.hpp b/src/hotspot/cpu/s390/vm_version_s390.hpp index 591d30c3a1c48..7783c4602fe6d 100644 --- a/src/hotspot/cpu/s390/vm_version_s390.hpp +++ b/src/hotspot/cpu/s390/vm_version_s390.hpp @@ -427,6 +427,8 @@ class VM_Version: public Abstract_VM_Version { constexpr static bool supports_recursive_fast_locking() { return true; } + constexpr static bool supports_stack_watermark_barrier() { return true; } + // CPU feature query functions static const char* get_model_string() { return _model_string; } static bool has_StoreFacilityListExtended() { return (_features[0] & StoreFacilityListExtendedMask) == StoreFacilityListExtendedMask; }