From c1c82dce696aef6a0b2c37a56f2506b537130961 Mon Sep 17 00:00:00 2001 From: yufengzjj Date: Sat, 16 May 2026 16:00:02 +0900 Subject: [PATCH 1/3] add notebook>7 support --- README.adoc | 6 ++++ ipyida/notebook.py | 90 +++++++++++++++++++++++++++++++++++++++++----- setup.py | 11 ++++-- 3 files changed, 97 insertions(+), 10 deletions(-) mode change 100755 => 100644 setup.py diff --git a/README.adoc b/README.adoc index ad4fccf..7319f4a 100644 --- a/README.adoc +++ b/README.adoc @@ -71,6 +71,12 @@ notebook attached to IPyIDA. The command takes care of installing dependencies on its first run and starting a Notebook server unless one is already running. Check the command help (by typing `%open_notebook?`) for further options. +Both `notebook` 6.x and 7.x are supported. On 7.x (which is served by +`jupyter-server`) IPyIDA pre-creates a kernel session against the `proxy` +kernel via the REST API so the JupyterLab-based frontend attaches to it on +load — the legacy `?kernel_name=` query string alone is not honoured on the +new UI. + == Customizing the IPython console By default, the console does not have any globals available. If you want to diff --git a/ipyida/notebook.py b/ipyida/notebook.py index 4d5f533..5268e80 100644 --- a/ipyida/notebook.py +++ b/ipyida/notebook.py @@ -15,12 +15,48 @@ import json import threading import webbrowser +import urllib.request +import urllib.error import idaapi import nbformat from jupyter_client.kernelspec import find_kernel_specs from jupyter_client import find_connection_file + +def _notebook_major_version(): + try: + import notebook + except ImportError: + return None + try: + return int(notebook.__version__.split('.')[0]) + except (AttributeError, ValueError): + return None + + +def _list_running_servers(): + """Return an iterator over running notebook/jupyter servers. + + notebook 7 dropped ``notebook.notebookapp`` and runs on top of + ``jupyter_server``; notebook 6 still exposes its own server list. + """ + major = _notebook_major_version() + if major is not None and major >= 7: + from jupyter_server.serverapp import list_running_servers + else: + from notebook.notebookapp import list_running_servers + return list_running_servers() + + +def _server_root_dir(server_info): + """Return the root directory of a running server. + + notebook 6 reports ``notebook_dir``; jupyter_server (notebook 7) reports + ``root_dir``. + """ + return server_info.get("root_dir") or server_info.get("notebook_dir") or "" + def _popen_python_module(module, *args, **kwargs): # We can't rely on sys.executable because it's set to ida{q,t}{.exe,} in IDA if sys.platform == 'win32': @@ -78,16 +114,50 @@ def ensure_notebook_installed(): except ImportError: print("-> Installing jupyter-notebook...") return _popen_python_module( - "pip", "install", "notebook<7" + "pip", "install", "notebook" ).wait() == 0 else: return True def _get_running_notebook_config(self): - from notebook.notebookapp import list_running_servers idb_path = idaapi.get_path(idaapi.PATH_TYPE_IDB) - is_idb_under_nb_dir = lambda c: idb_path.startswith(c.get("notebook_dir")) - return next(filter(is_idb_under_nb_dir, list_running_servers()), None) + for server_info in _list_running_servers(): + root = _server_root_dir(server_info) + if root and idb_path.startswith(root): + return server_info + return None + + @staticmethod + def _create_proxy_session(server_info, relative_path): + """Pre-create a kernel session bound to the proxy kernel. + + Needed on notebook 7 because its JupyterLab-based frontend does not + honour the legacy ``?kernel_name=`` query string the way notebook 6 + did. Posting the session beforehand makes the page reuse the + already-attached proxy kernel when it opens. Harmless on notebook 6 + (the existing session is reused instead of creating a duplicate). + """ + base = server_info.get("url", "").rstrip("/") + token = server_info.get("token", "") + if not base: + return + body = json.dumps({ + "path": "/".join(relative_path.split(os.path.sep)), + "type": "notebook", + "name": "", + "kernel": {"name": "proxy"}, + }).encode("utf-8") + headers = {"Content-Type": "application/json"} + if token: + headers["Authorization"] = "token " + token + req = urllib.request.Request( + base + "/api/sessions", data=body, headers=headers, method="POST", + ) + try: + urllib.request.urlopen(req, timeout=10).read() + except (urllib.error.URLError, OSError) as e: + # Non-fatal: fall back to the URL-query selection path. + print("-> Could not pre-create proxy session: %s" % e) def _parse_args(self, line): args = line.split() @@ -156,14 +226,18 @@ def open_notebook(self, line): with open(ipynb_path, "w") as f: nb = nbformat.versions[nbformat.current_nbformat].new_notebook() json.dump(nb, f) - relative_path = os.path.relpath(ipynb_path, nb_server_info.get("notebook_dir")) - url = nb_server_info.get("url") + \ - "notebooks/" + "/".join(relative_path.split(os.path.sep)) + \ - '?kernel_name=proxy&token=' + nb_server_info.get("token") + relative_path = os.path.relpath(ipynb_path, _server_root_dir(nb_server_info)) # Update access time of the file so it's picked up by the proxy. # jupyter-kernel-proxy will use the file with the most recent access # time (like `jupyter console --existing`) with open(find_connection_file(self.connection_file), "r"): pass + # On notebook 7 the JupyterLab-based frontend ignores the + # ?kernel_name= query argument. Posting a session in advance attaches + # the proxy kernel so the notebook page picks it up on load. + self._create_proxy_session(nb_server_info, relative_path) + url = nb_server_info.get("url") + \ + "notebooks/" + "/".join(relative_path.split(os.path.sep)) + \ + '?kernel_name=proxy&token=' + nb_server_info.get("token") webbrowser.open(url) return url diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 6e451ef..e570e8c --- a/setup.py +++ b/setup.py @@ -33,14 +33,21 @@ 'ipykernel>=5.1.4; python_version >= "3.8" and platform_system=="Windows"', 'qtconsole>=4.3', 'qasync; python_version >= "3"', - 'jupyter-client<6.1.13', + 'jupyter-client!=6.1.13', 'nbformat', ], extras_require={ "notebook": [ "notebook<7", + "jupyter-client<7", "jupyter-kernel-proxy", - ] + ], + "notebook7": [ + "notebook>=7", + "jupyter-server>=2", + "jupyter-client>=7.4.4", + "jupyter-kernel-proxy", + ], }, license="BSD", classifiers=[ From 2f37fe84db7d6282176f4e1796d620ccc23c298f Mon Sep 17 00:00:00 2001 From: yufengzjj Date: Sat, 16 May 2026 19:53:11 +0900 Subject: [PATCH 2/3] add debugger support for notebook>7 fix a bug when jupyter_core's dispatch to the jupyter-notebook script --- ipyida/kernel.py | 19 +++++++++++ ipyida/notebook.py | 76 +++++++++++++++++++++++++++++++++--------- ipyida/proxy_runner.py | 48 ++++++++++++++++++++++++++ setup.py | 1 + 4 files changed, 129 insertions(+), 15 deletions(-) create mode 100644 ipyida/proxy_runner.py diff --git a/ipyida/kernel.py b/ipyida/kernel.py index 814b191..9fb29f5 100644 --- a/ipyida/kernel.py +++ b/ipyida/kernel.py @@ -47,6 +47,23 @@ def is_using_ipykernel_5(): import ipykernel return hasattr(ipykernel.kernelbase.Kernel, "process_one") + +def _configure_debugpy_python(): + # ipykernel's Debugger drives debugpy.listen(), which spawns its DAP + # adapter via sys.executable. In IDA that's idaq.exe, so the adapter + # never starts and listen() hits its 30 s accept() timeout -- this is + # what makes the JupyterLab/Notebook 7 debug button hang then error + # with "timed out waiting for adapter to connect". Point debugpy at + # the real interpreter backing this environment. + try: + import debugpy + from .notebook import _python_executable + except ImportError: + return + python = _python_executable() + if os.path.exists(python): + debugpy.configure(python=python) + def get_ea_bounds(): """ Wraps getting the min and max ea to use either inf_get_min_ea @@ -166,6 +183,8 @@ def start(self): self.connection_file = app.connection_file + _configure_debugpy_python() + if not is_using_ipykernel_5(): app.kernel.do_one_iteration() diff --git a/ipyida/notebook.py b/ipyida/notebook.py index 5268e80..d830f10 100644 --- a/ipyida/notebook.py +++ b/ipyida/notebook.py @@ -57,22 +57,28 @@ def _server_root_dir(server_info): """ return server_info.get("root_dir") or server_info.get("notebook_dir") or "" -def _popen_python_module(module, *args, **kwargs): - # We can't rely on sys.executable because it's set to ida{q,t}{.exe,} in IDA +def _python_executable(): + # sys.executable in IDA is ida{q,t}{.exe,}, not a Python interpreter. + # Locate the real Python that backs this environment instead. if sys.platform == 'win32': - # Try in Scripts first. If a virtualenv is activated, Python.exe will - # be there + # Virtualenvs put Python.exe in Scripts/; regular installs at sys.prefix. python = os.path.join(sys.prefix, 'Scripts', 'Python.exe') if not os.path.exists(python): python = os.path.join(sys.prefix, 'Python.exe') - si_hidden_window = subprocess.STARTUPINFO() - si_hidden_window.dwFlags = subprocess.STARTF_USESHOWWINDOW - si_hidden_window.wShowWindow = subprocess.SW_HIDE - kwargs["startupinfo"] = si_hidden_window else: python = os.path.join(sys.prefix, 'bin', 'python') if sys.version_info.major >= 3: python += str(sys.version_info.major) + return python + + +def _popen_python_module(module, *args, **kwargs): + python = _python_executable() + if sys.platform == 'win32': + si_hidden_window = subprocess.STARTUPINFO() + si_hidden_window.dwFlags = subprocess.STARTF_USESHOWWINDOW + si_hidden_window.wShowWindow = subprocess.SW_HIDE + kwargs["startupinfo"] = si_hidden_window return subprocess.Popen([ python, "-m", module ] + list(args), **kwargs) @@ -99,13 +105,47 @@ def ensure_kernel_proxy_installed(): @staticmethod def ensure_kernelspec_installed(): - if "proxy" not in find_kernel_specs(): + specs = find_kernel_specs() + if "proxy" not in specs: print("-> Installing jupyter-kernel-proxy kernelspec...") - return _popen_python_module( - "jupyter_kernel_proxy", "install" - ).wait() == 0 - else: - return True + if _popen_python_module("jupyter_kernel_proxy", "install").wait() != 0: + return False + specs = find_kernel_specs() + if "proxy" not in specs: + return False + # Two patches on the kernelspec, idempotent: + # 1. argv -> our proxy_runner wrapper, which suppresses the + # fallback kernel_info_reply that lacks fields like + # ``implementation: "ipython"``. + # 2. metadata.debugger = true. JupyterLab / Notebook 7 enable the + # debug toolbar button purely from kernelspec metadata + # (see jupyterlab/packages/debugger/src/service.ts:isAvailable), + # not from kernel_info_reply. jupyter_kernel_proxy ships + # without it, so the button stays greyed out. + spec_path = os.path.join(specs["proxy"], "kernel.json") + try: + with open(spec_path, "r") as f: + spec = json.load(f) + changed = False + current_argv = spec.get("argv") or [] + desired_argv = ( + [current_argv[0] if current_argv else sys.executable] + + ["-m", "ipyida.proxy_runner", "{connection_file}"] + ) + if current_argv != desired_argv: + spec["argv"] = desired_argv + changed = True + metadata = spec.setdefault("metadata", {}) + if not metadata.get("debugger"): + metadata["debugger"] = True + changed = True + if changed: + with open(spec_path, "w") as f: + json.dump(spec, f) + except (OSError, ValueError) as e: + print("-> Could not patch proxy kernelspec: %s" % e) + return False + return True @staticmethod def ensure_notebook_installed(): @@ -199,8 +239,14 @@ def open_notebook(self, line): if nb_server_info is None: print("-> Starting notebook") + # Use ``python -m notebook`` instead of ``python -m jupyter notebook`` + # to bypass jupyter_core's dispatch to the ``jupyter-notebook`` script + # on PATH -- pip downgrades between notebook 7 and 6 can leave that + # script pointing at the wrong module (e.g. 7's ``notebook.app`` + # after a downgrade to 6). ``notebook/__main__.py`` exists in both + # 6 and 7 and routes to the right entry point. self.nb_proc = _popen_python_module( - "jupyter", "notebook", "--no-browser", "-y", + "notebook", "--no-browser", "-y", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True ) diff --git a/ipyida/proxy_runner.py b/ipyida/proxy_runner.py new file mode 100644 index 0000000..25aa1b8 --- /dev/null +++ b/ipyida/proxy_runner.py @@ -0,0 +1,48 @@ +# -*- encoding: utf8 -*- +# +# Wrapper around ``jupyter_kernel_proxy`` that patches its fallback +# ``kernel_info_reply`` so the JupyterLab/Notebook 7 debug button stays +# enabled when a real kernel is already connected to the proxy. +# +# ``jupyter_kernel_proxy.KernelProxyManager._send_proxy_kernel_info`` is a +# 3-second fallback: if the real kernel does not reply to ``kernel_info_request`` +# within that window the proxy synthesises a minimal reply that lacks the +# ``debugger`` field (and others). On notebook 7 / JupyterLab the debug +# button is enabled based on ``kernel_info_reply.debugger`` -- the +# fallback racing the real reply silently disables it. +# +# This wrapper suppresses the fallback when a real ``proxy_target`` is +# connected, so the front-end always sees the real kernel's reply. + +import sys + +import jupyter_kernel_proxy + + +_orig_send_proxy_kernel_info = ( + jupyter_kernel_proxy.KernelProxyManager._send_proxy_kernel_info +) + + +def _patched_send_proxy_kernel_info(self, request): + if getattr(self.server, "proxy_target", None) is not None: + # Real kernel will answer; do not race it with a stripped-down + # fallback reply that drops the ``debugger`` flag. + return + return _orig_send_proxy_kernel_info(self, request) + + +jupyter_kernel_proxy.KernelProxyManager._send_proxy_kernel_info = ( + _patched_send_proxy_kernel_info +) + + +def main(): + if len(sys.argv) < 2: + print("Usage: python -m ipyida.proxy_runner ") + sys.exit(1) + jupyter_kernel_proxy.start(sys.argv[1]) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index e570e8c..21230e0 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ extras_require={ "notebook": [ "notebook<7", + "jupyter-server<2", "jupyter-client<7", "jupyter-kernel-proxy", ], From 316cc88d030e2d035f320945fae2c9d9a053f59b Mon Sep 17 00:00:00 2001 From: yufengzjj Date: Sat, 16 May 2026 22:48:16 +0900 Subject: [PATCH 3/3] close notebook gracefully fix IDA unexpected exit when shutdown in notebook --- ipyida/notebook.py | 332 ++++++++++++++++++++++++++++++++++------- ipyida/proxy_runner.py | 106 ++++++++++--- 2 files changed, 368 insertions(+), 70 deletions(-) diff --git a/ipyida/notebook.py b/ipyida/notebook.py index d830f10..4e194af 100644 --- a/ipyida/notebook.py +++ b/ipyida/notebook.py @@ -10,6 +10,7 @@ import sys import os +import shutil import subprocess import time import json @@ -20,8 +21,9 @@ import idaapi import nbformat -from jupyter_client.kernelspec import find_kernel_specs +import psutil from jupyter_client import find_connection_file +from jupyter_core.paths import jupyter_data_dir def _notebook_major_version(): @@ -72,6 +74,130 @@ def _python_executable(): return python +# --- Subprocess lifetime binding ------------------------------------------- +# +# IDA's plugin term() is best-effort: if IDA crashes, is killed from Task +# Manager, or is shut down indirectly (e.g. the user clicks "Shutdown" in +# the notebook browser tab, which the proxy kernel forwards to IDA's +# embedded kernel), term() never runs and the notebook subprocess survives +# as an orphan -- still bound to port 8888 with stale runtime files in +# ``%JUPYTER_RUNTIME_DIR%``. The next IDA session can't reach its own +# kernel through that stale server. +# +# Bind the subprocess to IDA's process lifetime so the OS itself kills it +# when IDA goes away: +# * Windows: a Job Object with JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE. The +# job handle is held by IDA; when IDA's handle table is torn down, the +# last handle closes and the OS terminates every process in the job. +# * Linux: prctl(PR_SET_PDEATHSIG, SIGTERM) in the child via preexec_fn. + +if sys.platform == 'win32': + import ctypes + from ctypes import wintypes + + _kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + + _JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000 + _JobObjectExtendedLimitInformation = 9 + + class _IO_COUNTERS(ctypes.Structure): + _fields_ = [ + ("ReadOperationCount", ctypes.c_ulonglong), + ("WriteOperationCount", ctypes.c_ulonglong), + ("OtherOperationCount", ctypes.c_ulonglong), + ("ReadTransferCount", ctypes.c_ulonglong), + ("WriteTransferCount", ctypes.c_ulonglong), + ("OtherTransferCount", ctypes.c_ulonglong), + ] + + class _JOBOBJECT_BASIC_LIMIT_INFORMATION(ctypes.Structure): + _fields_ = [ + ("PerProcessUserTimeLimit", wintypes.LARGE_INTEGER), + ("PerJobUserTimeLimit", wintypes.LARGE_INTEGER), + ("LimitFlags", wintypes.DWORD), + ("MinimumWorkingSetSize", ctypes.c_size_t), + ("MaximumWorkingSetSize", ctypes.c_size_t), + ("ActiveProcessLimit", wintypes.DWORD), + ("Affinity", ctypes.c_size_t), + ("PriorityClass", wintypes.DWORD), + ("SchedulingClass", wintypes.DWORD), + ] + + class _JOBOBJECT_EXTENDED_LIMIT_INFORMATION(ctypes.Structure): + _fields_ = [ + ("BasicLimitInformation", _JOBOBJECT_BASIC_LIMIT_INFORMATION), + ("IoInfo", _IO_COUNTERS), + ("ProcessMemoryLimit", ctypes.c_size_t), + ("JobMemoryLimit", ctypes.c_size_t), + ("PeakProcessMemoryUsed", ctypes.c_size_t), + ("PeakJobMemoryUsed", ctypes.c_size_t), + ] + + _kernel32.CreateJobObjectW.restype = wintypes.HANDLE + _kernel32.CreateJobObjectW.argtypes = [wintypes.LPVOID, wintypes.LPCWSTR] + _kernel32.SetInformationJobObject.restype = wintypes.BOOL + _kernel32.SetInformationJobObject.argtypes = [ + wintypes.HANDLE, ctypes.c_int, wintypes.LPVOID, wintypes.DWORD, + ] + _kernel32.AssignProcessToJobObject.restype = wintypes.BOOL + _kernel32.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE] + + _ipyida_job_handle = None + + def _get_ipyida_job(): + global _ipyida_job_handle + if _ipyida_job_handle is not None: + return _ipyida_job_handle + job = _kernel32.CreateJobObjectW(None, None) + if not job: + return None + info = _JOBOBJECT_EXTENDED_LIMIT_INFORMATION() + info.BasicLimitInformation.LimitFlags = _JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + if not _kernel32.SetInformationJobObject( + job, _JobObjectExtendedLimitInformation, + ctypes.byref(info), ctypes.sizeof(info), + ): + _kernel32.CloseHandle(job) + return None + _ipyida_job_handle = job + return job + + def _bind_proc_to_ida_lifetime(proc): + job = _get_ipyida_job() + if job is None: + return + handle = getattr(proc, "_handle", None) + if handle is None: + return + if not _kernel32.AssignProcessToJobObject(job, int(handle)): + # Pre-Win8 a process can only belong to one job; if IDA is + # already inside an outer job that forbids breakaway we can't + # nest. Falling back to the term() path is still safe. + err = ctypes.get_last_error() + print("-> AssignProcessToJobObject failed (err=%d); " + "notebook will not be auto-killed on IDA crash" % err) + +else: + def _bind_proc_to_ida_lifetime(proc): + # Lifetime binding on Linux is set up in the child via preexec_fn; + # nothing to do in the parent after Popen. + pass + + +def _child_set_pdeathsig(): + # Linux only: ask the kernel to send SIGTERM when the parent dies. + if not sys.platform.startswith('linux'): + return + try: + import ctypes + libc = ctypes.CDLL("libc.so.6", use_errno=True) + PR_SET_PDEATHSIG = 1 + SIGTERM = 15 + libc.prctl(PR_SET_PDEATHSIG, SIGTERM, 0, 0, 0) + except Exception: + pass + + def _popen_python_module(module, *args, **kwargs): python = _python_executable() if sys.platform == 'win32': @@ -79,7 +205,11 @@ def _popen_python_module(module, *args, **kwargs): si_hidden_window.dwFlags = subprocess.STARTF_USESHOWWINDOW si_hidden_window.wShowWindow = subprocess.SW_HIDE kwargs["startupinfo"] = si_hidden_window - return subprocess.Popen([ python, "-m", module ] + list(args), **kwargs) + elif sys.platform.startswith('linux'): + kwargs.setdefault("preexec_fn", _child_set_pdeathsig) + proc = subprocess.Popen([ python, "-m", module ] + list(args), **kwargs) + _bind_proc_to_ida_lifetime(proc) + return proc class NotebookManager(object): @@ -103,47 +233,78 @@ def ensure_kernel_proxy_installed(): else: return True + def _kernelspec_name(self): + # One kernelspec per IDA process so concurrent IDAs don't fight + # over a single shared spec (each spec pins its proxy to a + # different IDA via env.IPYIDA_KERNEL_FILE). + return "ipyida-%d" % os.getpid() + @staticmethod - def ensure_kernelspec_installed(): - specs = find_kernel_specs() - if "proxy" not in specs: - print("-> Installing jupyter-kernel-proxy kernelspec...") - if _popen_python_module("jupyter_kernel_proxy", "install").wait() != 0: - return False - specs = find_kernel_specs() - if "proxy" not in specs: - return False - # Two patches on the kernelspec, idempotent: - # 1. argv -> our proxy_runner wrapper, which suppresses the - # fallback kernel_info_reply that lacks fields like - # ``implementation: "ipython"``. - # 2. metadata.debugger = true. JupyterLab / Notebook 7 enable the - # debug toolbar button purely from kernelspec metadata - # (see jupyterlab/packages/debugger/src/service.ts:isAvailable), - # not from kernel_info_reply. jupyter_kernel_proxy ships - # without it, so the button stays greyed out. - spec_path = os.path.join(specs["proxy"], "kernel.json") + def _cleanup_stale_kernelspecs(): + # Remove ipyida- kernelspec directories whose IDA process + # is no longer running -- otherwise crashed/force-killed IDAs + # leave entries cluttering JupyterLab's Launcher forever. + kernels_dir = os.path.join(jupyter_data_dir(), "kernels") + if not os.path.isdir(kernels_dir): + return + prefix = "ipyida-" + for name in os.listdir(kernels_dir): + if not name.startswith(prefix): + continue + try: + pid = int(name[len(prefix):]) + except ValueError: + continue + if pid == os.getpid(): + continue + if psutil.pid_exists(pid): + continue + shutil.rmtree(os.path.join(kernels_dir, name), ignore_errors=True) + + def ensure_kernelspec_installed(self): + """Install (or refresh) this IDA instance's own proxy kernelspec. + + Each IDA writes ``/kernels/ipyida-/kernel.json`` + with: + - argv: our ``ipyida.proxy_runner`` wrapper, plus IDA's + connection-file basename as the second positional arg. + proxy_runner uses it to dial the right kernel directly, + instead of guessing by ``kernel-*.json`` atime (which breaks + after a few open/shutdown cycles). + - metadata.debugger=true so JupyterLab / Notebook 7 enable the + debug toolbar button. + """ + # jupyter_kernel_proxy itself must be importable; proxy_runner + # wraps it. We do not call its ``install`` -- we install our + # own per-IDA spec rather than mutating the shared "proxy" one. try: - with open(spec_path, "r") as f: - spec = json.load(f) - changed = False - current_argv = spec.get("argv") or [] - desired_argv = ( - [current_argv[0] if current_argv else sys.executable] - + ["-m", "ipyida.proxy_runner", "{connection_file}"] - ) - if current_argv != desired_argv: - spec["argv"] = desired_argv - changed = True - metadata = spec.setdefault("metadata", {}) - if not metadata.get("debugger"): - metadata["debugger"] = True - changed = True - if changed: - with open(spec_path, "w") as f: - json.dump(spec, f) - except (OSError, ValueError) as e: - print("-> Could not patch proxy kernelspec: %s" % e) + import jupyter_kernel_proxy # noqa: F401 + except ImportError: + return False + + self._cleanup_stale_kernelspecs() + + spec_dir = os.path.join( + jupyter_data_dir(), "kernels", self._kernelspec_name() + ) + spec_path = os.path.join(spec_dir, "kernel.json") + spec = { + "argv": [ + _python_executable(), + "-m", "ipyida.proxy_runner", + "{connection_file}", + os.path.basename(self.connection_file), + ], + "display_name": "IDA Pro (PID %d)" % os.getpid(), + "language": "python", + "metadata": {"debugger": True}, + } + try: + os.makedirs(spec_dir, exist_ok=True) + with open(spec_path, "w") as f: + json.dump(spec, f) + except OSError as e: + print("-> Could not write ipyida kernelspec: %s" % e) return False return True @@ -167,9 +328,8 @@ def _get_running_notebook_config(self): return server_info return None - @staticmethod - def _create_proxy_session(server_info, relative_path): - """Pre-create a kernel session bound to the proxy kernel. + def _create_proxy_session(self, server_info, relative_path): + """Pre-create a kernel session bound to this IDA's proxy kernel. Needed on notebook 7 because its JupyterLab-based frontend does not honour the legacy ``?kernel_name=`` query string the way notebook 6 @@ -185,7 +345,7 @@ def _create_proxy_session(server_info, relative_path): "path": "/".join(relative_path.split(os.path.sep)), "type": "notebook", "name": "", - "kernel": {"name": "proxy"}, + "kernel": {"name": self._kernelspec_name()}, }).encode("utf-8") headers = {"Content-Type": "application/json"} if token: @@ -273,17 +433,26 @@ def open_notebook(self, line): nb = nbformat.versions[nbformat.current_nbformat].new_notebook() json.dump(nb, f) relative_path = os.path.relpath(ipynb_path, _server_root_dir(nb_server_info)) - # Update access time of the file so it's picked up by the proxy. - # jupyter-kernel-proxy will use the file with the most recent access - # time (like `jupyter console --existing`) - with open(find_connection_file(self.connection_file), "r"): pass + # Fallback ordering for jupyter_kernel_proxy: it picks the + # kernel-*.json with the newest st_atime. ``open(..., "r")`` does + # NOT update atime on NTFS volumes where LastAccessUpdate is + # disabled (the default on most Windows 10/11 boxes). Use + # ``os.utime`` which sets the timestamp directly via SetFileTime + # and isn't affected by that setting. (Primary selection happens + # via IPYIDA_KERNEL_FILE in the kernelspec env; this is only a + # safety net.) + try: + os.utime(find_connection_file(self.connection_file), None) + except OSError: + pass # On notebook 7 the JupyterLab-based frontend ignores the # ?kernel_name= query argument. Posting a session in advance attaches # the proxy kernel so the notebook page picks it up on load. self._create_proxy_session(nb_server_info, relative_path) url = nb_server_info.get("url") + \ "notebooks/" + "/".join(relative_path.split(os.path.sep)) + \ - '?kernel_name=proxy&token=' + nb_server_info.get("token") + '?kernel_name=' + self._kernelspec_name() + \ + '&token=' + nb_server_info.get("token") webbrowser.open(url) return url @@ -307,8 +476,67 @@ def notebook_log(self, line): def magic_functions(self): return [self.open_notebook, self.notebook_log] + def _shutdown_server_via_api(self, timeout=3): + """POST ``/api/shutdown`` so the server cleans up its runtime files. + + ``Popen.terminate()`` on Windows is ``TerminateProcess`` (unconditional + kill), which leaves the per-server token/secret files behind in + ``%JUPYTER_RUNTIME_DIR%`` and confuses ``list_running_servers()`` on + the next launch. Asking the server to shut itself down lets it unlink + those files. Returns True on a successful HTTP response, False if we + should fall back to ``terminate()``. + """ + if self.nb_proc is None: + return False + server_info = None + try: + for info in _list_running_servers(): + if info.get("pid") == self.nb_proc.pid: + server_info = info + break + except Exception: + return False + if server_info is None: + return False + base = server_info.get("url", "").rstrip("/") + if not base: + return False + token = server_info.get("token", "") + headers = {} + if token: + headers["Authorization"] = "token " + token + req = urllib.request.Request( + base + "/api/shutdown", data=b"", headers=headers, method="POST", + ) + try: + urllib.request.urlopen(req, timeout=timeout) + except (urllib.error.URLError, OSError): + return False + return True + def shutdown(self): if self.nb_proc: - self.nb_proc.terminate() + graceful = False + try: + graceful = self._shutdown_server_via_api(timeout=3) + except Exception: + graceful = False + if graceful: + try: + self.nb_proc.wait(timeout=5) + except subprocess.TimeoutExpired: + graceful = False + if not graceful: + try: + self.nb_proc.terminate() + except Exception: + pass if self.nb_pipe_thread: - self.nb_pipe_thread.join() + self.nb_pipe_thread.join(timeout=2) + # Drop our own per-IDA kernelspec so dead specs don't accumulate. + # (Crashed IDAs leave theirs behind; ensure_kernelspec_installed + # in the next IDA cleans those up.) + spec_dir = os.path.join( + jupyter_data_dir(), "kernels", self._kernelspec_name() + ) + shutil.rmtree(spec_dir, ignore_errors=True) diff --git a/ipyida/proxy_runner.py b/ipyida/proxy_runner.py index 25aa1b8..b5059ca 100644 --- a/ipyida/proxy_runner.py +++ b/ipyida/proxy_runner.py @@ -1,27 +1,33 @@ # -*- encoding: utf8 -*- # -# Wrapper around ``jupyter_kernel_proxy`` that patches its fallback -# ``kernel_info_reply`` so the JupyterLab/Notebook 7 debug button stays -# enabled when a real kernel is already connected to the proxy. +# Wrapper around ``jupyter_kernel_proxy`` that patches two behaviours: # -# ``jupyter_kernel_proxy.KernelProxyManager._send_proxy_kernel_info`` is a -# 3-second fallback: if the real kernel does not reply to ``kernel_info_request`` -# within that window the proxy synthesises a minimal reply that lacks the -# ``debugger`` field (and others). On notebook 7 / JupyterLab the debug -# button is enabled based on ``kernel_info_reply.debugger`` -- the -# fallback racing the real reply silently disables it. +# (a) ``KernelProxyManager._send_proxy_kernel_info`` is a 3-second fallback: +# if the real kernel does not reply to ``kernel_info_request`` within +# that window the proxy synthesises a minimal reply that lacks the +# ``debugger`` field (and others). On notebook 7 / JupyterLab the debug +# button is enabled based on ``kernel_info_reply.debugger`` -- the +# fallback racing the real reply silently disables it. # -# This wrapper suppresses the fallback when a real ``proxy_target`` is -# connected, so the front-end always sees the real kernel's reply. +# (b) Clicking "Shutdown" / "Restart" in the JupyterLab kernel menu sends a +# ``shutdown_request`` on the control (and historically shell) channel. +# If we forward it, IDA's embedded ipykernel calls ``IOLoop.stop()`` on +# the kernel's loop, which -- because qasync ties asyncio to IDA's Qt +# event loop -- tears IDA down with it. Instead we answer locally with +# ``shutdown_reply`` and drop the message: the notebook server then +# terminates this proxy subprocess only, and IDA's kernel keeps +# running. Restart looks the same to the user: the server respawns the +# proxy, which reconnects to IDA's still-live kernel. import sys import jupyter_kernel_proxy +from jupyter_kernel_proxy import KernelProxyManager, JupyterMessage -_orig_send_proxy_kernel_info = ( - jupyter_kernel_proxy.KernelProxyManager._send_proxy_kernel_info -) +# --- (a) keep the JupyterLab debug button enabled ------------------------- + +_orig_send_proxy_kernel_info = KernelProxyManager._send_proxy_kernel_info def _patched_send_proxy_kernel_info(self, request): @@ -32,14 +38,78 @@ def _patched_send_proxy_kernel_info(self, request): return _orig_send_proxy_kernel_info(self, request) -jupyter_kernel_proxy.KernelProxyManager._send_proxy_kernel_info = ( - _patched_send_proxy_kernel_info -) +KernelProxyManager._send_proxy_kernel_info = _patched_send_proxy_kernel_info + + +# --- (b) intercept browser-initiated shutdown so IDA survives ------------- + +def _make_shutdown_interceptor(channel_name): + def handler(server, target_stream, data): + msg = JupyterMessage.parse(data) + restart = (msg.content or {}).get("restart", False) + reply_stream = getattr(server.streams, channel_name) + reply_parts = msg.identities + server.make_multipart_message( + "shutdown_reply", + {"restart": restart, "status": "ok"}, + parent_header=msg.header, + ) + reply_stream.send_multipart(reply_parts) + reply_stream.flush() + # Return None to drop the request -- do NOT forward to IDA's kernel. + return None + return handler + + +_orig_init = KernelProxyManager.__init__ + + +def _patched_init(self, server): + _orig_init(self, server) + self.server.intercept_message( + "control", "shutdown_request", _make_shutdown_interceptor("control"), + ) + self.server.intercept_message( + "shell", "shutdown_request", _make_shutdown_interceptor("shell"), + ) + + +KernelProxyManager.__init__ = _patched_init + + +# --- (c) connect deterministically to IDA's kernel ------------------------ +# +# ``KernelProxyManager.connect_to_last`` picks the kernel-*.json with the +# newest ``st_atime``, which is fragile: after a few %open_notebook / +# browser-shutdown cycles, stale proxy connection files in +# JUPYTER_RUNTIME_DIR can out-rank IDA's, and the new proxy ends up +# dialing a dead ZMQ endpoint -- the cell shows "no kernel" and never +# replies. The ipyida kernelspec passes IDA's connection-file basename +# as the second positional argv argument; use it to force the +# connection here. + +_target_kernel_file = sys.argv[2] if len(sys.argv) > 2 else None + +if _target_kernel_file: + def _patched_connect_to_last(self): + self.update_running_kernels() + try: + self.connect_to(_target_kernel_file) + return + except ValueError: + pass + # Target not present (yet?) -- fall back to the original + # newest-by-atime selection so the proxy still has *something* + # to talk to. + if self.kernels: + self.connect_to(next(iter(self.kernels.keys()))) + + KernelProxyManager.connect_to_last = _patched_connect_to_last def main(): if len(sys.argv) < 2: - print("Usage: python -m ipyida.proxy_runner ") + print("Usage: python -m ipyida.proxy_runner " + " []") sys.exit(1) jupyter_kernel_proxy.start(sys.argv[1])