From 990f5a9ce45baf38cf897f227e54d5c3185bc266 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Thu, 16 Apr 2026 10:42:36 -0400 Subject: [PATCH 1/9] feat: enable mypy session for ndb --- packages/google-cloud-ndb/noxfile.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/google-cloud-ndb/noxfile.py b/packages/google-cloud-ndb/noxfile.py index b457368bb75f..7eb6de82bebd 100644 --- a/packages/google-cloud-ndb/noxfile.py +++ b/packages/google-cloud-ndb/noxfile.py @@ -453,10 +453,20 @@ def core_deps_from_source(session): @nox.session(python=DEFAULT_INTERPRETER) def mypy(session): """Run the type checker.""" - - # TODO(https://github.com/googleapis/google-cloud-python/issues/16014): - # Enable mypy once this bug is fixed. - session.skip("Temporarily skip mypy. See issue 16014") + session.install( + "mypy<1.16.0", + "types-requests", + "types-protobuf", + ) + session.install("-e", ".") + session.run( + "mypy", + "-p", + "google.cloud.ndb", + "--check-untyped-defs", + "--ignore-missing-imports", + *session.posargs, + ) @nox.session(python=DEFAULT_INTERPRETER) From d4f531079a086ea943faa86aa7a1ca74c24f6a11 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 17 Apr 2026 10:35:07 -0400 Subject: [PATCH 2/9] feat(ndb): enable mypy and resolve type errors --- .../google/cloud/ndb/__init__.py | 2 +- .../google/cloud/ndb/_cache.py | 23 ++- .../google/cloud/ndb/_datastore_api.py | 2 +- .../google/cloud/ndb/_datastore_query.py | 12 +- .../google/cloud/ndb/_eventloop.py | 11 +- .../google-cloud-ndb/google/cloud/ndb/_gql.py | 22 ++- .../google/cloud/ndb/_legacy_entity_pb.py | 5 +- .../cloud/ndb/_legacy_protocol_buffer.py | 3 + .../google/cloud/ndb/_options.py | 5 + .../google/cloud/ndb/_remote.py | 2 +- .../google/cloud/ndb/_retry.py | 6 +- .../google/cloud/ndb/_transaction.py | 4 +- .../google/cloud/ndb/context.py | 17 +- .../google/cloud/ndb/global_cache.py | 9 +- .../google-cloud-ndb/google/cloud/ndb/key.py | 7 +- .../google/cloud/ndb/model.py | 147 +++++++++++------- .../google/cloud/ndb/polymodel.py | 4 +- .../google/cloud/ndb/query.py | 31 +++- .../google/cloud/ndb/tasklets.py | 1 + packages/google-cloud-ndb/noxfile.py | 1 + 20 files changed, 204 insertions(+), 110 deletions(-) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/__init__.py b/packages/google-cloud-ndb/google/cloud/ndb/__init__.py index 3375db72e07b..3bd2c035ee6b 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/__init__.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/__init__.py @@ -23,7 +23,7 @@ from google.cloud.ndb import version -__version__ = version.__version__ +__version__: str = version.__version__ from google.cloud.ndb.client import Client from google.cloud.ndb.context import AutoBatcher diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_cache.py b/packages/google-cloud-ndb/google/cloud/ndb/_cache.py index 40be51190bef..0f49d7329384 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_cache.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_cache.py @@ -117,18 +117,18 @@ def done_callback(self, cache_call): """ exception = cache_call.exception() if exception: - for future in self.futures: + for future in self.futures: # type: ignore[attr-defined] future.set_exception(exception) else: - for future in self.futures: + for future in self.futures: # type: ignore[attr-defined] future.set_result(None) def make_call(self): """Make the actual call to the global cache. To be overridden.""" raise NotImplementedError - def future_info(self, key): + def future_info(self, key, value=None): """Generate info string for Future. To be overridden.""" raise NotImplementedError @@ -279,7 +279,7 @@ def make_call(self): """Call :method:`GlobalCache.get`.""" return _global_cache().get(self.keys) - def future_info(self, key): + def future_info(self, key, value=None): """Generate info string for Future.""" return "GlobalCache.get({})".format(key) @@ -373,7 +373,7 @@ def make_call(self): """Call :method:`GlobalCache.set`.""" return _global_cache().set(self.todo, expires=self.expires) - def future_info(self, key, value): + def future_info(self, key, value=None): """Generate info string for Future.""" return "GlobalCache.set({}, {})".format(key, value) @@ -436,7 +436,7 @@ def make_call(self): """Call :method:`GlobalCache.set`.""" return _global_cache().set_if_not_exists(self.todo, expires=self.expires) - def future_info(self, key, value): + def future_info(self, key, value=None): """Generate info string for Future.""" return "GlobalCache.set_if_not_exists({}, {})".format(key, value) @@ -482,7 +482,7 @@ def make_call(self): """Call :method:`GlobalCache.delete`.""" return _global_cache().delete(self.keys) - def future_info(self, key): + def future_info(self, key, value=None): """Generate info string for Future.""" return "GlobalCache.delete({})".format(key) @@ -513,7 +513,7 @@ def make_call(self): """Call :method:`GlobalCache.watch`.""" return _global_cache().watch(self.todo) - def future_info(self, key, value): + def future_info(self, key, value=None): """Generate info string for Future.""" return "GlobalCache.watch({}, {})".format(key, value) @@ -543,7 +543,7 @@ def make_call(self): """Call :method:`GlobalCache.unwatch`.""" return _global_cache().unwatch(self.keys) - def future_info(self, key): + def future_info(self, key, value=None): """Generate info string for Future.""" return "GlobalCache.unwatch({})".format(key) @@ -580,7 +580,7 @@ def make_call(self): """Call :method:`GlobalCache.compare_and_swap`.""" return _global_cache().compare_and_swap(self.todo, expires=self.expires) - def future_info(self, key, value): + def future_info(self, key, value=None): """Generate info string for Future.""" return "GlobalCache.compare_and_swap({}, {})".format(key, value) @@ -627,8 +627,7 @@ def global_lock_for_write(key): tasklets.Future: Eventual result will be a lock value to be used later with :func:`global_unlock`. """ - lock = "." + str(uuid.uuid4()) - lock = lock.encode("ascii") + lock = ("." + str(uuid.uuid4())).encode("ascii") utils.logging_debug(log, "lock for write: {}", lock) def new_value(old_value): diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_datastore_api.py b/packages/google-cloud-ndb/google/cloud/ndb/_datastore_api.py index bca130a78271..96150e84f971 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_datastore_api.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_datastore_api.py @@ -168,7 +168,7 @@ def lookup(key, options): if use_global_cache and not key_locked: if entity_pb is not _NOT_FOUND: expires = context._global_cache_timeout(key, options) - serialized = entity_pb._pb.SerializeToString() + serialized = entity_pb._pb.SerializeToString() # type: ignore[attr-defined] yield _cache.global_compare_and_swap( cache_key, serialized, expires=expires ) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_datastore_query.py b/packages/google-cloud-ndb/google/cloud/ndb/_datastore_query.py index 72a9f8a3f761..2f982a5faa92 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_datastore_query.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_datastore_query.py @@ -342,6 +342,8 @@ def has_next_async(self): if self._batch is None: yield self._next_batch() # First time + assert self._batch is not None + assert self._index is not None if self._index < len(self._batch): raise tasklets.Return(True) @@ -359,7 +361,7 @@ def probably_has_next(self): return ( self._batch is None # Haven't even started yet or self._has_next_batch # There's another batch to fetch - or self._index < len(self._batch) # Not done with current batch + or (self._index is not None and self._index < len(self._batch)) # Not done with current batch ) @tasklets.tasklet @@ -421,6 +423,8 @@ def next(self): self._cursor_before = None raise StopIteration + assert self._batch is not None + assert self._index is not None # Won't block next_result = self._batch[self._index] self._index += 1 @@ -446,7 +450,7 @@ def _peek(self): batch = self._batch index = self._index - if batch and index < len(batch): + if batch and index is not None and index < len(batch): return batch[index] raise KeyError(index) @@ -554,6 +558,7 @@ def next(self): if not self.has_next(): raise StopIteration() + assert self._next_result is not None # Won't block next_result = self._next_result self._next_result = None @@ -718,6 +723,7 @@ def next(self): if not self.has_next(): raise StopIteration() + assert self._next_result is not None # Won't block next_result = self._next_result self._next_result = None @@ -949,7 +955,7 @@ def _query_to_protobuf(query): filter_pb = ancestor_filter_pb elif isinstance(filter_pb, query_pb2.CompositeFilter): - filter_pb.filters._pb.add(property_filter=ancestor_filter_pb._pb) + filter_pb.filters._pb.add(property_filter=ancestor_filter_pb._pb) # type: ignore[attr-defined] else: filter_pb = query_pb2.CompositeFilter( diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_eventloop.py b/packages/google-cloud-ndb/google/cloud/ndb/_eventloop.py index 4d54865d54a2..e71dc0c12b58 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_eventloop.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_eventloop.py @@ -116,12 +116,15 @@ class EventLoop(object): """ def __init__(self): - self.current = collections.deque() - self.idlers = collections.deque() + self._init() + + def _init(self): + self.current: collections.deque = collections.deque() + self.idlers: collections.deque = collections.deque() self.inactive = 0 self.queue = [] self.rpcs = {} - self.rpc_results = queue.Queue() + self.rpc_results: queue.Queue = queue.Queue() def clear(self): """Remove all pending events without running any.""" @@ -139,7 +142,7 @@ def clear(self): utils.logging_debug(log, " queue = {}", queue) if rpcs: utils.logging_debug(log, " rpcs = {}", rpcs) - self.__init__() + self._init() current.clear() idlers.clear() queue[:] = [] diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_gql.py b/packages/google-cloud-ndb/google/cloud/ndb/_gql.py index 50e2d65de540..ec7962e2643b 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_gql.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_gql.py @@ -1,6 +1,7 @@ import datetime import re import time +from typing import Any from google.cloud.ndb import context as context_module from google.cloud.ndb import exceptions @@ -485,7 +486,7 @@ def _Literal(self): a string, integer, floating point value, boolean or None). """ - literal = None + literal: Any = None if self._next_symbol < len(self._symbols): try: @@ -770,27 +771,34 @@ def _raise_cast_error(message): def _time_function(values): + t_tuple: tuple[int, ...] if len(values) == 1: value = values[0] if isinstance(value, str): try: - time_tuple = time.strptime(value, "%H:%M:%S") + st = time.strptime(value, "%H:%M:%S") except ValueError as error: _raise_cast_error( "Error during time conversion, {}, {}".format(error, values) ) - time_tuple = time_tuple[3:] - time_tuple = time_tuple[0:3] + t_tuple = (st.tm_hour, st.tm_min, st.tm_sec) elif isinstance(value, int): - time_tuple = (value,) + t_tuple = (value,) else: _raise_cast_error("Invalid argument for time(), {}".format(value)) elif len(values) < 4: - time_tuple = tuple(values) + t_tuple = tuple(values) else: _raise_cast_error("Too many arguments for time(), {}".format(values)) try: - return datetime.time(*time_tuple) + if len(t_tuple) == 1: + return datetime.time(t_tuple[0]) + elif len(t_tuple) == 2: + return datetime.time(t_tuple[0], t_tuple[1]) + elif len(t_tuple) == 3: + return datetime.time(t_tuple[0], t_tuple[1], t_tuple[2]) + else: + _raise_cast_error("Invalid arguments for time()") except ValueError as error: _raise_cast_error("Error during time conversion, {}, {}".format(error, values)) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_legacy_entity_pb.py b/packages/google-cloud-ndb/google/cloud/ndb/_legacy_entity_pb.py index d171d2737822..54e928578f01 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_legacy_entity_pb.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_legacy_entity_pb.py @@ -389,11 +389,10 @@ class Property(ProtocolBuffer.ProtocolMessage): 24: "EMPTY_LIST", } + @classmethod def Meaning_Name(cls, x): return cls._Meaning_NAMES.get(x, "") - Meaning_Name = classmethod(Meaning_Name) - has_meaning_ = 0 meaning_ = 0 has_meaning_uri_ = 0 @@ -526,7 +525,7 @@ class Path_Element(ProtocolBuffer.ProtocolMessage): def type(self): # Force legacy byte-str to be a str. if type(self.type_) is bytes: - return self.type_.decode() + return self.type_.decode() # type: ignore[attr-defined] return self.type_ def set_type(self, x): diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_legacy_protocol_buffer.py b/packages/google-cloud-ndb/google/cloud/ndb/_legacy_protocol_buffer.py index 0b10f0b4674a..7431b288f1de 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_legacy_protocol_buffer.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_legacy_protocol_buffer.py @@ -32,6 +32,9 @@ def MergePartialFromString(self, s): d = Decoder(a, 0, len(a)) self.TryMerge(d) + def TryMerge(self, d): + raise NotImplementedError + class Decoder: NUMERIC = 0 diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_options.py b/packages/google-cloud-ndb/google/cloud/ndb/_options.py index d6caf13a20ee..92ab694b354e 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_options.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_options.py @@ -19,11 +19,16 @@ import logging from google.cloud.ndb import exceptions +from typing import Any log = logging.getLogger(__name__) class Options(object): + max_memcache_items: Any + force_writes: Any + propagation: Any + __slots__ = ( # Supported "retries", diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_remote.py b/packages/google-cloud-ndb/google/cloud/ndb/_remote.py index 193a7ba7620a..c422af249058 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_remote.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_remote.py @@ -41,7 +41,7 @@ def __init__(self, future, info): self.future = future self.info = info self.start_time = time.time() - self.elapsed_time = 0 + self.elapsed_time = 0.0 def record_time(future): self.elapsed_time = time.time() - self.start_time diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_retry.py b/packages/google-cloud-ndb/google/cloud/ndb/_retry.py index cef5f516539d..44494fffab14 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_retry.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_retry.py @@ -33,7 +33,7 @@ def wraps_safely(obj, attr_names=functools.WRAPPER_ASSIGNMENTS): are not copied to the wrappers and thus cause attribute errors. This wrapper prevents that problem.""" return functools.wraps( - obj, assigned=(name for name in attr_names if hasattr(obj, name)) + obj, assigned=tuple(name for name in attr_names if hasattr(obj, name)) ) @@ -84,7 +84,7 @@ def retry_wrapper(*args, **kwargs): error = e except BaseException as e: # `e` is removed from locals at end of block - error = e # See: https://goo.gl/5J8BMK + error = e # type: ignore[assignment] # See: https://goo.gl/5J8BMK if not is_transient_error(error): # If we are in an inner retry block, use special nested @@ -107,7 +107,7 @@ def retry_wrapper(*args, **kwargs): # Unknown errors really want to show up as None, so manually set the error. if isinstance(error, core_exceptions.Unknown): - error = "google.api_core.exceptions.Unknown" + error = "google.api_core.exceptions.Unknown" # type: ignore[assignment] raise core_exceptions.RetryError( "Maximum number of {} retries exceeded while calling {}".format( diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_transaction.py b/packages/google-cloud-ndb/google/cloud/ndb/_transaction.py index f07d752ca92b..637d4b200d60 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_transaction.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_transaction.py @@ -257,8 +257,8 @@ def _transaction_async(context, callback, read_only=False): transaction_id = yield _datastore_api.begin_transaction(read_only, retries=0) utils.logging_debug(log, "Transaction Id: {}", transaction_id) - on_commit_callbacks = [] - transaction_complete_callbacks = [] + on_commit_callbacks: list = [] + transaction_complete_callbacks: list = [] tx_context = context.new( transaction=transaction_id, on_commit_callbacks=on_commit_callbacks, diff --git a/packages/google-cloud-ndb/google/cloud/ndb/context.py b/packages/google-cloud-ndb/google/cloud/ndb/context.py index d8c47f523449..9068e5a511b9 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/context.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/context.py @@ -22,6 +22,7 @@ import os import threading import uuid +from typing import Any, cast from google.cloud.ndb import _eventloop from google.cloud.ndb import exceptions @@ -254,6 +255,11 @@ class _Context(_ContextTuple): client (client.Client): The NDB client for this context. """ + cache_policy: Any + global_cache_policy: Any + global_cache_timeout_policy: Any + datastore_policy: Any + def __new__( cls, client, @@ -313,11 +319,12 @@ def __new__( legacy_data=legacy_data, ) - context.set_cache_policy(cache_policy) - context.set_global_cache_policy(global_cache_policy) - context.set_global_cache_timeout_policy(global_cache_timeout_policy) - context.set_datastore_policy(datastore_policy) - context.set_retry_state(retry) + ctx = cast(Any, context) + ctx.set_cache_policy(cache_policy) + ctx.set_global_cache_policy(global_cache_policy) + ctx.set_global_cache_timeout_policy(global_cache_timeout_policy) + ctx.set_datastore_policy(datastore_policy) + ctx.set_retry_state(retry) return context diff --git a/packages/google-cloud-ndb/google/cloud/ndb/global_cache.py b/packages/google-cloud-ndb/google/cloud/ndb/global_cache.py index 4e3c6b7c6d3f..74202c7c13d5 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/global_cache.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/global_cache.py @@ -26,6 +26,7 @@ import pymemcache import redis as redis_module +from typing import Any # Python 2.7 doesn't have ConnectionError. In Python 3, ConnectionError is subclass of # OSError, which Python 2.7 does have. @@ -71,7 +72,7 @@ class GlobalCache(object): __metaclass__ = abc.ABCMeta - transient_errors = () + transient_errors: tuple[type, ...] = () """Exceptions that should be treated as transient errors in non-strict modes. Instances of these exceptions, if raised, will be logged as warnings but will not @@ -202,7 +203,7 @@ class _InProcessGlobalCache(GlobalCache): reference implementation. Thread safety is potentially a little sketchy. """ - cache = {} + cache: dict[Any, Any] = {} """Dict: The cache. Relies on atomicity of ``__setitem__`` for thread safety. See: @@ -521,9 +522,9 @@ def _parse_host_string(host_string): @staticmethod def _key(key): - encoded = base64.b64encode(key) + encoded = base64.b64encode(key).decode("ascii") if len(encoded) > 250: - encoded = hashlib.sha1(encoded).hexdigest() + encoded = hashlib.sha1(encoded.encode("ascii")).hexdigest() return encoded @classmethod diff --git a/packages/google-cloud-ndb/google/cloud/ndb/key.py b/packages/google-cloud-ndb/google/cloud/ndb/key.py index b168e55a190e..65e95ba61352 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/key.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/key.py @@ -88,6 +88,7 @@ """ +from typing import Optional import base64 import functools @@ -1315,7 +1316,7 @@ def _parse_from_ref( urlsafe=None, app=None, namespace=None, - database: str = None, + database: Optional[str] = None, **kwargs ): """Construct a key from a Reference. @@ -1523,7 +1524,7 @@ def _clean_flat_path(flat): # Make sure the ``kind`` is either a string or a Model. kind = flat[i] if isinstance(kind, type): - kind = kind._get_kind() + kind = kind._get_kind() # type: ignore[attr-defined] flat[i] = kind if not isinstance(kind, str): raise TypeError( @@ -1607,7 +1608,7 @@ def _to_legacy_path(dict_path): element_kwargs["id"] = _verify_path_value(part["id"], False) elif "name" in part: element_kwargs["name"] = _verify_path_value(part["name"], True) - element = _app_engine_key_pb2.Path.Element(**element_kwargs) + element = _app_engine_key_pb2.Path.Element(**element_kwargs) # type: ignore[attr-defined] elements.append(element) return _app_engine_key_pb2.Path(element=elements) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/model.py b/packages/google-cloud-ndb/google/cloud/ndb/model.py index c4d3cdb66ed4..d38107d41520 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/model.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/model.py @@ -251,6 +251,7 @@ class Person(Model): """ +from typing import Any, Optional import copy import datetime import functools @@ -401,6 +402,9 @@ def __ne__(self, other): class IndexProperty(_NotEqualMixin): """Immutable object representing a single property in an index.""" + _name: str + _direction: str + @utils.positional(1) def __new__(cls, name, direction): instance = super(IndexProperty, cls).__new__(cls) @@ -437,6 +441,10 @@ def __hash__(self): class Index(_NotEqualMixin): """Immutable object representing an index.""" + _kind: str + _properties: list + _ancestor: bool + @utils.positional(1) def __new__(cls, kind, properties, ancestor): instance = super(Index, cls).__new__(cls) @@ -484,6 +492,10 @@ def __hash__(self): class IndexState(_NotEqualMixin): """Immutable object representing an index and its state.""" + _definition: "Index" + _state: str + _id: int + @utils.positional(1) def __new__(cls, definition, state, id): instance = super(IndexState, cls).__new__(cls) @@ -599,6 +611,7 @@ def new_entity(key): subvalue = value value = structprop._get_base_value(entity) if value in (None, []): # empty list for repeated props + assert structprop._model_class is not None kind = structprop._model_class._get_kind() key = key_module.Key(kind, None) if structprop._repeated: @@ -629,6 +642,7 @@ def new_entity(key): # the other entries in the value list while len(subvalue) > len(value): # Need to make some more subentities + assert structprop._model_class is not None expando_kind = structprop._model_class._get_kind() expando_key = key_module.Key(expando_kind, None) value.append(new_entity(expando_key._key)) @@ -747,7 +761,7 @@ def _entity_to_ds_entity(entity, set_key=True): Raises: ndb.exceptions.BadValueError: If entity has uninitialized properties. """ - data = {"_exclude_from_indexes": []} + data: dict[str, Any] = {"_exclude_from_indexes": []} uninitialized = [] for prop in _properties_of(entity): @@ -1017,7 +1031,7 @@ def _from_base_type(self, value): _verbose_name = None _write_empty_list = False # Non-public class attributes. - _FIND_METHODS_CACHE = {} + _FIND_METHODS_CACHE: dict[Any, Any] = {} @utils.positional(2) def __init__( @@ -1151,8 +1165,8 @@ def _constructor_info(self): """ # inspect.signature not available in Python 2.7, so we use positional # decorator combined with argspec instead. - argspec = getattr(self.__init__, "_argspec", _getfullargspec(self.__init__)) - positional = getattr(self.__init__, "_positional_args", 1) + argspec = getattr(self.__init__, "_argspec", _getfullargspec(self.__init__)) # type: ignore[misc] + positional = getattr(self.__init__, "_positional_args", 1) # type: ignore[misc] for index, name in enumerate(argspec.args): if name == "self": continue @@ -3619,6 +3633,7 @@ class SimpleModel(ndb.Model): _kind = None + @staticmethod def _handle_positional(wrapped): @functools.wraps(wrapped) def wrapper(self, *args, **kwargs): @@ -3640,7 +3655,7 @@ def wrapper(self, *args, **kwargs): return wrapped(self, **kwargs) - wrapper._wrapped = wrapped + wrapper._wrapped = wrapped # type: ignore[attr-defined] return wrapper @utils.positional(3) @@ -4146,6 +4161,7 @@ def _get_for_dict(self, entity): def __getattr__(self, attrname): """Dynamically get a subproperty.""" + assert self._model_class is not None # Optimistically try to use the dict key. prop = self._model_class._properties.get(attrname) @@ -4164,6 +4180,7 @@ def __getattr__(self, attrname): ) prop_copy = copy.copy(prop) + assert self._name is not None prop_copy._name = self._name + "." + prop_copy._name # Cache the outcome, so subsequent requests for the same attribute @@ -4194,6 +4211,7 @@ def _comparison(self, op, value): value = self._do_validate(value) filters = [] match_keys = [] + assert self._model_class is not None for prop_name, prop in self._model_class._properties.items(): subvalue = prop._get_value(value) if prop._repeated: @@ -4226,7 +4244,7 @@ def _comparison(self, op, value): return ConjunctionNode(*filters) - def _IN(self, value): + def _IN(self, value, server_op=None): if not isinstance(value, (list, tuple, set, frozenset)): raise exceptions.BadArgumentError( "Expected list, tuple or set, got %r" % (value,) @@ -4246,6 +4264,7 @@ def _IN(self, value): IN = _IN def _validate(self, value): + assert self._model_class is not None if isinstance(value, dict): # A dict is assumed to be the result of a _to_dict() call. return self._model_class(**value) @@ -4306,6 +4325,7 @@ def _check_property(self, rest=None, require_indexed=True): raise InvalidPropertyError( "Structured property %s requires a subproperty" % self._name ) + assert self._model_class is not None self._model_class._check_properties([rest], require_indexed=require_indexed) def _to_base_type(self, value): @@ -4320,6 +4340,7 @@ def _to_base_type(self, value): Raises: TypeError: If ``value`` is not the correct ``Model`` type. """ + assert self._model_class is not None if not isinstance(value, self._model_class): raise TypeError( "Cannot convert to protocol buffer. Expected {} value; " @@ -4454,6 +4475,7 @@ def _validate(self, value): Raises: exceptions.BadValueError: If ``value`` is not a given class. """ + assert self._model_class is not None if isinstance(value, dict): # A dict is assumed to be the result of a _to_dict() call. return self._model_class(**value) @@ -4482,6 +4504,7 @@ def _to_base_type(self, value): Raises: TypeError: If ``value`` is not the correct ``Model`` type. """ + assert self._model_class is not None if not isinstance(value, self._model_class): raise TypeError( "Cannot convert to bytes expected {} value; " @@ -4499,6 +4522,7 @@ def _from_base_type(self, value): Returns: The converted value with given class. """ + assert self._model_class is not None if isinstance(value, bytes): pb = entity_pb2.Entity() pb._pb.MergeFromString(value) @@ -4507,9 +4531,9 @@ def _from_base_type(self, value): # No properties. Maybe dealing with legacy pb format. from google.cloud.ndb._legacy_entity_pb import EntityProto - pb = EntityProto() - pb.MergePartialFromString(value) - entity_value.update(pb.entity_props()) + legacy_pb = EntityProto() + legacy_pb.MergePartialFromString(value) + entity_value.update(legacy_pb.entity_props()) value = entity_value if not self._keep_keys and value.key: value.key = None @@ -4560,8 +4584,9 @@ def _to_datastore(self, entity, data, prefix="", repeated=False): ds_entity = _entity_to_ds_entity(value, set_key=self._keep_keys) legacy_values.append(ds_entity) if not self._repeated: - legacy_values = legacy_values[0] - data[self._name] = legacy_values + data[self._name] = legacy_values[0] + else: + data[self._name] = legacy_values return keys @@ -4674,6 +4699,7 @@ def _get_value(self, entity): # UnprojectedPropertyError which will just bubble up. if entity._projection and self._name in entity._projection: return super(ComputedProperty, self)._get_value(entity) + assert self._func is not None value = self._func(entity) self._store_value(entity, value) return value @@ -4700,11 +4726,11 @@ class itself. def __init__(cls, name, bases, classdict): super(MetaModel, cls).__init__(name, bases, classdict) - cls._fix_up_properties() + cls._fix_up_properties() # type: ignore[attr-defined] def __repr__(cls): props = [] - for _, prop in sorted(cls._properties.items()): + for _, prop in sorted(cls._properties.items()): # type: ignore[attr-defined] props.append("{}={!r}".format(prop._code_name, prop)) return "{}<{}>".format(cls.__name__, ", ".join(props)) @@ -4886,13 +4912,13 @@ class MyModel(ndb.Model): """ # Class variables updated by _fix_up_properties() - _properties = None + _properties: Optional[dict] = None _has_repeated = False - _kind_map = {} # Dict mapping {kind: Model subclass} + _kind_map: dict = {} # Dict mapping {kind: Model subclass} # Defaults for instance variables. _entity_key = None - _values = None + _values: Optional[dict] = None _projection = () # Tuple of names of projected properties. # Hardcoded pseudo-property for the key. @@ -4913,13 +4939,13 @@ class MyModel(ndb.Model): def __setstate__(self, state): if type(state) is dict: # this is not a legacy pb. set __dict__ - self.__init__() + self.__init__() # type: ignore[misc] self.__dict__.update(state) else: # this is a legacy pickled object. We need to deserialize. pb = _legacy_entity_pb.EntityProto() pb.MergePartialFromString(state) - self.__init__() + self.__init__() # type: ignore[misc] self.__class__._from_pb(pb, set_key=False, ent=self) def __init__(_self, **kwargs): @@ -4987,7 +5013,7 @@ def _get_property_for(self, p, indexed=True, depth=0): # since the latter doesn't match the current schema.) return None next = parts[depth] - prop = self._properties.get(next) + prop = self._properties.get(next) if self._properties is not None else None if prop is None: prop = self._fake_property(p, next, indexed) return prop @@ -5000,13 +5026,15 @@ def _clone_properties(self): """ cls = type(self) if self._properties is cls._properties: - self._properties = dict(cls._properties) + self._properties = dict(cls._properties) if cls._properties is not None else {} def _fake_property(self, p, next, indexed=True): """Internal helper to create a fake Property. Ported from legacy datastore""" # A custom 'meaning' for compressed properties. _MEANING_URI_COMPRESSED = "ZLIB" self._clone_properties() + assert self._properties is not None + prop: Property if p.name() != next.encode("utf-8") and not p.name().endswith( b"." + next.encode("utf-8") ): @@ -5114,23 +5142,24 @@ def __repr__(self): """Return an unambiguous string representation of an entity.""" by_args = [] has_key_property = False - for prop in self._properties.values(): - if prop._code_name == "key": - has_key_property = True + if self._properties is not None: + for prop in self._properties.values(): + if prop._code_name == "key": + has_key_property = True - if not prop._has_value(self): - continue + if not prop._has_value(self): + continue - value = prop._retrieve_value(self) - if value is None: - arg_repr = "None" - elif prop._repeated: - arg_reprs = [prop._value_to_repr(sub_value) for sub_value in value] - arg_repr = "[{}]".format(", ".join(arg_reprs)) - else: - arg_repr = prop._value_to_repr(value) + value = prop._retrieve_value(self) + if value is None: + arg_repr = "None" + elif prop._repeated: + arg_reprs = [prop._value_to_repr(sub_value) for sub_value in value] + arg_repr = "[{}]".format(", ".join(arg_reprs)) + else: + arg_repr = prop._value_to_repr(value) - by_args.append("{}={}".format(prop._code_name, arg_repr)) + by_args.append("{}={}".format(prop._code_name, arg_repr)) by_args.sort() @@ -5210,11 +5239,14 @@ def _equivalent(self, other): if set(self._projection) != set(other._projection): return False - if len(self._properties) != len(other._properties): + self_props = self._properties if self._properties is not None else {} + other_props = other._properties if other._properties is not None else {} + + if len(self_props) != len(other_props): return False # Can only happen for Expandos. - prop_names = set(self._properties.keys()) - other_prop_names = set(other._properties.keys()) + prop_names = set(self_props.keys()) + other_prop_names = set(other_props.keys()) if prop_names != other_prop_names: return False # Again, only possible for Expandos @@ -5223,8 +5255,8 @@ def _equivalent(self, other): prop_names = set(self._projection) for name in prop_names: - value = self._properties[name]._get_value(self) - if value != other._properties[name]._get_value(other): + value = self_props[name]._get_value(self) + if value != other_props[name]._get_value(other): return False return True @@ -5282,20 +5314,22 @@ def _set_projection(self, projection): # Handle projections for structured properties by recursively setting # projections on sub-entities. - by_prefix = {} + by_prefix: dict[str, list] = {} for name in projection: if "." in name: head, tail = name.split(".", 1) by_prefix.setdefault(head, []).append(tail) - for name, projection in by_prefix.items(): - prop = self._properties.get(name) - value = prop._get_user_value(self) - if prop._repeated: - for entity in value: - entity._set_projection(projection) - else: - value._set_projection(projection) + if self._properties is not None: + for name, projection in by_prefix.items(): + prop = self._properties.get(name) + if prop is not None: + value = prop._get_user_value(self) + if prop._repeated: + for entity in value: + entity._set_projection(projection) + else: + value._set_projection(projection) @classmethod def _check_properties(cls, property_names, require_indexed=True): @@ -5319,6 +5353,7 @@ def _check_properties(cls, property_names, require_indexed=True): name, rest = name.split(".", 1) else: rest = None + assert cls._properties is not None prop = cls._properties.get(name) if prop is None: raise InvalidPropertyError("Unknown property {}".format(name)) @@ -5366,6 +5401,7 @@ def _fix_up_properties(cls): if isinstance(attr, Property): if attr._repeated or ( isinstance(attr, StructuredProperty) + and attr._model_class is not None and attr._model_class._has_repeated ): cls._has_repeated = True @@ -5745,13 +5781,13 @@ def allocate_ids(): kind = cls._get_kind() keys = [key_module.Key(kind, None, parent=parent)._key for _ in range(size)] key_pbs = yield _datastore_api.allocate(keys, _options) - keys = tuple( + allocated_keys = tuple( ( key_module.Key._from_ds_key(helpers.key_from_protobuf(key_pb)) for key_pb in key_pbs ) ) - raise tasklets.Return(keys) + raise tasklets.Return(allocated_keys) future = allocate_ids() future.add_done_callback( @@ -5872,7 +5908,7 @@ def _get_by_id_async( max_memcache_items=None, force_writes=None, _options=None, - database: str = None, + database: Optional[str] = None, ): """Get an instance of Model class by ID. @@ -6161,6 +6197,7 @@ def _to_dict(self, include=None, exclude=None): to exclude. Default is to not exclude any names. """ values = {} + assert self._properties is not None for prop in self._properties.values(): name = prop._code_name if include is not None and name not in include: @@ -6182,6 +6219,7 @@ def _to_dict(self, include=None, exclude=None): def _code_name_from_stored_name(cls, name): """Return the code name from a property when it's different from the stored name. Used in deserialization from datastore.""" + assert cls._properties is not None if name in cls._properties: return cls._properties[name]._code_name @@ -6283,12 +6321,14 @@ def _set_attributes(self, kwds): setattr(self, name, value) def __getattr__(self, name): + assert self._properties is not None prop = self._properties.get(name) if prop is None: return super(Expando, self).__getattribute__(name) return prop._get_value(self) def __setattr__(self, name, value): + assert self._properties is not None if ( name.startswith("_") or isinstance(getattr(self.__class__, name, None), (Property, property)) @@ -6306,6 +6346,7 @@ def __setattr__(self, name, value): self._clone_properties() + prop: Property if isinstance(value, Model): prop = StructuredProperty(Model, name) elif isinstance(value, dict): @@ -6322,6 +6363,7 @@ def __setattr__(self, name, value): prop._set_value(self, value) def __delattr__(self, name): + assert self._properties is not None if name.startswith("_") or isinstance( getattr(self.__class__, name, None), (Property, property) ): @@ -6332,7 +6374,8 @@ def __delattr__(self, name): "Model properties must be Property instances; not %r" % prop ) prop._delete_value(self) - if name in super(Expando, self)._properties: + base_props = super(Expando, self)._properties + if base_props is not None and name in base_props: raise RuntimeError( "Property %s still in the list of properties for the " "base class." % name diff --git a/packages/google-cloud-ndb/google/cloud/ndb/polymodel.py b/packages/google-cloud-ndb/google/cloud/ndb/polymodel.py index da192568b2ec..f69a6b6271af 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/polymodel.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/polymodel.py @@ -184,7 +184,7 @@ class PolyModel(model.Model): class_ = _ClassKeyProperty() - _class_map = {} # Map class key -> suitable subclass. + _class_map: dict[tuple, type] = {} # Map class key -> suitable subclass. @classmethod def _update_kind_map(cls): @@ -224,7 +224,7 @@ def _get_kind(cls): # super(PolyModel, cls)._get_kind(). Second, we can't just call # Model._get_kind() because that always returns 'Model'. Hence # the '__func__' hack. - return model.Model._get_kind.__func__(cls) + return model.Model._get_kind.__func__(cls) # type: ignore[attr-defined] else: return bases[0]._class_name() diff --git a/packages/google-cloud-ndb/google/cloud/ndb/query.py b/packages/google-cloud-ndb/google/cloud/ndb/query.py index 76731ede2337..5a159b870a74 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/query.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/query.py @@ -137,6 +137,7 @@ def ranked(cls, rank): print(emp.name, emp.age) """ +from typing import Any, Optional import functools import logging @@ -288,6 +289,9 @@ def __ne__(self, other): eq = not eq return eq + def resolve(self, bindings, used): + raise NotImplementedError + class Parameter(ParameterizedThing): """Represents a bound variable in a GQL query. @@ -513,6 +517,10 @@ class ParameterNode(Node): :class:`.ParameterizedFunction`. """ + _prop: Any + _op: str + _param: Any + def __new__(cls, prop, op, param): # Avoid circular import in Python 2.7 from google.cloud.ndb import model @@ -722,6 +730,8 @@ class PostFilterNode(Node): the given filter. """ + predicate: Any + def __new__(cls, predicate): instance = super(PostFilterNode, cls).__new__(cls) instance.predicate = predicate @@ -797,7 +807,7 @@ def __init__(self, name, combine_or): self.combine_or = combine_or if combine_or: # For ``OR()`` the parts are just nodes. - self.or_parts = [] + self.or_parts: list = [] else: # For ``AND()`` the parts are "segments", i.e. node lists. self.or_parts = [[]] @@ -883,6 +893,8 @@ class ConjunctionNode(Node): expression. """ + _nodes: list + def __new__(cls, *nodes): if not nodes: raise TypeError("ConjunctionNode() requires at least one node.") @@ -1037,6 +1049,7 @@ class DisjunctionNode(Node): """ _multiquery = True + _nodes: list def __new__(cls, *nodes): if not nodes: @@ -1204,6 +1217,10 @@ def wrapper(self, *args, **kwargs): class QueryOptions(_options.ReadOptions): + namespace: Optional[str] + project: Optional[str] + ancestor: Any + __slots__ = ( # Query options "kind", @@ -1535,13 +1552,13 @@ def filter(self, *filters): ) new_filters.append(filter) if len(new_filters) == 1: - new_filters = new_filters[0] + final_filters = new_filters[0] else: - new_filters = ConjunctionNode(*new_filters) + final_filters = ConjunctionNode(*new_filters) return self.__class__( kind=self.kind, ancestor=self.ancestor, - filters=new_filters, + filters=final_filters, order_by=self.order_by, project=self.project, namespace=self.namespace, @@ -1603,7 +1620,7 @@ def __contains__(self, key): return True bindings = MockBindings() - used = {} + used: dict[Any, Any] = {} ancestor = self.ancestor if isinstance(ancestor, ParameterizedThing): ancestor = ancestor.resolve(bindings, used) @@ -1633,10 +1650,10 @@ def bind(self, *positional, **keyword): google.cloud.ndb.exceptions.BadArgumentError: If one of the positional parameters is not used in the query. """ - bindings = dict(keyword) + bindings: dict[Any, Any] = dict(keyword) for i, arg in enumerate(positional): bindings[i + 1] = arg - used = {} + used: dict[Any, Any] = {} ancestor = self.ancestor if isinstance(ancestor, ParameterizedThing): ancestor = ancestor.resolve(bindings, used) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/tasklets.py b/packages/google-cloud-ndb/google/cloud/ndb/tasklets.py index 960c48d34b95..c62ae97584f4 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/tasklets.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/tasklets.py @@ -496,6 +496,7 @@ def tasklet_wrapper(*args, **kwargs): # then we'll extract the result from the StopIteration exception. returned = _get_return_value(stop) + future: Future if isinstance(returned, types.GeneratorType): # We have a tasklet, start it future = _TaskletFuture(returned, context, info=wrapped.__name__) diff --git a/packages/google-cloud-ndb/noxfile.py b/packages/google-cloud-ndb/noxfile.py index 7eb6de82bebd..20ffac1aa36d 100644 --- a/packages/google-cloud-ndb/noxfile.py +++ b/packages/google-cloud-ndb/noxfile.py @@ -457,6 +457,7 @@ def mypy(session): "mypy<1.16.0", "types-requests", "types-protobuf", + "types-pytz", ) session.install("-e", ".") session.run( From 9934515625cce149c28da4fbf2f88b4f38ab306f Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 17 Apr 2026 10:43:27 -0400 Subject: [PATCH 3/9] fix(ndb): fix python 3.9 compatibility and __all__ test failures --- .../google/cloud/ndb/_datastore_query.py | 4 +++- packages/google-cloud-ndb/google/cloud/ndb/key.py | 4 ++-- .../google-cloud-ndb/google/cloud/ndb/model.py | 15 ++++++++------- .../google-cloud-ndb/google/cloud/ndb/query.py | 14 +++++++------- packages/google-cloud-ndb/tests/unit/utils.py | 6 +++++- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_datastore_query.py b/packages/google-cloud-ndb/google/cloud/ndb/_datastore_query.py index 2f982a5faa92..b7111e05097d 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_datastore_query.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_datastore_query.py @@ -361,7 +361,9 @@ def probably_has_next(self): return ( self._batch is None # Haven't even started yet or self._has_next_batch # There's another batch to fetch - or (self._index is not None and self._index < len(self._batch)) # Not done with current batch + or ( + self._index is not None and self._index < len(self._batch) + ) # Not done with current batch ) @tasklets.tasklet diff --git a/packages/google-cloud-ndb/google/cloud/ndb/key.py b/packages/google-cloud-ndb/google/cloud/ndb/key.py index 65e95ba61352..c3043ff7e135 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/key.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/key.py @@ -88,7 +88,7 @@ """ -from typing import Optional +import typing import base64 import functools @@ -1316,7 +1316,7 @@ def _parse_from_ref( urlsafe=None, app=None, namespace=None, - database: Optional[str] = None, + database: typing.Optional[str] = None, **kwargs ): """Construct a key from a Reference. diff --git a/packages/google-cloud-ndb/google/cloud/ndb/model.py b/packages/google-cloud-ndb/google/cloud/ndb/model.py index d38107d41520..7fffb80461a2 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/model.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/model.py @@ -251,7 +251,7 @@ class Person(Model): """ -from typing import Any, Optional +import typing import copy import datetime import functools @@ -1031,7 +1031,7 @@ def _from_base_type(self, value): _verbose_name = None _write_empty_list = False # Non-public class attributes. - _FIND_METHODS_CACHE: dict[Any, Any] = {} + _FIND_METHODS_CACHE: dict[typing.Any, typing.Any] = {} @utils.positional(2) def __init__( @@ -3633,7 +3633,6 @@ class SimpleModel(ndb.Model): _kind = None - @staticmethod def _handle_positional(wrapped): @functools.wraps(wrapped) def wrapper(self, *args, **kwargs): @@ -4912,13 +4911,13 @@ class MyModel(ndb.Model): """ # Class variables updated by _fix_up_properties() - _properties: Optional[dict] = None + _properties: typing.Optional[dict] = None _has_repeated = False _kind_map: dict = {} # Dict mapping {kind: Model subclass} # Defaults for instance variables. _entity_key = None - _values: Optional[dict] = None + _values: typing.Optional[dict] = None _projection = () # Tuple of names of projected properties. # Hardcoded pseudo-property for the key. @@ -5026,7 +5025,9 @@ def _clone_properties(self): """ cls = type(self) if self._properties is cls._properties: - self._properties = dict(cls._properties) if cls._properties is not None else {} + self._properties = ( + dict(cls._properties) if cls._properties is not None else {} + ) def _fake_property(self, p, next, indexed=True): """Internal helper to create a fake Property. Ported from legacy datastore""" @@ -5908,7 +5909,7 @@ def _get_by_id_async( max_memcache_items=None, force_writes=None, _options=None, - database: Optional[str] = None, + database: typing.Optional[str] = None, ): """Get an instance of Model class by ID. diff --git a/packages/google-cloud-ndb/google/cloud/ndb/query.py b/packages/google-cloud-ndb/google/cloud/ndb/query.py index 5a159b870a74..7ca7dc18046e 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/query.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/query.py @@ -137,7 +137,7 @@ def ranked(cls, rank): print(emp.name, emp.age) """ -from typing import Any, Optional +import typing import functools import logging @@ -517,9 +517,9 @@ class ParameterNode(Node): :class:`.ParameterizedFunction`. """ - _prop: Any + _prop: typing.Any _op: str - _param: Any + _param: typing.Any def __new__(cls, prop, op, param): # Avoid circular import in Python 2.7 @@ -730,7 +730,7 @@ class PostFilterNode(Node): the given filter. """ - predicate: Any + predicate: typing.Any def __new__(cls, predicate): instance = super(PostFilterNode, cls).__new__(cls) @@ -1217,9 +1217,9 @@ def wrapper(self, *args, **kwargs): class QueryOptions(_options.ReadOptions): - namespace: Optional[str] - project: Optional[str] - ancestor: Any + namespace: typing.Optional[str] + project: typing.Optional[str] + ancestor: typing.Any __slots__ = ( # Query options diff --git a/packages/google-cloud-ndb/tests/unit/utils.py b/packages/google-cloud-ndb/tests/unit/utils.py index e20d4710ec99..8a57016100b0 100644 --- a/packages/google-cloud-ndb/tests/unit/utils.py +++ b/packages/google-cloud-ndb/tests/unit/utils.py @@ -25,7 +25,11 @@ def verify___all__(module_obj): if not isinstance(value, types.ModuleType): expected.append(name) expected.sort(key=str.lower) - assert sorted(module_obj.__all__, key=str.lower) == expected + actual = sorted(module_obj.__all__, key=str.lower) + if actual != expected: + print("In __all__ but not expected:", set(actual) - set(expected)) + print("In expected but not __all__:", set(expected) - set(actual)) + assert actual == expected def future_result(result): From ec2312c8f3347e3657d577b1e6fe55dbd6d386fa Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 17 Apr 2026 10:57:44 -0400 Subject: [PATCH 4/9] fix(ndb): fix mypy errors and undefined Any name --- .../google/cloud/ndb/model.py | 53 ++++++++++--------- .../google/cloud/ndb/query.py | 6 +-- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/model.py b/packages/google-cloud-ndb/google/cloud/ndb/model.py index 7fffb80461a2..ac2fc866e09b 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/model.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/model.py @@ -761,7 +761,7 @@ def _entity_to_ds_entity(entity, set_key=True): Raises: ndb.exceptions.BadValueError: If entity has uninitialized properties. """ - data: dict[str, Any] = {"_exclude_from_indexes": []} + data: dict[str, typing.Any] = {"_exclude_from_indexes": []} uninitialized = [] for prop in _properties_of(entity): @@ -3577,6 +3577,31 @@ def _to_datastore(self, entity, data, prefix="", repeated=False): ) +def _handle_key_property_positional(wrapped): + @functools.wraps(wrapped) + def wrapper(self, *args, **kwargs): + for arg in args: + if isinstance(arg, str): + if "name" in kwargs: + raise TypeError("You can only specify name once") + + kwargs["name"] = arg + + elif isinstance(arg, type): + if "kind" in kwargs: + raise TypeError("You can only specify kind once") + + kwargs["kind"] = arg + + elif arg is not None: + raise TypeError("Unexpected positional argument: {!r}".format(arg)) + + return wrapped(self, **kwargs) + + wrapper._wrapped = wrapped # type: ignore[attr-defined] + return wrapper + + class KeyProperty(Property): """A property that contains :class:`~google.cloud.ndb.key.Key` values. @@ -3633,32 +3658,8 @@ class SimpleModel(ndb.Model): _kind = None - def _handle_positional(wrapped): - @functools.wraps(wrapped) - def wrapper(self, *args, **kwargs): - for arg in args: - if isinstance(arg, str): - if "name" in kwargs: - raise TypeError("You can only specify name once") - - kwargs["name"] = arg - - elif isinstance(arg, type): - if "kind" in kwargs: - raise TypeError("You can only specify kind once") - - kwargs["kind"] = arg - - elif arg is not None: - raise TypeError("Unexpected positional argument: {!r}".format(arg)) - - return wrapped(self, **kwargs) - - wrapper._wrapped = wrapped # type: ignore[attr-defined] - return wrapper - @utils.positional(3) - @_handle_positional + @_handle_key_property_positional def __init__( self, name=None, diff --git a/packages/google-cloud-ndb/google/cloud/ndb/query.py b/packages/google-cloud-ndb/google/cloud/ndb/query.py index 7ca7dc18046e..ab4f11dcdcef 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/query.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/query.py @@ -1620,7 +1620,7 @@ def __contains__(self, key): return True bindings = MockBindings() - used: dict[Any, Any] = {} + used: dict[typing.Any, typing.Any] = {} ancestor = self.ancestor if isinstance(ancestor, ParameterizedThing): ancestor = ancestor.resolve(bindings, used) @@ -1650,10 +1650,10 @@ def bind(self, *positional, **keyword): google.cloud.ndb.exceptions.BadArgumentError: If one of the positional parameters is not used in the query. """ - bindings: dict[Any, Any] = dict(keyword) + bindings: dict[typing.Any, typing.Any] = dict(keyword) for i, arg in enumerate(positional): bindings[i + 1] = arg - used: dict[Any, Any] = {} + used: dict[typing.Any, typing.Any] = {} ancestor = self.ancestor if isinstance(ancestor, ParameterizedThing): ancestor = ancestor.resolve(bindings, used) From a316b6faf0322235fe40278f544ad248d4300051 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 17 Apr 2026 12:02:25 -0400 Subject: [PATCH 5/9] fix(ndb): replace asserts in production code with explicit checks --- .../google/cloud/ndb/_datastore_query.py | 18 ++++-- .../google-cloud-ndb/google/cloud/ndb/_gql.py | 3 +- .../google/cloud/ndb/model.py | 63 ++++++++++++------- 3 files changed, 56 insertions(+), 28 deletions(-) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_datastore_query.py b/packages/google-cloud-ndb/google/cloud/ndb/_datastore_query.py index b7111e05097d..8da0238bfdc4 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_datastore_query.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_datastore_query.py @@ -342,8 +342,10 @@ def has_next_async(self): if self._batch is None: yield self._next_batch() # First time - assert self._batch is not None - assert self._index is not None + if self._batch is None: + raise TypeError("self._batch cannot be None") + if self._index is None: + raise TypeError("self._index cannot be None") if self._index < len(self._batch): raise tasklets.Return(True) @@ -425,8 +427,10 @@ def next(self): self._cursor_before = None raise StopIteration - assert self._batch is not None - assert self._index is not None + if self._batch is None: + raise TypeError("self._batch cannot be None") + if self._index is None: + raise TypeError("self._index cannot be None") # Won't block next_result = self._batch[self._index] self._index += 1 @@ -560,7 +564,8 @@ def next(self): if not self.has_next(): raise StopIteration() - assert self._next_result is not None + if self._next_result is None: + raise TypeError("self._next_result cannot be None") # Won't block next_result = self._next_result self._next_result = None @@ -725,7 +730,8 @@ def next(self): if not self.has_next(): raise StopIteration() - assert self._next_result is not None + if self._next_result is None: + raise TypeError("self._next_result cannot be None") # Won't block next_result = self._next_result self._next_result = None diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_gql.py b/packages/google-cloud-ndb/google/cloud/ndb/_gql.py index ec7962e2643b..f70cf8ad255e 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_gql.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_gql.py @@ -409,7 +409,8 @@ def _AddProcessedParameterFilter(self, identifier, condition, operator, paramete if identifier.lower() == "ancestor": self._has_ancestor = True filter_rule = (self._ANCESTOR, "is") - assert condition.lower() == "is" + if condition.lower() != "is": + raise ValueError("condition must be 'is'") if operator == "list" and condition.lower() not in ["in", "not_in"]: self._Error("Only IN can process a list of values, given '%s'" % condition) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/model.py b/packages/google-cloud-ndb/google/cloud/ndb/model.py index ac2fc866e09b..483d7544f2fe 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/model.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/model.py @@ -611,7 +611,8 @@ def new_entity(key): subvalue = value value = structprop._get_base_value(entity) if value in (None, []): # empty list for repeated props - assert structprop._model_class is not None + if structprop._model_class is None: + raise TypeError("structprop._model_class cannot be None") kind = structprop._model_class._get_kind() key = key_module.Key(kind, None) if structprop._repeated: @@ -642,7 +643,8 @@ def new_entity(key): # the other entries in the value list while len(subvalue) > len(value): # Need to make some more subentities - assert structprop._model_class is not None + if structprop._model_class is None: + raise TypeError("structprop._model_class cannot be None") expando_kind = structprop._model_class._get_kind() expando_key = key_module.Key(expando_kind, None) value.append(new_entity(expando_key._key)) @@ -2055,7 +2057,8 @@ def _legacy_deserialize(self, entity, p, unused_depth=1): if self._repeated: if self._has_value(entity): value = self._retrieve_value(entity) - assert isinstance(value, list), repr(value) + if not isinstance(value, list): + raise TypeError(f"Expected list, got {type(value)}") value.append(val) else: # We promote single values to lists if we are a list property @@ -4161,7 +4164,8 @@ def _get_for_dict(self, entity): def __getattr__(self, attrname): """Dynamically get a subproperty.""" - assert self._model_class is not None + if self._model_class is None: + raise TypeError("self._model_class cannot be None") # Optimistically try to use the dict key. prop = self._model_class._properties.get(attrname) @@ -4180,7 +4184,8 @@ def __getattr__(self, attrname): ) prop_copy = copy.copy(prop) - assert self._name is not None + if self._name is None: + raise TypeError("self._name cannot be None") prop_copy._name = self._name + "." + prop_copy._name # Cache the outcome, so subsequent requests for the same attribute @@ -4211,7 +4216,8 @@ def _comparison(self, op, value): value = self._do_validate(value) filters = [] match_keys = [] - assert self._model_class is not None + if self._model_class is None: + raise TypeError("self._model_class cannot be None") for prop_name, prop in self._model_class._properties.items(): subvalue = prop._get_value(value) if prop._repeated: @@ -4264,7 +4270,8 @@ def _IN(self, value, server_op=None): IN = _IN def _validate(self, value): - assert self._model_class is not None + if self._model_class is None: + raise TypeError("self._model_class cannot be None") if isinstance(value, dict): # A dict is assumed to be the result of a _to_dict() call. return self._model_class(**value) @@ -4325,7 +4332,8 @@ def _check_property(self, rest=None, require_indexed=True): raise InvalidPropertyError( "Structured property %s requires a subproperty" % self._name ) - assert self._model_class is not None + if self._model_class is None: + raise TypeError("self._model_class cannot be None") self._model_class._check_properties([rest], require_indexed=require_indexed) def _to_base_type(self, value): @@ -4340,7 +4348,8 @@ def _to_base_type(self, value): Raises: TypeError: If ``value`` is not the correct ``Model`` type. """ - assert self._model_class is not None + if self._model_class is None: + raise TypeError("self._model_class cannot be None") if not isinstance(value, self._model_class): raise TypeError( "Cannot convert to protocol buffer. Expected {} value; " @@ -4475,7 +4484,8 @@ def _validate(self, value): Raises: exceptions.BadValueError: If ``value`` is not a given class. """ - assert self._model_class is not None + if self._model_class is None: + raise TypeError("self._model_class cannot be None") if isinstance(value, dict): # A dict is assumed to be the result of a _to_dict() call. return self._model_class(**value) @@ -4504,7 +4514,8 @@ def _to_base_type(self, value): Raises: TypeError: If ``value`` is not the correct ``Model`` type. """ - assert self._model_class is not None + if self._model_class is None: + raise TypeError("self._model_class cannot be None") if not isinstance(value, self._model_class): raise TypeError( "Cannot convert to bytes expected {} value; " @@ -4522,7 +4533,8 @@ def _from_base_type(self, value): Returns: The converted value with given class. """ - assert self._model_class is not None + if self._model_class is None: + raise TypeError("self._model_class cannot be None") if isinstance(value, bytes): pb = entity_pb2.Entity() pb._pb.MergeFromString(value) @@ -4699,7 +4711,8 @@ def _get_value(self, entity): # UnprojectedPropertyError which will just bubble up. if entity._projection and self._name in entity._projection: return super(ComputedProperty, self)._get_value(entity) - assert self._func is not None + if self._func is None: + raise TypeError("self._func cannot be None") value = self._func(entity) self._store_value(entity, value) return value @@ -5035,7 +5048,8 @@ def _fake_property(self, p, next, indexed=True): # A custom 'meaning' for compressed properties. _MEANING_URI_COMPRESSED = "ZLIB" self._clone_properties() - assert self._properties is not None + if self._properties is None: + raise TypeError("self._properties cannot be None") prop: Property if p.name() != next.encode("utf-8") and not p.name().endswith( b"." + next.encode("utf-8") @@ -5349,13 +5363,15 @@ def _check_properties(cls, property_names, require_indexed=True): InvalidPropertyError: if one of the properties is invalid. AssertionError: if the argument is not a list or tuple of strings. """ - assert isinstance(property_names, (list, tuple)), repr(property_names) + if not isinstance(property_names, (list, tuple)): + raise TypeError(f"Expected list or tuple, got {type(property_names)}") for name in property_names: if "." in name: name, rest = name.split(".", 1) else: rest = None - assert cls._properties is not None + if cls._properties is None: + raise TypeError("cls._properties cannot be None") prop = cls._properties.get(name) if prop is None: raise InvalidPropertyError("Unknown property {}".format(name)) @@ -6199,7 +6215,8 @@ def _to_dict(self, include=None, exclude=None): to exclude. Default is to not exclude any names. """ values = {} - assert self._properties is not None + if self._properties is None: + raise TypeError("self._properties cannot be None") for prop in self._properties.values(): name = prop._code_name if include is not None and name not in include: @@ -6221,7 +6238,8 @@ def _to_dict(self, include=None, exclude=None): def _code_name_from_stored_name(cls, name): """Return the code name from a property when it's different from the stored name. Used in deserialization from datastore.""" - assert cls._properties is not None + if cls._properties is None: + raise TypeError("cls._properties cannot be None") if name in cls._properties: return cls._properties[name]._code_name @@ -6323,14 +6341,16 @@ def _set_attributes(self, kwds): setattr(self, name, value) def __getattr__(self, name): - assert self._properties is not None + if self._properties is None: + raise TypeError("self._properties cannot be None") prop = self._properties.get(name) if prop is None: return super(Expando, self).__getattribute__(name) return prop._get_value(self) def __setattr__(self, name, value): - assert self._properties is not None + if self._properties is None: + raise TypeError("self._properties cannot be None") if ( name.startswith("_") or isinstance(getattr(self.__class__, name, None), (Property, property)) @@ -6365,7 +6385,8 @@ def __setattr__(self, name, value): prop._set_value(self, value) def __delattr__(self, name): - assert self._properties is not None + if self._properties is None: + raise TypeError("self._properties cannot be None") if name.startswith("_") or isinstance( getattr(self.__class__, name, None), (Property, property) ): From 51c6e18bc6306e4a9e46256782380292270ac299 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 17 Apr 2026 12:21:16 -0400 Subject: [PATCH 6/9] adds comment about the transition from context to ctx --- packages/google-cloud-ndb/google/cloud/ndb/context.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/context.py b/packages/google-cloud-ndb/google/cloud/ndb/context.py index 9068e5a511b9..4b05a1604987 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/context.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/context.py @@ -319,6 +319,9 @@ def __new__( legacy_data=legacy_data, ) + # 'context' is dynamically composed at runtime and may include methods + # from multiple sources that Mypy cannot statically resolve here. + # We cast to `Any` to access the extended policy interface. ctx = cast(Any, context) ctx.set_cache_policy(cache_policy) ctx.set_global_cache_policy(global_cache_policy) From 787e904b34c331ee07ee199396ca7af549f9e6d4 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 17 Apr 2026 12:30:09 -0400 Subject: [PATCH 7/9] style: blacken ndb/model.py --- packages/google-cloud-ndb/google/cloud/ndb/model.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/model.py b/packages/google-cloud-ndb/google/cloud/ndb/model.py index 483d7544f2fe..cd636dfd2df7 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/model.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/model.py @@ -644,7 +644,9 @@ def new_entity(key): while len(subvalue) > len(value): # Need to make some more subentities if structprop._model_class is None: - raise TypeError("structprop._model_class cannot be None") + raise TypeError( + "structprop._model_class cannot be None" + ) expando_kind = structprop._model_class._get_kind() expando_key = key_module.Key(expando_kind, None) value.append(new_entity(expando_key._key)) From 4f9468df3b1437fe1cd640f62f1b60b3d784a810 Mon Sep 17 00:00:00 2001 From: chalmer lowe Date: Fri, 17 Apr 2026 12:55:50 -0400 Subject: [PATCH 8/9] style: blacken ndb/context.py --- packages/google-cloud-ndb/google/cloud/ndb/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/context.py b/packages/google-cloud-ndb/google/cloud/ndb/context.py index 4b05a1604987..25e90763b5bc 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/context.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/context.py @@ -319,8 +319,8 @@ def __new__( legacy_data=legacy_data, ) - # 'context' is dynamically composed at runtime and may include methods - # from multiple sources that Mypy cannot statically resolve here. + # 'context' is dynamically composed at runtime and may include methods + # from multiple sources that Mypy cannot statically resolve here. # We cast to `Any` to access the extended policy interface. ctx = cast(Any, context) ctx.set_cache_policy(cache_policy) From 0ee10cce82ecde20948e29903fe20eb5e234aa96 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 17 Apr 2026 14:24:55 -0400 Subject: [PATCH 9/9] Update temp variable name from st to parsed_time --- packages/google-cloud-ndb/google/cloud/ndb/_gql.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/google-cloud-ndb/google/cloud/ndb/_gql.py b/packages/google-cloud-ndb/google/cloud/ndb/_gql.py index f70cf8ad255e..9a6b225ec0ab 100644 --- a/packages/google-cloud-ndb/google/cloud/ndb/_gql.py +++ b/packages/google-cloud-ndb/google/cloud/ndb/_gql.py @@ -777,12 +777,16 @@ def _time_function(values): value = values[0] if isinstance(value, str): try: - st = time.strptime(value, "%H:%M:%S") + parsed_time = time.strptime(value, "%H:%M:%S") except ValueError as error: _raise_cast_error( "Error during time conversion, {}, {}".format(error, values) ) - t_tuple = (st.tm_hour, st.tm_min, st.tm_sec) + t_tuple = ( + parsed_time.tm_hour, + parsed_time.tm_min, + parsed_time.tm_sec, + ) elif isinstance(value, int): t_tuple = (value,) else: