Add pycroscope to the conformance suite

conformance/README.md

@@ -81,7 +81,7 @@ Note that some type checkers may not run on some platforms. If a type checker fa
 
 Different type checkers report errors in different ways (with different wording in error messages and different line numbers or character ranges for errors). This variation makes it difficult to fully automate test validation given that tests will want to check for both false positive and false negative type errors. Some level of manual inspection will therefore be needed to determine whether a type checker is fully conformant with all tests in any given test file. This "scoring" process is required only when the output of a test changes — e.g. when a new version of that type checker is released and the tests are rerun. We assume that the output of a type checker will be the same from one run to the next unless/until a new version is released that fixes or introduces a bug. In this case, the output will need to be manually inspected and the conformance results re-scored for those tests whose output has changed.
 
-[Conformance results](https://htmlpreview.github.io/?https://github.com/python/typing/blob/main/conformance/results/results.html) are reported and summarized for each supported type checker. Currently, results are reported for mypy, pyrefly, pyright, zuban and ty. It is the goal and desire to add additional type checkers over time.
+[Conformance results](https://htmlpreview.github.io/?https://github.com/python/typing/blob/main/conformance/results/results.html) are reported and summarized for each supported type checker. Currently, results are reported for mypy, pyrefly, pyright, zuban, ty, and pycroscope. It is the goal and desire to add additional type checkers over time.
 
 ## Adding a New Test Case
 

conformance/pyproject.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 requires-python = "==3.12.*"
 dependencies = [
     "mypy",
+    "pycroscope==0.4.0",
     "pyrefly",
     "pyright",
     "tomli",

conformance/results/pycroscope/aliases_explicit.toml

@@ -0,0 +1,26 @@
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+./aliases_explicit.py:67:8: Expected 0 type arguments for type alias, got 1 [invalid_specialization]
+./aliases_explicit.py:68:8: Expected 0 type arguments for type alias, got 1 [invalid_specialization]
+./aliases_explicit.py:69:8: Expected 1 type arguments for type alias, got 2 [invalid_specialization]
+./aliases_explicit.py:70:8: Expected 1 type arguments for type alias, got 2 [invalid_specialization]
+./aliases_explicit.py:71:8: ParamSpec specialization must use list form, Concatenate[..., P], P, or ... [invalid_annotation]
+./aliases_explicit.py:79:20: Invalid type annotation [invalid_annotation]
+./aliases_explicit.py:80:20: Unrecognized annotation <list containing [type 'int', type 'str']> [invalid_annotation]
+./aliases_explicit.py:81:20: Unrecognized annotation tuple[tuple[type 'int', type 'str']] [invalid_annotation]
+./aliases_explicit.py:82:20: Invalid type annotation [invalid_annotation]
+./aliases_explicit.py:83:20: Unrecognized annotation <dict containing {Literal['a']: Literal['b']}> [invalid_annotation]
+./aliases_explicit.py:84:20: Invalid type annotation [invalid_annotation]
+./aliases_explicit.py:85:20: Cannot resolve subscripted annotation: <list containing [type 'int']> [invalid_annotation]
+./aliases_explicit.py:86:20: Invalid type annotation [invalid_annotation]
+./aliases_explicit.py:87:20: Invalid type annotation 3 [invalid_annotation]
+./aliases_explicit.py:88:21: Invalid type annotation True [invalid_annotation]
+./aliases_explicit.py:89:21: Invalid type annotation 1 [invalid_annotation]
+./aliases_explicit.py:90:21: Invalid type annotation [invalid_annotation]
+./aliases_explicit.py:91:21: Invalid type annotation [invalid_annotation]
+./aliases_explicit.py:100:4: Expected 0 type arguments for type alias, got 1 [invalid_specialization]
+./aliases_explicit.py:101:5: TypeForm[.ListOrSetAlias] (synthetic from Literal[list | set]) is not callable [not_callable]
+./aliases_explicit.py:102:4: Expected 0 type arguments for type alias, got 1 [invalid_specialization]
+"""

conformance/results/pycroscope/aliases_implicit.toml

@@ -0,0 +1,42 @@
+conformant = "Partial"
+notes = """
+Fails to handle various weird annotations.
+Various bugs with resolving generic aliases.
+"""
+conformance_automated = "Fail"
+errors_diff = """
+Line 76: Expected 1 errors
+Line 106: Expected 1 errors
+Line 111: Expected 1 errors
+Line 112: Expected 1 errors
+Line 113: Expected 1 errors
+Line 117: Expected 1 errors
+Line 118: Expected 1 errors
+Line 119: Expected 1 errors
+Line 68: Unexpected errors ['./aliases_implicit.py:68:16:   .GoodTypeAlias9[<list containing [str, str]>, None] = (int, /, ****P) -> ~R is not equivalent to (int, str, str, /) -> None']
+Line 70: Unexpected errors ['./aliases_implicit.py:70:16:   Any[from_another] is not equivalent to int | str | list[list[int]] | None']
+Line 72: Unexpected errors ['./aliases_implicit.py:72:16:   .GoodTypeAlias13 = (**__P: **P) -> None is not equivalent to (...) -> None']
+Line 95: Unexpected errors ["./aliases_implicit.py:95:16: Cannot resolve subscripted annotation: <list containing [type 'int']> [invalid_annotation]"]
+Line 100: Unexpected errors ["./aliases_implicit.py:100:17: Annotated[type 'list', (<+list is_truthy None> == NullConstraint())] is always True because it does not provide __bool__ [type_always_true]"]
+"""
+output = """
+./aliases_implicit.py:68:16:   .GoodTypeAlias9[<list containing [str, str]>, None] = (int, /, ****P) -> ~R is not equivalent to (int, str, str, /) -> None
+./aliases_implicit.py:70:16:   Any[from_another] is not equivalent to int | str | list[list[int]] | None
+./aliases_implicit.py:72:16:   .GoodTypeAlias13 = (**__P: **P) -> None is not equivalent to (...) -> None
+./aliases_implicit.py:77:8: Unrecognized annotation types.GenericAlias [invalid_annotation]
+./aliases_implicit.py:78:8: Expected 1 type arguments for type alias, got 2 [invalid_specialization]
+./aliases_implicit.py:79:8: Expected 1 type arguments for type alias, got 2 [invalid_specialization]
+./aliases_implicit.py:80:8: ParamSpec specialization must use list form, Concatenate[..., P], P, or ... [invalid_annotation]
+./aliases_implicit.py:81:8: Type argument str is not compatible with ~TFloat@./aliases_implicit.py.GoodTypeAlias12 [invalid_specialization]
+./aliases_implicit.py:95:16: Cannot resolve subscripted annotation: <list containing [type 'int']> [invalid_annotation]
+./aliases_implicit.py:100:17: Annotated[type 'list', (<+list is_truthy None> == NullConstraint())] is always True because it does not provide __bool__ [type_always_true]
+./aliases_implicit.py:107:8: Invalid type annotation [<class 'int'>, <class 'str'>] [invalid_annotation]
+./aliases_implicit.py:108:8: Invalid type annotation ((<class 'int'>, <class 'str'>),) [invalid_annotation]
+./aliases_implicit.py:109:8: Unrecognized annotation <list containing [type 'int']> [invalid_annotation]
+./aliases_implicit.py:110:8: Invalid type annotation {'a': 'b'} [invalid_annotation]
+./aliases_implicit.py:114:8: Invalid type annotation 3 [invalid_annotation]
+./aliases_implicit.py:115:9: Invalid type annotation True [invalid_annotation]
+./aliases_implicit.py:116:9: Invalid type annotation 1 [invalid_annotation]
+./aliases_implicit.py:133:5: Literal[list | set] is not callable [not_callable]
+./aliases_implicit.py:135:4: Unrecognized annotation object [invalid_annotation]
+"""

conformance/results/pycroscope/aliases_newtype.toml

@@ -0,0 +1,19 @@
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+./aliases_newtype.py:11:7: Incompatible argument type for x: expected int but got Literal['user'] [incompatible_argument]
+./aliases_newtype.py:12:0: Incompatible assignment: expected NewType('UserId', int), got Literal[42] [incompatible_assignment]
+./aliases_newtype.py:18:0: Incompatible assignment: expected type, got Literal[pycroscope.implementation.UserId] [incompatible_assignment]
+./aliases_newtype.py:23:15: Second argument to "isinstance" must be a type, union, or tuple of types, not "pycroscope.implementation.UserId" [incompatible_argument]
+./aliases_newtype.py:26:20: NewType cannot be used as a base class [invalid_base]
+./aliases_newtype.py:35:19: NewType named 'BadName' must be assigned to a variable with the same name [must_have_same_name]
+./aliases_newtype.py:41:5: Object Literal[pycroscope.implementation.GoodNewType1] does not support subscripting [unsupported_operation]
+./aliases_newtype.py:47:37: NewType base type cannot be generic [incompatible_call]
+./aliases_newtype.py:50:37: NewType base type cannot be generic [incompatible_call]
+./aliases_newtype.py:52:37: NewType base type cannot be a protocol [incompatible_call]
+./aliases_newtype.py:54:37: NewType base type cannot be a literal type [incompatible_call]
+./aliases_newtype.py:61:37: NewType base type cannot be a TypedDict [incompatible_call]
+./aliases_newtype.py:63:14: In call to typing.NewType: Takes 2 positional arguments but 3 were given [incompatible_call]
+./aliases_newtype.py:65:37: NewType base type cannot be Any [incompatible_call]
+"""

conformance/results/pycroscope/aliases_recursive.toml

@@ -0,0 +1,17 @@
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+./aliases_recursive.py:19:0: Incompatible assignment: expected int | str | float | list[int | str | float | list[<unknown>.Json = int | str | float | list[int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | dict[str, int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | None] | dict[str, <unknown>.Json = int | str | float | list[int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | dict[str, int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | None] | None] | dict[str, int | str | float | list[<unknown>.Json = int | str | float | list[int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | dict[str, int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | None] | dict[str, <unknown>.Json = int | str | float | list[int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | dict[str, int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | None] | None] | None, got Literal[{'a': 1, 'b': 3j}] [incompatible_assignment]
+./aliases_recursive.py:20:0: Incompatible assignment: expected int | str | float | list[int | str | float | list[<unknown>.Json = int | str | float | list[int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | dict[str, int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | None] | dict[str, <unknown>.Json = int | str | float | list[int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | dict[str, int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | None] | None] | dict[str, int | str | float | list[<unknown>.Json = int | str | float | list[int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | dict[str, int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | None] | dict[str, <unknown>.Json = int | str | float | list[int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | dict[str, int | str | float | list[<unknown>.Json] | dict[str, <unknown>.Json] | None] | None] | None] | None, got Literal[[2, 3j]] [incompatible_assignment]
+./aliases_recursive.py:38:0: Incompatible assignment: expected str | int | tuple[str | int | tuple[<unknown>.RecursiveTuple = str | int | tuple[str | int | tuple[<unknown>.RecursiveTuple, ...], ...], ...], ...], got Literal[(1, ('1', 1), (1, (1, [2])))] [incompatible_assignment]
+./aliases_recursive.py:39:0: Incompatible assignment: expected str | int | tuple[str | int | tuple[<unknown>.RecursiveTuple = str | int | tuple[str | int | tuple[<unknown>.RecursiveTuple, ...], ...], ...], ...], got Literal[(1, [1])] [incompatible_assignment]
+./aliases_recursive.py:50:0: Incompatible assignment: expected str | int | collections.abc.Mapping[str, str | int | collections.abc.Mapping[str, None.RecursiveMapping = str | int | collections.abc.Mapping[str, str | int | collections.abc.Mapping[str, None.RecursiveMapping]]]], got Literal[{'1': [1]}] [incompatible_assignment]
+./aliases_recursive.py:51:0: Incompatible assignment: expected str | int | collections.abc.Mapping[str, str | int | collections.abc.Mapping[str, None.RecursiveMapping = str | int | collections.abc.Mapping[str, str | int | collections.abc.Mapping[str, None.RecursiveMapping]]]], got Literal[{'1': '1', '2': 1, '3': [1, 2]}] [incompatible_assignment]
+./aliases_recursive.py:52:0: Incompatible assignment: expected str | int | collections.abc.Mapping[str, str | int | collections.abc.Mapping[str, None.RecursiveMapping = str | int | collections.abc.Mapping[str, str | int | collections.abc.Mapping[str, None.RecursiveMapping]]]], got Literal[{'1': '1', '2': 1, '3': {'0': '0', '1': 1, '2': [1, 2, 3]}}] [incompatible_assignment]
+./aliases_recursive.py:63:0: Incompatible assignment: expected aliases_recursive.GenericTypeAlias1[str] = list[aliases_recursive.GenericTypeAlias1[~T1] = list[aliases_recursive.GenericTypeAlias1[~T1] | ~T1] | ~T1], got Literal[['hi', [2.4]]] [incompatible_assignment]
+./aliases_recursive.py:69:0: Incompatible assignment: expected aliases_recursive.GenericTypeAlias2[str, int] = list[aliases_recursive.GenericTypeAlias2[~T1, ~T2] = list[aliases_recursive.GenericTypeAlias2[~T1, ~T2] | ~T1 | ~T2] | ~T1 | ~T2], got Literal[[[3, ['hi', 3, [3.4]]], 'hi']] [incompatible_assignment]
+./aliases_recursive.py:72:0: Type alias RecursiveUnion has a circular definition [invalid_type_alias]
+./aliases_recursive.py:75:0: Type alias MutualReference1 has a circular definition [invalid_type_alias]
+./aliases_recursive.py:75:62: Type alias MutualReference2 has a circular definition [invalid_type_alias]
+"""

conformance/results/pycroscope/aliases_type_statement.toml

@@ -0,0 +1,35 @@
+conformant = "Partial"
+notes = """
+Fails to reject various weird annotations.
+"""
+conformance_automated = "Fail"
+errors_diff = """
+Line 37: Expected 1 errors
+Line 38: Expected 1 errors
+Line 39: Expected 1 errors
+Line 40: Expected 1 errors
+Line 41: Expected 1 errors
+Line 42: Expected 1 errors
+Line 43: Expected 1 errors
+Line 44: Expected 1 errors
+Line 45: Expected 1 errors
+Line 46: Expected 1 errors
+Line 47: Expected 1 errors
+Line 49: Expected 1 errors
+"""
+output = """
+./aliases_type_statement.py:17:0: typing.TypeAliasType has no attribute 'bit_count' [undefined_attribute]
+./aliases_type_statement.py:19:0: TypeForm[.GoodAlias1] (synthetic from typing.TypeAliasType) is not callable [not_callable]
+./aliases_type_statement.py:23:6: typing.TypeAliasType has no attribute 'other_attrib' [undefined_attribute]
+./aliases_type_statement.py:26:17: Type alias cannot be used as a base class [invalid_base]
+./aliases_type_statement.py:31:21: Second argument to "isinstance" cannot be a type alias [incompatible_argument]
+./aliases_type_statement.py:48:22: Annotated[type 'list', (<+list is_truthy None> == NullConstraint())] is always True because it does not provide __bool__ [type_always_true]
+./aliases_type_statement.py:53:22: Type alias cannot combine old-style TypeVar declarations with type statement parameters [invalid_type_alias]
+./aliases_type_statement.py:58:16: Type alias must declare type parameters in the type statement [invalid_type_alias]
+./aliases_type_statement.py:68:6: Type argument str is not compatible with ~S@./aliases_type_statement.py.RecursiveTypeAlias2 [invalid_specialization]
+./aliases_type_statement.py:70:6: Type argument int is not compatible with ~T@./aliases_type_statement.py.RecursiveTypeAlias2 [invalid_specialization]
+./aliases_type_statement.py:73:0: Type alias RecursiveTypeAlias3 has a circular definition [invalid_type_alias]
+./aliases_type_statement.py:75:0: Type alias RecursiveTypeAlias4 has a circular definition [invalid_type_alias]
+./aliases_type_statement.py:79:0: Type alias RecursiveTypeAlias6 has a circular definition [invalid_type_alias]
+./aliases_type_statement.py:80:0: Type alias RecursiveTypeAlias7 has a circular definition [invalid_type_alias]
+"""

conformance/results/pycroscope/aliases_typealiastype.toml

@@ -0,0 +1,34 @@
+conformant = "Partial"
+notes = """
+Rejects valid ParamSpec specialization.
+"""
+conformance_automated = "Fail"
+errors_diff = """
+Line 39: Unexpected errors ["./aliases_typealiastype.py:39:4: Unrecognized annotation <list containing [type 'int', type 'str']> [invalid_annotation]"]
+"""
+output = """
+./aliases_typealiastype.py:32:6: typing.TypeAliasType has no attribute 'other_attrib' [undefined_attribute]
+./aliases_typealiastype.py:39:4: Unrecognized annotation <list containing [type 'int', type 'str']> [invalid_annotation]
+./aliases_typealiastype.py:40:4: Type argument int is not compatible with ~TStr@./aliases_typealiastype.py.GoodAlias5 [invalid_specialization]
+./aliases_typealiastype.py:43:44: Type alias cannot combine old-style TypeVar declarations with type statement parameters [invalid_type_alias]
+./aliases_typealiastype.py:44:44: Type alias must declare type parameters in the type statement [invalid_type_alias]
+./aliases_typealiastype.py:45:56: type_params argument to TypeAliasType must be a literal tuple [invalid_type_alias]
+./aliases_typealiastype.py:46:0: Type alias BadAlias4 has a circular definition [invalid_type_alias]
+./aliases_typealiastype.py:47:0: Type alias BadAlias5 has a circular definition [invalid_type_alias]
+./aliases_typealiastype.py:48:0: Type alias BadAlias6 has a circular definition [invalid_type_alias]
+./aliases_typealiastype.py:49:0: Type alias BadAlias7 has a circular definition [invalid_type_alias]
+./aliases_typealiastype.py:52:39: Invalid type annotation [invalid_annotation]
+./aliases_typealiastype.py:53:39: Unrecognized annotation <list containing [type 'int', type 'str']> [invalid_annotation]
+./aliases_typealiastype.py:54:41: Unrecognized annotation tuple[tuple[type 'int', type 'str']] [invalid_annotation]
+./aliases_typealiastype.py:55:41: Invalid type annotation [invalid_annotation]
+./aliases_typealiastype.py:56:41: Unrecognized annotation <dict containing {Literal['a']: Literal['b']}> [invalid_annotation]
+./aliases_typealiastype.py:57:41: Invalid type annotation [invalid_annotation]
+./aliases_typealiastype.py:58:41: Cannot resolve subscripted annotation: <list containing [type 'int']> [invalid_annotation]
+./aliases_typealiastype.py:59:41: Invalid type annotation [invalid_annotation]
+./aliases_typealiastype.py:60:41: Invalid type annotation 3 [invalid_annotation]
+./aliases_typealiastype.py:61:41: Invalid type annotation True [invalid_annotation]
+./aliases_typealiastype.py:62:41: Invalid type annotation 1 [invalid_annotation]
+./aliases_typealiastype.py:63:41: Invalid type annotation [invalid_annotation]
+./aliases_typealiastype.py:64:41: Invalid type annotation [invalid_annotation]
+./aliases_typealiastype.py:66:0: Type alias BadAlias21 has a circular definition [invalid_type_alias]
+"""

conformance/results/pycroscope/aliases_variance.toml

@@ -0,0 +1,9 @@
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+./aliases_variance.py:24:15: T_co has incompatible variance in base class [invalid_base]
+./aliases_variance.py:28:15: T_co has incompatible variance in base class [invalid_base]
+./aliases_variance.py:32:15: T_co has incompatible variance in base class [invalid_base]
+./aliases_variance.py:44:15: T_contra has incompatible variance in base class [invalid_base]
+"""

conformance/results/pycroscope/annotations_coroutines.toml

@@ -0,0 +1,5 @@
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+"""