diff --git a/auditwheel/error.py b/auditwheel/error.py new file mode 100644 index 00000000..52887b2c --- /dev/null +++ b/auditwheel/error.py @@ -0,0 +1,6 @@ +class AuditwheelException(Exception): + pass + + +class InvalidPlatform(AuditwheelException): + pass diff --git a/auditwheel/main_repair.py b/auditwheel/main_repair.py index 0f357cf6..dcac4ba7 100644 --- a/auditwheel/main_repair.py +++ b/auditwheel/main_repair.py @@ -2,7 +2,7 @@ from auditwheel.patcher import Patchelf from .policy import (load_policies, get_policy_by_name, get_policy_name, - get_priority_by_name, POLICY_PRIORITY_HIGHEST) + get_priority_by_name, get_policy_platform) from .tools import EnvironmentDefault import argparse import logging @@ -12,20 +12,21 @@ def configure_parser(sub_parsers): - policies = load_policies() - policy_names = [p['name'] for p in policies] - policy_names += [alias for p in policies for alias in p['aliases']] + policies = load_policies(get_policy_platform()) + policy_names = [p['name'] for p in policies.policies] + policy_names += [alias for p in policies.policies + for alias in p['aliases']] epilog = """PLATFORMS: These are the possible target platform tags, as specified by PEP 600. Note that old, pre-PEP 600 tags are still usable and are listed as aliases below. """ - for p in policies: + for p in policies.policies: epilog += f"- {p['name']}" if len(p['aliases']) > 0: epilog += f" (aliased by {', '.join(p['aliases'])})" epilog += '\n' - highest_policy = get_policy_name(POLICY_PRIORITY_HIGHEST) + highest_policy = get_policy_name(policies, policies.highest) help = "Vendor in external shared library dependencies of a wheel." p = sub_parsers.add_parser( 'repair', help=help, description=help, epilog=epilog, @@ -92,17 +93,18 @@ def execute(args, p): logger.info('This does not look like a platform wheel') return 1 - policy = get_policy_by_name(args.PLAT) + policies = load_policies(get_policy_platform()) + policy = get_policy_by_name(policies, args.PLAT) reqd_tag = policy['priority'] - if reqd_tag > get_priority_by_name(wheel_abi.sym_tag): + if reqd_tag > get_priority_by_name(policies, wheel_abi.sym_tag): msg = ('cannot repair "%s" to "%s" ABI because of the presence ' 'of too-recent versioned symbols. You\'ll need to compile ' 'the wheel on an older toolchain.' % (args.WHEEL_FILE, args.PLAT)) p.error(msg) - if reqd_tag > get_priority_by_name(wheel_abi.ucs_tag): + if reqd_tag > get_priority_by_name(policies, wheel_abi.ucs_tag): msg = ('cannot repair "%s" to "%s" ABI because it was compiled ' 'against a UCS2 build of Python. You\'ll need to compile ' 'the wheel against a wide-unicode build of Python.' % @@ -111,12 +113,12 @@ def execute(args, p): abis = [policy['name']] + policy['aliases'] if not args.ONLY_PLAT: - if reqd_tag < get_priority_by_name(wheel_abi.overall_tag): + if reqd_tag < get_priority_by_name(policies, wheel_abi.overall_tag): logger.info(('Wheel is eligible for a higher priority tag. ' 'You requested %s but I have found this wheel is ' 'eligible for %s.'), args.PLAT, wheel_abi.overall_tag) - higher_policy = get_policy_by_name(wheel_abi.overall_tag) + higher_policy = get_policy_by_name(policies, wheel_abi.overall_tag) abis = [higher_policy['name']] + higher_policy['aliases'] + abis patcher = Patchelf() diff --git a/auditwheel/main_show.py b/auditwheel/main_show.py index f380a87e..eedd30aa 100644 --- a/auditwheel/main_show.py +++ b/auditwheel/main_show.py @@ -2,6 +2,8 @@ from collections import OrderedDict +from auditwheel.policy import get_policy_platform + logger = logging.getLogger(__name__) @@ -82,7 +84,9 @@ def execute(args, p): 'by the wheel:') print(json.dumps(OrderedDict(sorted(libs.items())), indent=4)) - for p in sorted(load_policies(), key=lambda p: p['priority']): + policies = load_policies(get_policy_platform()) + for p in sorted(policies.policies, + key=lambda p: p['priority']): if p['priority'] > get_priority_by_name(winfo.overall_tag): printp(('In order to achieve the tag platform tag "%s" ' 'the following shared library dependencies ' diff --git a/auditwheel/musllinux.py b/auditwheel/musllinux.py new file mode 100644 index 00000000..2e3658f2 --- /dev/null +++ b/auditwheel/musllinux.py @@ -0,0 +1,65 @@ +import logging +import pathlib +import subprocess +import re +import sys +from typing import NamedTuple + +from auditwheel.error import InvalidPlatform + +LOG = logging.getLogger(__name__) + + +class MuslVersion(NamedTuple): + major: int + minor: int + patch: int + + +def find_musl_libc() -> pathlib.Path: + try: + ldd = subprocess.run(["ldd", "/bin/ls"], + text=True, + capture_output=True) + except subprocess.CalledProcessError: + LOG.error("Failed to determine libc version", exc_info=True) + raise + + match = re.search( + r"libc\.musl-(?P\w+)\.so.1 " # TODO drop the platform + r"=> (?P[/\-\w.]+)", + ldd.stdout) + + if not match: + raise InvalidPlatform + + return pathlib.Path(match.group("path")) + + +def get_musl_version(ld_path: pathlib.Path) -> MuslVersion: + try: + ld = subprocess.run([ld_path], text=True, capture_output=True) + except subprocess.CalledProcessError: + LOG.error("Failed to determine musl version", exc_info=True) + raise + + match = re.search( + r"Version " + r"(?P\d)." + r"(?P\d)." + r"(?P\d)", + ld.stderr) + if not match: + raise InvalidPlatform + + return MuslVersion( + int(match.group("major")), + int(match.group("minor")), + int(match.group("patch"))) + + +if __name__ == "__main__": + libc_path = find_musl_libc() + version = get_musl_version(libc_path) + print(f"Found musl version {version} in {libc_path}") + wheel_path = pathlib.Path(sys.argv[1]) diff --git a/auditwheel/policy/__init__.py b/auditwheel/policy/__init__.py index 96898bac..5f1ea014 100644 --- a/auditwheel/policy/__init__.py +++ b/auditwheel/policy/__init__.py @@ -1,14 +1,35 @@ +import functools +import pathlib import sys import json import platform as _platform_module +import typing from collections import defaultdict +from enum import Enum from typing import Dict, List, Optional, Set from os.path import join, dirname, abspath import logging +from ..error import InvalidPlatform +from ..musllinux import find_musl_libc + +_HERE = pathlib.Path(__file__).parent logger = logging.getLogger(__name__) + +class Platform(Enum): + Manylinux = 1, + Musllinux = 2 + + +class PlatformPolicies(typing.NamedTuple): + platform: Platform + policies: List + lowest: int + highest: int + + # https://docs.python.org/3/library/platform.html#platform.architecture bits = 8 * (8 if sys.maxsize > 2 ** 32 else 4) @@ -21,6 +42,16 @@ def get_arch_name() -> str: return {64: 'x86_64', 32: 'i686'}[bits] +def get_policy_platform() -> Platform: + try: + find_musl_libc() + logger.debug("Detected musl libc") + return Platform.Musllinux + except InvalidPlatform: + logger.debug("Falling back to GNU libc") + return Platform.Manylinux + + _ARCH_NAME = get_arch_name() @@ -56,25 +87,40 @@ def _validate_pep600_compliance(policies) -> None: symbol_versions[arch] = symbol_versions_arch -with open(join(dirname(abspath(__file__)), 'manylinux-policy.json')) as f: - _POLICIES = [] - _policies_temp = json.load(f) - _validate_pep600_compliance(_policies_temp) +def _load_policy(path: pathlib.Path, platform: Platform): + _policies_temp = json.loads(path.read_text()) + _policies = [] + if platform == Platform.Manylinux: + _validate_pep600_compliance(_policies_temp) for _p in _policies_temp: - if _ARCH_NAME in _p['symbol_versions'].keys() or _p['name'] == 'linux': + arch = _p['symbol_versions'].keys() + if _ARCH_NAME in arch or _p['name'] == 'linux': if _p['name'] != 'linux': _p['symbol_versions'] = _p['symbol_versions'][_ARCH_NAME] _p['name'] = _p['name'] + '_' + _ARCH_NAME _p['aliases'] = [alias + '_' + _ARCH_NAME for alias in _p['aliases']] - _POLICIES.append(_p) + _policies.append(_p) -POLICY_PRIORITY_HIGHEST = max(p['priority'] for p in _POLICIES) -POLICY_PRIORITY_LOWEST = min(p['priority'] for p in _POLICIES) + priority_highest = max(p['priority'] for p in _policies) + priority_lowest = min(p['priority'] for p in _policies) + return PlatformPolicies(platform=platform, + policies=_policies, + lowest=priority_lowest, + highest=priority_highest) -def load_policies(): - return _POLICIES + +@functools.lru_cache() +def load_policies(policy: Platform): + if policy == Platform.Manylinux: + return _load_policy(_HERE / "manylinux-policy.json", + Platform.Manylinux) + elif policy == Platform.Musllinux: + return _load_policy(_HERE / "musllinux-policy.json", + Platform.Musllinux) + else: + raise ValueError("Invalid policy") def _load_policy_schema(): @@ -83,8 +129,10 @@ def _load_policy_schema(): return schema -def get_policy_by_name(name: str) -> Optional[Dict]: - matches = [p for p in _POLICIES +def get_policy_by_name( + policies: PlatformPolicies, + name: str) -> Optional[Dict]: + matches = [p for p in policies.policies if p['name'] == name or name in p['aliases']] if len(matches) == 0: return None @@ -93,8 +141,11 @@ def get_policy_by_name(name: str) -> Optional[Dict]: return matches[0] -def get_policy_name(priority: int) -> Optional[str]: - matches = [p['name'] for p in _POLICIES if p['priority'] == priority] +def get_policy_name( + policies: PlatformPolicies, + priority: int) -> Optional[str]: + matches = [p['name'] for p in policies.policies + if p['priority'] == priority] if len(matches) == 0: return None if len(matches) > 1: @@ -102,8 +153,10 @@ def get_policy_name(priority: int) -> Optional[str]: return matches[0] -def get_priority_by_name(name: str) -> Optional[int]: - policy = get_policy_by_name(name) +def get_priority_by_name( + policies: PlatformPolicies, + name: str) -> Optional[int]: + policy = get_policy_by_name(policies, name) return None if policy is None else policy['priority'] @@ -132,5 +185,4 @@ def get_replace_platforms(name: str) -> List[str]: from .versioned_symbols import versioned_symbols_policy # noqa __all__ = ['lddtree_external_references', 'versioned_symbols_policy', - 'load_policies', 'POLICY_PRIORITY_HIGHEST', - 'POLICY_PRIORITY_LOWEST'] + 'load_policies'] diff --git a/auditwheel/policy/external_references.py b/auditwheel/policy/external_references.py index dd188ed8..9682a4df 100644 --- a/auditwheel/policy/external_references.py +++ b/auditwheel/policy/external_references.py @@ -3,7 +3,7 @@ from typing import Dict, Set, Any, Generator from ..elfutils import is_subdir -from . import load_policies +from . import load_policies, get_policy_platform log = logging.getLogger(__name__) LIBPYTHON_RE = re.compile(r'^libpython\d\.\dm?.so(.\d)*$') @@ -12,7 +12,7 @@ def lddtree_external_references(lddtree: Dict, wheel_path: str) -> Dict: # XXX: Document the lddtree structure, or put it in something # more stable than a big nested dict - policies = load_policies() + policies = load_policies(get_policy_platform()) def filter_libs(libs: Set[str], whitelist: Set[str]) -> Generator[str, None, None]: @@ -44,7 +44,7 @@ def get_req_external(libs: Set[str], whitelist: Set[str]) -> Set[str]: return reqs ret = {} # type: Dict[str, Dict[str, Any]] - for p in policies: + for p in policies.policies: needed_external_libs = set() # type: Set[str] if not (p['name'] == 'linux' and p['priority'] == 0): diff --git a/auditwheel/policy/musllinux-policy.json b/auditwheel/policy/musllinux-policy.json new file mode 100644 index 00000000..0d07b05c --- /dev/null +++ b/auditwheel/policy/musllinux-policy.json @@ -0,0 +1,22 @@ +[ + {"name": "musllinux_1_1", + "aliases": [], + "priority": 100, + "symbol_versions": { + "i686": { + }, + "x86_64": { + }, + "aarch64": { + }, + "ppc64le": { + }, + "s390x": { + }, + "armv7l": { + } + }, + "lib_whitelist": [ + "libc.so", "libz.so" + ]} +] diff --git a/auditwheel/policy/versioned_symbols.py b/auditwheel/policy/versioned_symbols.py index 8c3e25f9..5c1ce014 100644 --- a/auditwheel/policy/versioned_symbols.py +++ b/auditwheel/policy/versioned_symbols.py @@ -1,7 +1,7 @@ import logging from typing import Dict, List, Set -from . import load_policies +from . import load_policies, get_policy_platform log = logging.getLogger(__name__) @@ -25,7 +25,8 @@ def policy_is_satisfied(policy_name: str, sym_name, _, _ = symbol.partition("_") required_vers.setdefault(sym_name, set()).add(symbol) matching_policies = [] # type: List[int] - for p in load_policies(): + policies = load_policies(get_policy_platform()) + for p in policies.policies: policy_sym_vers = { sym_name: {sym_name + '_' + version for version in versions} for sym_name, versions in p['symbol_versions'].items() diff --git a/auditwheel/wheel_abi.py b/auditwheel/wheel_abi.py index de671aa8..5d39eb62 100644 --- a/auditwheel/wheel_abi.py +++ b/auditwheel/wheel_abi.py @@ -15,8 +15,7 @@ elf_references_PyFPE_jbuf, elf_find_ucs2_symbols, elf_is_python_extension) from .policy import (lddtree_external_references, versioned_symbols_policy, - get_policy_name, POLICY_PRIORITY_LOWEST, - POLICY_PRIORITY_HIGHEST, load_policies) + get_policy_name, load_policies, get_policy_platform) log = logging.getLogger(__name__) @@ -200,10 +199,11 @@ def get_symbol_policies(versioned_symbols, external_versioned_symbols, def analyze_wheel_abi(wheel_fn: str) -> WheelAbIInfo: + policies = load_policies(get_policy_platform()) external_refs = { p['name']: {'libs': {}, 'priority': p['priority']} - for p in load_policies() + for p in policies.policies } (elftree_by_fn, external_refs_by_fn, versioned_symbols, has_ucs2, @@ -230,25 +230,27 @@ def analyze_wheel_abi(wheel_fn: str) -> WheelAbIInfo: default=(symbol_policy, versioned_symbols) ) + policies = load_policies(get_policy_platform()) ref_policy = max( (e['priority'] for e in external_refs.values() if len(e['libs']) == 0), - default=POLICY_PRIORITY_LOWEST) + default=policies.lowest) if has_ucs2: - ucs_policy = POLICY_PRIORITY_LOWEST + ucs_policy = policies.lowest else: - ucs_policy = POLICY_PRIORITY_HIGHEST + ucs_policy = policies.lowest if uses_PyFPE_jbuf: - pyfpe_policy = POLICY_PRIORITY_LOWEST + pyfpe_policy = policies.lowest else: - pyfpe_policy = POLICY_PRIORITY_HIGHEST - - ref_tag = get_policy_name(ref_policy) - sym_tag = get_policy_name(symbol_policy) - ucs_tag = get_policy_name(ucs_policy) - pyfpe_tag = get_policy_name(pyfpe_policy) - overall_tag = get_policy_name(min(symbol_policy, ref_policy, ucs_policy, + pyfpe_policy = policies.lowest + + ref_tag = get_policy_name(policies, ref_policy) + sym_tag = get_policy_name(policies, symbol_policy) + ucs_tag = get_policy_name(policies, ucs_policy) + pyfpe_tag = get_policy_name(policies, pyfpe_policy) + overall_tag = get_policy_name(policies, + min(symbol_policy, ref_policy, ucs_policy, pyfpe_policy)) return WheelAbIInfo(overall_tag, external_refs, ref_tag, versioned_symbols, diff --git a/tests/integration/test_policy_files.py b/tests/integration/test_policy_files.py index b0e5f650..7d9652d3 100644 --- a/tests/integration/test_policy_files.py +++ b/tests/integration/test_policy_files.py @@ -1,22 +1,23 @@ from jsonschema import validate from auditwheel.policy import (load_policies, _load_policy_schema, versioned_symbols_policy, - POLICY_PRIORITY_HIGHEST, - POLICY_PRIORITY_LOWEST) + Platform, get_policy_platform) -def test_policy(): - policy = load_policies() +def test_manylinux_policy(): + policies = load_policies(Platform.Manylinux) policy_schema = _load_policy_schema() - validate(policy, policy_schema) + validate(policies.policies, policy_schema) def test_policy_checks_glibc(): + policies = load_policies(get_policy_platform()) + policy = versioned_symbols_policy({"some_library.so": {"GLIBC_2.17"}}) - assert policy > POLICY_PRIORITY_LOWEST + assert policy > policies.lowest policy = versioned_symbols_policy({"some_library.so": {"GLIBC_999"}}) - assert policy == POLICY_PRIORITY_LOWEST + assert policy == policies.lowest policy = versioned_symbols_policy({"some_library.so": {"OPENSSL_1_1_0"}}) - assert policy == POLICY_PRIORITY_HIGHEST + assert policy == policies.highest policy = versioned_symbols_policy({"some_library.so": {"IAMALIBRARY"}}) - assert policy == POLICY_PRIORITY_HIGHEST + assert policy == policies.highest diff --git a/tests/unit/test_policy.py b/tests/unit/test_policy.py index 6694ad76..58f4c178 100644 --- a/tests/unit/test_policy.py +++ b/tests/unit/test_policy.py @@ -3,7 +3,8 @@ import pytest from auditwheel.policy import get_arch_name, get_policy_name, \ - get_priority_by_name, get_replace_platforms, _validate_pep600_compliance + get_priority_by_name, get_replace_platforms, _validate_pep600_compliance, \ + load_policies, Platform, PlatformPolicies @patch("auditwheel.policy._platform_module.machine") @@ -100,45 +101,57 @@ class TestPolicyAccess: def test_get_by_priority(self): _arch = get_arch_name() - assert get_policy_name(65) == f'manylinux_2_27_{_arch}' - assert get_policy_name(70) == f'manylinux_2_24_{_arch}' - assert get_policy_name(80) == f'manylinux_2_17_{_arch}' + policies = load_policies(Platform.Manylinux) + assert get_policy_name(policies, 65) == f'manylinux_2_27_{_arch}' + assert get_policy_name(policies, 70) == f'manylinux_2_24_{_arch}' + assert get_policy_name(policies, 80) == f'manylinux_2_17_{_arch}' if _arch in {'x86_64', 'i686'}: - assert get_policy_name(90) == f'manylinux_2_12_{_arch}' - assert get_policy_name(100) == f'manylinux_2_5_{_arch}' - assert get_policy_name(0) == f'linux_{_arch}' + assert get_policy_name(policies, 90) == f'manylinux_2_12_{_arch}' + assert get_policy_name(policies, 100) == f'manylinux_2_5_{_arch}' + assert get_policy_name(policies, 0) == f'linux_{_arch}' def test_get_by_priority_missing(self): - assert get_policy_name(101) is None + policies = load_policies(Platform.Manylinux) + assert get_policy_name(policies, 101) is None - @patch("auditwheel.policy._POLICIES", [ - {"name": "duplicate", "priority": 0}, - {"name": "duplicate", "priority": 0}, - ]) def test_get_by_priority_duplicate(self): + policies = PlatformPolicies( + Platform.Manylinux, + policies=[ + {"name": "duplicate", "priority": 0}, + {"name": "duplicate", "priority": 0}, + ], + lowest=0, + highest=0) with pytest.raises(RuntimeError): - get_policy_name(0) + get_policy_name(policies, 0) def test_get_by_name(self): _arch = get_arch_name() - assert get_priority_by_name(f"manylinux_2_27_{_arch}") == 65 - assert get_priority_by_name(f"manylinux_2_24_{_arch}") == 70 - assert get_priority_by_name(f"manylinux2014_{_arch}") == 80 - assert get_priority_by_name(f"manylinux_2_17_{_arch}") == 80 + policies = load_policies(Platform.Manylinux) + assert get_priority_by_name(policies, f"manylinux_2_27_{_arch}") == 65 + assert get_priority_by_name(policies, f"manylinux_2_24_{_arch}") == 70 + assert get_priority_by_name(policies, f"manylinux2014_{_arch}") == 80 + assert get_priority_by_name(policies, f"manylinux_2_17_{_arch}") == 80 if _arch in {'x86_64', 'i686'}: - assert get_priority_by_name(f"manylinux2010_{_arch}") == 90 - assert get_priority_by_name(f"manylinux_2_12_{_arch}") == 90 - assert get_priority_by_name(f"manylinux1_{_arch}") == 100 - assert get_priority_by_name(f"manylinux_2_5_{_arch}") == 100 + assert get_priority_by_name(policies, f"manylinux2010_{_arch}") == 90 + assert get_priority_by_name(policies, f"manylinux_2_12_{_arch}") == 90 + assert get_priority_by_name(policies, f"manylinux1_{_arch}") == 100 + assert get_priority_by_name(policies, f"manylinux_2_5_{_arch}") == 100 def test_get_by_name_missing(self): - assert get_priority_by_name("nosuchpolicy") is None + policies = load_policies(Platform.Manylinux) + assert get_priority_by_name(policies, "nosuchpolicy") is None - @patch("auditwheel.policy._POLICIES", [ - {"name": "duplicate", "priority": 0}, - {"name": "duplicate", "priority": 0}, - ]) def test_get_by_name_duplicate(self): + policies = PlatformPolicies( + Platform.Manylinux, + policies=[ + {"name": "duplicate", "priority": 0}, + {"name": "duplicate", "priority": 0}, + ], + lowest=0, + highest=0) with pytest.raises(RuntimeError): - get_priority_by_name("duplicate") + get_priority_by_name(policies, "duplicate")