Add support for musllinux

auditwheel/error.py

@@ -0,0 +1,6 @@
+class AuditwheelException(Exception):
+    pass
+
+
+class InvalidPlatform(AuditwheelException):
+    pass

auditwheel/main_repair.py

@@ -2,7 +2,7 @@
 
 from auditwheel.patcher import Patchelf
 from .policy import (load_policies, get_policy_by_name, get_policy_name,
-                     get_priority_by_name, POLICY_PRIORITY_HIGHEST)
+                     get_priority_by_name, get_policy_platform)
 from .tools import EnvironmentDefault
 import argparse
 import logging
@@ -12,20 +12,21 @@
 
 
 def configure_parser(sub_parsers):
-    policies = load_policies()
-    policy_names = [p['name'] for p in policies]
-    policy_names += [alias for p in policies for alias in p['aliases']]
+    policies = load_policies(get_policy_platform())
+    policy_names = [p['name'] for p in policies.policies]
+    policy_names += [alias for p in policies.policies
+                     for alias in p['aliases']]
     epilog = """PLATFORMS:
 These are the possible target platform tags, as specified by PEP 600.
 Note that old, pre-PEP 600 tags are still usable and are listed as aliases
 below.
 """
-    for p in policies:
+    for p in policies.policies:
         epilog += f"- {p['name']}"
         if len(p['aliases']) > 0:
             epilog += f" (aliased by {', '.join(p['aliases'])})"
         epilog += '\n'
-    highest_policy = get_policy_name(POLICY_PRIORITY_HIGHEST)
+    highest_policy = get_policy_name(policies, policies.highest)
     help = "Vendor in external shared library dependencies of a wheel."
     p = sub_parsers.add_parser(
         'repair', help=help, description=help, epilog=epilog,
@@ -92,17 +93,18 @@ def execute(args, p):
         logger.info('This does not look like a platform wheel')
         return 1
 
-    policy = get_policy_by_name(args.PLAT)
+    policies = load_policies(get_policy_platform())
+    policy = get_policy_by_name(policies, args.PLAT)
     reqd_tag = policy['priority']
 
-    if reqd_tag > get_priority_by_name(wheel_abi.sym_tag):
+    if reqd_tag > get_priority_by_name(policies, wheel_abi.sym_tag):
         msg = ('cannot repair "%s" to "%s" ABI because of the presence '
                'of too-recent versioned symbols. You\'ll need to compile '
                'the wheel on an older toolchain.' %
                (args.WHEEL_FILE, args.PLAT))
         p.error(msg)
 
-    if reqd_tag > get_priority_by_name(wheel_abi.ucs_tag):
+    if reqd_tag > get_priority_by_name(policies, wheel_abi.ucs_tag):
         msg = ('cannot repair "%s" to "%s" ABI because it was compiled '
                'against a UCS2 build of Python. You\'ll need to compile '
                'the wheel against a wide-unicode build of Python.' %
@@ -111,12 +113,12 @@ def execute(args, p):
 
     abis = [policy['name']] + policy['aliases']
     if not args.ONLY_PLAT:
-        if reqd_tag < get_priority_by_name(wheel_abi.overall_tag):
+        if reqd_tag < get_priority_by_name(policies, wheel_abi.overall_tag):
             logger.info(('Wheel is eligible for a higher priority tag. '
                          'You requested %s but I have found this wheel is '
                          'eligible for %s.'),
                         args.PLAT, wheel_abi.overall_tag)
-            higher_policy = get_policy_by_name(wheel_abi.overall_tag)
+            higher_policy = get_policy_by_name(policies, wheel_abi.overall_tag)
             abis = [higher_policy['name']] + higher_policy['aliases'] + abis
 
     patcher = Patchelf()

auditwheel/main_show.py

@@ -2,6 +2,8 @@
 
 from collections import OrderedDict
 
+from auditwheel.policy import get_policy_platform
+
 logger = logging.getLogger(__name__)
 
 
@@ -82,7 +84,9 @@ def execute(args, p):
                'by the wheel:')
         print(json.dumps(OrderedDict(sorted(libs.items())), indent=4))
 
-    for p in sorted(load_policies(), key=lambda p: p['priority']):
+    policies = load_policies(get_policy_platform())
+    for p in sorted(policies.policies,
+                    key=lambda p: p['priority']):
         if p['priority'] > get_priority_by_name(winfo.overall_tag):
             printp(('In order to achieve the tag platform tag "%s" '
                     'the following shared library dependencies '

auditwheel/musllinux.py

@@ -0,0 +1,65 @@
+import logging
+import pathlib
+import subprocess
+import re
+import sys
+from typing import NamedTuple
+
+from auditwheel.error import InvalidPlatform
+
+LOG = logging.getLogger(__name__)
+
+
+class MuslVersion(NamedTuple):
+    major: int
+    minor: int
+    patch: int
+
+
+def find_musl_libc() -> pathlib.Path:
+    try:
+        ldd = subprocess.run(["ldd", "/bin/ls"],
+                             text=True,
+                             capture_output=True)
+    except subprocess.CalledProcessError:
+        LOG.error("Failed to determine libc version", exc_info=True)
+        raise
+
+    match = re.search(
+        r"libc\.musl-(?P<platform>\w+)\.so.1 "  # TODO drop the platform
+        r"=> (?P<path>[/\-\w.]+)",
+        ldd.stdout)
+
+    if not match:
+        raise InvalidPlatform
+
+    return pathlib.Path(match.group("path"))
+
+
+def get_musl_version(ld_path: pathlib.Path) -> MuslVersion:
+    try:
+        ld = subprocess.run([ld_path], text=True, capture_output=True)
+    except subprocess.CalledProcessError:
+        LOG.error("Failed to determine musl version", exc_info=True)
+        raise
+
+    match = re.search(
+        r"Version "
+        r"(?P<major>\d)."
+        r"(?P<minor>\d)."
+        r"(?P<patch>\d)",
+        ld.stderr)
+    if not match:
+        raise InvalidPlatform
+
+    return MuslVersion(
+        int(match.group("major")),
+        int(match.group("minor")),
+        int(match.group("patch")))
+
+
+if __name__ == "__main__":
+    libc_path = find_musl_libc()
+    version = get_musl_version(libc_path)
+    print(f"Found musl version {version} in {libc_path}")
+    wheel_path = pathlib.Path(sys.argv[1])

auditwheel/policy/__init__.py

@@ -1,14 +1,35 @@
+import functools
+import pathlib
 import sys
 import json
 import platform as _platform_module
+import typing
 from collections import defaultdict
+from enum import Enum
 from typing import Dict, List, Optional, Set
 from os.path import join, dirname, abspath
 import logging
 
+from ..error import InvalidPlatform
+from ..musllinux import find_musl_libc
+
+_HERE = pathlib.Path(__file__).parent
 
 logger = logging.getLogger(__name__)
 
+
+class Platform(Enum):
+    Manylinux = 1,
+    Musllinux = 2
+
+
+class PlatformPolicies(typing.NamedTuple):
+    platform: Platform
+    policies: List
+    lowest: int
+    highest: int
+
+
 # https://docs.python.org/3/library/platform.html#platform.architecture
 bits = 8 * (8 if sys.maxsize > 2 ** 32 else 4)
 
@@ -21,6 +42,16 @@ def get_arch_name() -> str:
         return {64: 'x86_64', 32: 'i686'}[bits]
 
 
+def get_policy_platform() -> Platform:
+    try:
+        find_musl_libc()
+        logger.debug("Detected musl libc")
+        return Platform.Musllinux
+    except InvalidPlatform:
+        logger.debug("Falling back to GNU libc")
+        return Platform.Manylinux
+
+
 _ARCH_NAME = get_arch_name()
 
 
@@ -56,25 +87,40 @@ def _validate_pep600_compliance(policies) -> None:
             symbol_versions[arch] = symbol_versions_arch
 
 
-with open(join(dirname(abspath(__file__)), 'manylinux-policy.json')) as f:
-    _POLICIES = []
-    _policies_temp = json.load(f)
-    _validate_pep600_compliance(_policies_temp)
+def _load_policy(path: pathlib.Path, platform: Platform):
+    _policies_temp = json.loads(path.read_text())
+    _policies = []
+    if platform == Platform.Manylinux:
+        _validate_pep600_compliance(_policies_temp)
     for _p in _policies_temp:
-        if _ARCH_NAME in _p['symbol_versions'].keys() or _p['name'] == 'linux':
+        arch = _p['symbol_versions'].keys()
+        if _ARCH_NAME in arch or _p['name'] == 'linux':
             if _p['name'] != 'linux':
                 _p['symbol_versions'] = _p['symbol_versions'][_ARCH_NAME]
             _p['name'] = _p['name'] + '_' + _ARCH_NAME
             _p['aliases'] = [alias + '_' + _ARCH_NAME
                              for alias in _p['aliases']]
-            _POLICIES.append(_p)
+            _policies.append(_p)
 
-POLICY_PRIORITY_HIGHEST = max(p['priority'] for p in _POLICIES)
-POLICY_PRIORITY_LOWEST = min(p['priority'] for p in _POLICIES)
+    priority_highest = max(p['priority'] for p in _policies)
+    priority_lowest = min(p['priority'] for p in _policies)
 
+    return PlatformPolicies(platform=platform,
+                            policies=_policies,
+                            lowest=priority_lowest,
+                            highest=priority_highest)
 
-def load_policies():
-    return _POLICIES
+
+@functools.lru_cache()
+def load_policies(policy: Platform):
+    if policy == Platform.Manylinux:
+        return _load_policy(_HERE / "manylinux-policy.json",
+                            Platform.Manylinux)
+    elif policy == Platform.Musllinux:
+        return _load_policy(_HERE / "musllinux-policy.json",
+                            Platform.Musllinux)
+    else:
+        raise ValueError("Invalid policy")
 
 
 def _load_policy_schema():
@@ -83,8 +129,10 @@ def _load_policy_schema():
     return schema
 
 
-def get_policy_by_name(name: str) -> Optional[Dict]:
-    matches = [p for p in _POLICIES
+def get_policy_by_name(
+        policies: PlatformPolicies,
+        name: str) -> Optional[Dict]:
+    matches = [p for p in policies.policies
                if p['name'] == name or name in p['aliases']]
     if len(matches) == 0:
         return None
@@ -93,17 +141,22 @@ def get_policy_by_name(name: str) -> Optional[Dict]:
     return matches[0]
 
 
-def get_policy_name(priority: int) -> Optional[str]:
-    matches = [p['name'] for p in _POLICIES if p['priority'] == priority]
+def get_policy_name(
+        policies: PlatformPolicies,
+        priority: int) -> Optional[str]:
+    matches = [p['name'] for p in policies.policies
+               if p['priority'] == priority]
     if len(matches) == 0:
         return None
     if len(matches) > 1:
         raise RuntimeError('Internal error. priorities should be unique')
     return matches[0]
 
 
-def get_priority_by_name(name: str) -> Optional[int]:
-    policy = get_policy_by_name(name)
+def get_priority_by_name(
+        policies: PlatformPolicies,
+        name: str) -> Optional[int]:
+    policy = get_policy_by_name(policies, name)
     return None if policy is None else policy['priority']
 
 
@@ -132,5 +185,4 @@ def get_replace_platforms(name: str) -> List[str]:
 from .versioned_symbols import versioned_symbols_policy  # noqa
 
 __all__ = ['lddtree_external_references', 'versioned_symbols_policy',
-           'load_policies', 'POLICY_PRIORITY_HIGHEST',
-           'POLICY_PRIORITY_LOWEST']
+           'load_policies']

auditwheel/policy/external_references.py

@@ -3,7 +3,7 @@
 from typing import Dict, Set, Any, Generator
 
 from ..elfutils import is_subdir
-from . import load_policies
+from . import load_policies, get_policy_platform
 
 log = logging.getLogger(__name__)
 LIBPYTHON_RE = re.compile(r'^libpython\d\.\dm?.so(.\d)*$')
@@ -12,7 +12,7 @@
 def lddtree_external_references(lddtree: Dict, wheel_path: str) -> Dict:
     # XXX: Document the lddtree structure, or put it in something
     # more stable than a big nested dict
-    policies = load_policies()
+    policies = load_policies(get_policy_platform())
 
     def filter_libs(libs: Set[str],
                     whitelist: Set[str]) -> Generator[str, None, None]:
@@ -44,7 +44,7 @@ def get_req_external(libs: Set[str], whitelist: Set[str]) -> Set[str]:
         return reqs
 
     ret = {}  # type: Dict[str, Dict[str, Any]]
-    for p in policies:
+    for p in policies.policies:
         needed_external_libs = set()  # type: Set[str]
 
         if not (p['name'] == 'linux' and p['priority'] == 0):

auditwheel/policy/musllinux-policy.json

@@ -0,0 +1,22 @@
+[
+  {"name": "musllinux_1_1",
+    "aliases": [],
+    "priority": 100,
+    "symbol_versions": {
+      "i686": {
+      },
+      "x86_64": {
+      },
+      "aarch64": {
+      },
+      "ppc64le": {
+      },
+      "s390x": {
+      },
+      "armv7l": {
+      }
+    },
+    "lib_whitelist": [
+      "libc.so", "libz.so"
+    ]}
+]

auditwheel/policy/versioned_symbols.py

@@ -1,7 +1,7 @@
 import logging
 from typing import Dict, List, Set
 
-from . import load_policies
+from . import load_policies, get_policy_platform
 
 log = logging.getLogger(__name__)
 
@@ -25,7 +25,8 @@ def policy_is_satisfied(policy_name: str,
             sym_name, _, _ = symbol.partition("_")
             required_vers.setdefault(sym_name, set()).add(symbol)
     matching_policies = []  # type: List[int]
-    for p in load_policies():
+    policies = load_policies(get_policy_platform())
+    for p in policies.policies:
         policy_sym_vers = {
             sym_name: {sym_name + '_' + version for version in versions}
             for sym_name, versions in p['symbol_versions'].items()