From e9915cd1a76634d6530873eae96e1a557dd9405d Mon Sep 17 00:00:00 2001
From: ch8matt <g.matthieu49@gmail.com>
Date: Tue, 27 Jan 2026 14:37:15 +0000
Subject: [PATCH] Fix(PypiMetaAnalyzer): Compliant with the Simple Repository
 API (PEP 503|691) [#5735]

---
 pom.xml                                       |   6 +
 .../repositories/AbstractMetaAnalyzer.java    |  35 +-
 .../tasks/repositories/PypiMetaAnalyzer.java  | 302 +++++++++++++++---
 .../repositories/PypiMetaAnalyzerTest.java    | 148 ++++++++-
 4 files changed, 432 insertions(+), 59 deletions(-)

diff --git a/pom.xml b/pom.xml
index 37c202cae1..e08c240771 100644
--- a/pom.xml
+++ b/pom.xml
@@ -181,6 +181,12 @@
             <artifactId>cyclonedx-core-java</artifactId>
             <version>${lib.cyclonedx-java.version}</version>
         </dependency>
+        <!-- HTML parsing -->
+         <dependency>
+             <groupId>org.jsoup</groupId>
+             <artifactId>jsoup</artifactId>
+             <version>1.22.1</version>
+        </dependency>
 
         <!-- org.json
         This was previously transitively included with Unirest. However, Unirest v3.x removed reliance on org.json
diff --git a/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java
index 381e10e282..af5d1eb8f5 100644
--- a/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java
+++ b/src/main/java/org/dependencytrack/tasks/repositories/AbstractMetaAnalyzer.java
@@ -26,7 +26,6 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.client.utils.URIBuilder;
 import org.dependencytrack.common.HttpClientPool;
 import org.dependencytrack.model.Component;
@@ -111,22 +110,40 @@ protected void handleRequestException(final Logger logger, final Exception e) {
                 .level(NotificationLevel.ERROR));
     }
 
-    protected CloseableHttpResponse processHttpRequest(String url) throws IOException {
+    protected CloseableHttpResponse processHttpRequest(final String url) throws IOException {
+        return processHttpRequest(url, "application/json");
+    }
+
+    protected CloseableHttpResponse processHttpRequest(final String url, final String acceptValue) throws IOException {
         final Logger logger = Logger.getLogger(getClass());
         try {
-            URIBuilder uriBuilder = new URIBuilder(url);
-            final HttpUriRequest request = new HttpGet(uriBuilder.build().toString());
-            request.addHeader("accept", "application/json");
-            if (!StringUtils.isEmpty(username)) { // for some reason there is a testcase for password being null
-                request.addHeader("Authorization", HttpUtil.basicAuthHeaderValue(username, password));
-            } else if (!StringUtils.isEmpty(password)) {
-                request.addHeader("Authorization", "Bearer " + password);
+            final HttpGet request = new HttpGet(new URIBuilder(url).build());
+
+            if (acceptValue != null && !acceptValue.isBlank()) {
+                request.addHeader("Accept", acceptValue);
             }
+
+            final String authHeader = buildAuthHeaderValue();
+            if (authHeader != null) {
+                request.addHeader("Authorization", authHeader);
+            }
+
             return HttpClientPool.getClient().execute(request);
+
         } catch (URISyntaxException ex) {
             handleRequestException(logger, ex);
             return null;
         }
     }
 
+    private String buildAuthHeaderValue() {
+        if (StringUtils.isNotEmpty(username)) { // for some reason there is a testcase for password being null
+            return HttpUtil.basicAuthHeaderValue(username, password);
+        }
+        if (StringUtils.isNotEmpty(password)) {
+            return "Bearer " + password;
+        }
+        return null;
+    }
 }
+
diff --git a/src/main/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzer.java b/src/main/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzer.java
index 115653ff97..003a59ad2e 100644
--- a/src/main/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzer.java
+++ b/src/main/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzer.java
@@ -20,32 +20,68 @@
 
 import alpine.common.logging.Logger;
 import com.github.packageurl.PackageURL;
+import org.apache.http.HttpHeaders;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.entity.ContentType;
 import org.apache.http.util.EntityUtils;
+import org.apache.maven.artifact.versioning.ComparableVersion;
 import org.dependencytrack.exception.MetaAnalyzerException;
 import org.dependencytrack.model.Component;
 import org.dependencytrack.model.RepositoryType;
 import org.json.JSONArray;
 import org.json.JSONObject;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
 
 import java.io.IOException;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.format.DateTimeParseException;
+import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
- * An IMetaAnalyzer implementation that supports Pypi.
+ * An IMetaAnalyzer implementation that supports Python package repositories.
+ * This analyzer uses the standardized Simple Repository API.
  *
- * @author Steve Springett
- * @since 3.4.0
+ * @author ch8matt
  */
+
 public class PypiMetaAnalyzer extends AbstractMetaAnalyzer {
 
     private static final Logger LOGGER = Logger.getLogger(PypiMetaAnalyzer.class);
+
     private static final String DEFAULT_BASE_URL = "https://pypi.org";
-    private static final String API_URL = "/pypi/%s/json";
+
+    /**
+     * Prefer JSON (PEP 691) but accept HTML (PEP 503) as a fallback.
+     */
+    private static final String ACCEPT_SIMPLE_API = String.join(", ",
+            "application/vnd.pypi.simple.v1+json",
+            "application/vnd.pypi.simple.v1+html;q=0.2",
+            "text/html;q=0.01");
+
+    /**
+     * PEP 503 normalization: lowercase and replace runs of '.', '_' and '-' with a single '-'.
+     */
+    private static final Pattern PEP503_NORMALIZE = Pattern.compile("[-_.]+", Pattern.CASE_INSENSITIVE);
+
+    /**
+     * Extracts a version from a distribution filename by taking the segment that starts with the first '-' followed by a digit and continues up to the next '-'.
+     */
+    private static final Pattern VERSION_FROM_FILENAME = Pattern.compile("-(\\d[^-]*?)(?:-|\\.(?:tar\\.gz|tar\\.bz2|tar\\.xz|tgz|zip|whl|tar))$", Pattern.CASE_INSENSITIVE);
 
     PypiMetaAnalyzer() {
         this.baseUrl = DEFAULT_BASE_URL;
@@ -70,43 +106,223 @@ public RepositoryType supportedRepositoryType() {
      */
     public MetaModel analyze(final Component component) {
         final MetaModel meta = new MetaModel(component);
-        if (component.getPurl() != null) {
-            final String url = String.format(baseUrl + API_URL, urlEncode(component.getPurl().getName()));
-            try (final CloseableHttpResponse response = processHttpRequest(url)) {
-                if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
-                    if (response.getEntity() != null) {
-                        String stringResponse = EntityUtils.toString(response.getEntity());
-                        JSONObject jsonObject = new JSONObject(stringResponse);
-                        final JSONObject info = jsonObject.getJSONObject("info");
-                        final String latest = info.optString("version", null);
-                        if (latest != null) {
-                            meta.setLatestVersion(latest);
-                            final JSONObject releases = jsonObject.getJSONObject("releases");
-                            final JSONArray latestArray = releases.getJSONArray(latest);
-                            if (latestArray.length() > 0) {
-                                final JSONObject release = latestArray.getJSONObject(0);
-                                final String updateTime = release.optString("upload_time", null);
-                                if (updateTime != null) {
-                                    final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
-                                    try {
-                                        final Date published = dateFormat.parse(updateTime);
-                                        meta.setPublishedTimestamp(published);
-                                    } catch (ParseException e) {
-                                        LOGGER.warn("An error occurred while parsing upload time", e);
-                                    }
-                                }
-                            }
-                        }
-                    }
-                } else {
-                    handleUnexpectedHttpResponse(LOGGER, url, response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(), component);
-                }
-            } catch (IOException e) {
-                handleRequestException(LOGGER, e);
-            } catch (Exception ex) {
-                throw new MetaAnalyzerException(ex);
+        if (component.getPurl() == null) {
+            return meta;
+        }
+
+        final String projectName = component.getPurl().getName();
+        if (projectName == null || projectName.isEmpty()) {
+            return meta;
+        }
+
+        final String normalizedName = normalizePep503(projectName);
+
+        final String simpleBaseUrl = baseUrl.endsWith("/simple")
+                ? baseUrl
+                : baseUrl + "/simple";
+
+        final String url = String.format(
+                Locale.ROOT,
+                simpleBaseUrl + "/%s/",
+                urlEncode(normalizedName)
+        );
+
+        try (final CloseableHttpResponse response = processHttpRequest(url, ACCEPT_SIMPLE_API)) {
+            if (response == null) {
+                return meta;
             }
+
+            final int status = response.getStatusLine().getStatusCode();
+            if (status != HttpStatus.SC_OK) {
+                handleUnexpectedHttpResponse(LOGGER, url, status, response.getStatusLine().getReasonPhrase(), component);
+                return meta;
+            }
+
+            final var entity = response.getEntity();
+            if (entity == null) {
+                return meta;
+            }
+
+            final String body = EntityUtils.toString(entity);
+            final ContentType ct = ContentType.getOrDefault(entity);
+            final String mimeType = ct.getMimeType() != null ? ct.getMimeType().toLowerCase(Locale.ROOT) : "";
+            final String fullContentType = response.getFirstHeader(HttpHeaders.CONTENT_TYPE) != null
+                    ? response.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue().toLowerCase(Locale.ROOT)
+                    : mimeType;
+
+            // PEP 691 uses "application/vnd.pypi.simple.v1+json".
+            // Some repositories may return application/json with the same structure.
+            if (fullContentType.contains("json")) {
+                parseSimpleJson(body, meta);
+            } else if (fullContentType.contains("html")) {
+                parseSimpleHtml(body, meta);
+            } else {
+                // Only parse what the PEPs specify.
+                LOGGER.warn("Unsupported Content-Type for PyPI Simple API response: " + fullContentType);
+            }
+
+        } catch (IOException e) {
+            handleRequestException(LOGGER, e);
+        } catch (Exception ex) {
+            throw new MetaAnalyzerException(ex);
         }
+
         return meta;
     }
-}
\ No newline at end of file
+
+    private static String normalizePep503(final String name) {
+        final String lower = name.toLowerCase(Locale.ROOT);
+        return PEP503_NORMALIZE.matcher(lower).replaceAll("-");
+    }
+
+    private static void parseSimpleJson(final String body, final MetaModel meta) {
+        final JSONObject root = new JSONObject(body);
+
+        final JSONArray files = root.optJSONArray("files");
+        final JSONArray versionsArray = root.optJSONArray("versions");
+
+        final Set<String> allVersions = new HashSet<>();
+        final Set<String> nonYankedVersions = new HashSet<>();
+        final Map<String, Date> earliestUploadByVersion = new HashMap<>();
+
+        // Collect versions from the PEP 700 field, if present.
+        if (versionsArray != null) {
+            for (int i = 0; i < versionsArray.length(); i++) {
+                final String v = versionsArray.optString(i, null);
+                if (v != null && !v.isEmpty()) {
+                    allVersions.add(v);
+                }
+            }
+        }
+
+        // Collect versions and yanked/upload-time information from files.
+        if (files != null) {
+            for (int i = 0; i < files.length(); i++) {
+                final JSONObject file = files.optJSONObject(i);
+                if (file == null) {
+                    continue;
+                }
+                final String filename = file.optString("filename", null);
+                if (filename == null) {
+                    continue;
+                }
+                final String v = extractVersionFromFilename(filename);
+                if (v == null) {
+                    continue;
+                }
+
+                allVersions.add(v);
+
+                final boolean yanked = file.optBoolean("yanked", false);
+                if (!yanked) {
+                    nonYankedVersions.add(v);
+                }
+
+                final String uploadTime = file.optString("upload-time", null);
+                final Date parsedUpload = parseIsoInstant(uploadTime);
+                if (parsedUpload != null) {
+                    earliestUploadByVersion.merge(v, parsedUpload, (a, b) -> a.before(b) ? a : b);
+                }
+            }
+        }
+
+        if (allVersions.isEmpty()) {
+            return;
+        }
+
+        // Prefer non-yanked versions unless all available versions are yanked.
+        final List<String> candidates = new ArrayList<>();
+        if (!nonYankedVersions.isEmpty()) {
+            candidates.addAll(nonYankedVersions);
+        } else {
+            candidates.addAll(allVersions);
+        }
+
+        final String latest = selectLatestVersion(candidates);
+        if (latest != null) {
+            meta.setLatestVersion(latest);
+            final Date published = earliestUploadByVersion.get(latest);
+            if (published != null) {
+                meta.setPublishedTimestamp(published);
+            }
+        }
+    }
+
+    private static void parseSimpleHtml(final String body, final MetaModel meta) {
+        final Document doc = Jsoup.parse(body);
+        final Elements links = doc.select("a[href]");
+
+        final Set<String> allVersions = new HashSet<>();
+        final Set<String> nonYankedVersions = new HashSet<>();
+
+        for (final Element link : links) {
+            // PEP 503: anchor text SHOULD be the filename.
+            final String filename = link.text();
+            final String v = extractVersionFromFilename(filename);
+            if (v == null) {
+                continue;
+            }
+
+            allVersions.add(v);
+
+            // PEP 592 yanking: data-yanked attribute indicates yanked.
+            final boolean yanked = link.hasAttr("data-yanked");
+            if (!yanked) {
+                nonYankedVersions.add(v);
+            }
+        }
+
+        if (allVersions.isEmpty()) {
+            return;
+        }
+
+        final List<String> candidates = new ArrayList<>();
+        if (!nonYankedVersions.isEmpty()) {
+            candidates.addAll(nonYankedVersions);
+        } else {
+            candidates.addAll(allVersions);
+        }
+
+        final String latest = selectLatestVersion(candidates);
+        if (latest != null) {
+            meta.setLatestVersion(latest);
+        }
+        // PEP 503: provides no standardized timestamp fields.
+    }
+
+    private static String extractVersionFromFilename(final String filename) {
+        if (filename == null) {
+            return null;
+        }
+        final Matcher m = VERSION_FROM_FILENAME.matcher(filename);
+        if (!m.find()) {
+            return null;
+        }
+        final String v = m.group(1);
+        return (v == null || v.isEmpty()) ? null : v;
+    }
+
+    private static String selectLatestVersion(final List<String> versions) {
+        if (versions == null || versions.isEmpty()) {
+            return null;
+        }
+
+        return versions.stream()
+                .filter(Objects::nonNull)
+                .max(Comparator.comparing(ComparableVersion::new))
+                .orElse(null);
+    }
+
+    private static Date parseIsoInstant(final String value) {
+        if (value == null || value.isEmpty()) {
+            return null;
+        }
+        try {
+            return Date.from(Instant.parse(value));
+        } catch (DateTimeParseException ex) {
+            return null;
+        }
+    }
+}
+
+
diff --git a/src/test/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzerTest.java b/src/test/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzerTest.java
index 425473ac71..b19afeb709 100644
--- a/src/test/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzerTest.java
+++ b/src/test/java/org/dependencytrack/tasks/repositories/PypiMetaAnalyzerTest.java
@@ -19,23 +19,157 @@
 package org.dependencytrack.tasks.repositories;
 
 import com.github.packageurl.PackageURL;
+import org.apache.http.HttpHeaders;
 import org.dependencytrack.model.Component;
 import org.dependencytrack.model.RepositoryType;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mockserver.client.MockServerClient;
+import org.mockserver.integration.ClientAndServer;
+
+import java.text.SimpleDateFormat;
+
+import static org.mockserver.model.HttpRequest.request;
+import static org.mockserver.model.HttpResponse.response;
 
 class PypiMetaAnalyzerTest {
 
+    private static ClientAndServer mockServer;
+
+    @BeforeAll
+    static void beforeAll() {
+        mockServer = ClientAndServer.startClientAndServer(0);
+    }
+
+    @AfterAll
+    static void afterAll() {
+        mockServer.stop();
+    }
+
+    @BeforeEach
+    void beforeEach() {
+        mockServer.reset();
+    }
+
     @Test
-    void testAnalyzer() throws Exception {
-        Component component = new Component();
-        component.setPurl(new PackageURL("pkg:pypi/Flask@1.0.0"));
+    void testAnalyzerJsonPrefersNonYankedAndSetsUploadTime() throws Exception {
+        final Component component = new Component();
+        component.setPurl(new PackageURL("pkg:pypi/Example_Pkg@0.0.1"));
+
+        final PypiMetaAnalyzer analyzer = new PypiMetaAnalyzer();
+        analyzer.setRepositoryId("1");
+        analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", mockServer.getPort()));
 
-        PypiMetaAnalyzer analyzer = new PypiMetaAnalyzer();
         Assertions.assertTrue(analyzer.isApplicable(component));
         Assertions.assertEquals(RepositoryType.PYPI, analyzer.supportedRepositoryType());
-        MetaModel metaModel = analyzer.analyze(component);
-        Assertions.assertNotNull(metaModel.getLatestVersion());
-        Assertions.assertNotNull(metaModel.getPublishedTimestamp());
+
+        final String body = "{" +
+                "\"meta\":{\"api-version\":\"1.1\"}," +
+                "\"name\":\"example-pkg\"," +
+                "\"versions\":[\"1.0.0\",\"2.0.0\",\"2.1.0\"]," +
+                "\"files\":[" +
+                "{\"filename\":\"example_pkg-1.0.0.tar.gz\",\"yanked\":false,\"upload-time\":\"2023-01-01T10:00:00Z\"}," +
+                "{\"filename\":\"example_pkg-2.0.0.tar.gz\",\"yanked\":true,\"upload-time\":\"2024-01-01T10:00:00Z\"}," +
+                "{\"filename\":\"example_pkg-2.1.0.tar.gz\",\"yanked\":false,\"upload-time\":\"2025-01-01T10:00:00Z\"}" +
+                "]" +
+                "}";
+
+        @SuppressWarnings("resource")
+        final MockServerClient mockClient = new MockServerClient("localhost", mockServer.getPort());
+        mockClient.when(
+                        request()
+                                .withMethod("GET")
+                                // PEP 503 normalization: Example_Pkg -> example-pkg
+                                .withPath("/simple/example-pkg/")
+                                .withHeader(HttpHeaders.ACCEPT)
+                )
+                .respond(
+                        response()
+                                .withStatusCode(200)
+                                .withHeader(HttpHeaders.CONTENT_TYPE, "application/vnd.pypi.simple.v1+json")
+                                .withBody(body)
+                );
+
+        final MetaModel meta = analyzer.analyze(component);
+        Assertions.assertEquals("2.1.0", meta.getLatestVersion());
+        Assertions.assertEquals(
+                new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX").parse("2025-01-01T10:00:00Z"),
+                meta.getPublishedTimestamp());
+    }
+
+    @Test
+    void testAnalyzerJsonAllYankedStillReturnsLatestVersion() throws Exception {
+        final Component component = new Component();
+        component.setPurl(new PackageURL("pkg:pypi/example-pkg@0.0.1"));
+
+        final PypiMetaAnalyzer analyzer = new PypiMetaAnalyzer();
+        analyzer.setRepositoryId("1");
+        analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", mockServer.getPort()));
+
+        final String body = "{" +
+                "\"meta\":{\"api-version\":\"1.1\"}," +
+                "\"name\":\"example-pkg\"," +
+                "\"versions\":[\"1.0.0\",\"2.0.0\"]," +
+                "\"files\":[" +
+                "{\"filename\":\"example_pkg-1.0.0.tar.gz\",\"yanked\":true}," +
+                "{\"filename\":\"example_pkg-2.0.0.tar.gz\",\"yanked\":true}" +
+                "]" +
+                "}";
+
+        @SuppressWarnings("resource")
+        final MockServerClient mockClient = new MockServerClient("localhost", mockServer.getPort());
+        mockClient.when(
+                        request()
+                                .withMethod("GET")
+                                .withPath("/simple/example-pkg/")
+                )
+                .respond(
+                        response()
+                                .withStatusCode(200)
+                                .withHeader(HttpHeaders.CONTENT_TYPE, "application/vnd.pypi.simple.v1+json")
+                                .withBody(body)
+                );
+
+        final MetaModel meta = analyzer.analyze(component);
+        Assertions.assertEquals("2.0.0", meta.getLatestVersion());
+        Assertions.assertNull(meta.getPublishedTimestamp());
+    }
+
+    @Test
+    void testAnalyzerHtmlPrefersNonYankedAndLeavesTimestampNull() throws Exception {
+        final Component component = new Component();
+        component.setPurl(new PackageURL("pkg:pypi/example-pkg@0.0.1"));
+
+        final PypiMetaAnalyzer analyzer = new PypiMetaAnalyzer();
+        analyzer.setRepositoryId("1");
+        analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", mockServer.getPort()));
+
+        final String html = "<html><body>" +
+                "<a href=\"/packages/example_pkg-1.0.0.tar.gz\">example_pkg-1.0.0.tar.gz</a>" +
+                "<a href=\"/packages/example_pkg-2.0.0.tar.gz\" data-yanked>example_pkg-2.0.0.tar.gz</a>" +
+                "<a href=\"/packages/example_pkg-2.1.0.tar.gz\">example_pkg-2.1.0.tar.gz</a>" +
+                "</body></html>";
+
+        @SuppressWarnings("resource")
+        final MockServerClient mockClient = new MockServerClient("localhost", mockServer.getPort());
+        mockClient.when(
+                        request()
+                                .withMethod("GET")
+                                .withPath("/simple/example-pkg/")
+                )
+                .respond(
+                        response()
+                                .withStatusCode(200)
+                                .withHeader(HttpHeaders.CONTENT_TYPE, "text/html; charset=utf-8")
+                                .withBody(html)
+                );
+
+        final MetaModel meta = analyzer.analyze(component);
+        Assertions.assertEquals("2.1.0", meta.getLatestVersion());
+        Assertions.assertNull(meta.getPublishedTimestamp());
     }
 }
+