Integrate BugsInPy

.github/workflows/lint.yml

@@ -11,10 +11,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4.3.0
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
 
       - name: Run linter
-        uses: psf/black@stable
+        uses: psf/black@87928e6d6761a4a6d22250e1fee5601b3998086e # v26.5.1
         with:
           options: "--check --verbose"
           src: ${{ matrix.src-dir }}

.github/workflows/tests.yml

@@ -5,11 +5,13 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.3.0
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+        with:
+            submodules: recursive
 
-      - uses: jlumbroso/free-disk-space@v1.3.1
+      - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
 
-      - uses: actions/setup-python@v5.6.0
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
           python-version: '3.13'
 
@@ -22,7 +24,7 @@ jobs:
           python -m poetry config virtualenvs.in-project true
 
       - name: Cache the virtualenv
-        uses: actions/cache@v4.2.4
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
         with:
           path: ./.venv
           key: ${{ runner.os }}-venv-${{ hashFiles('**/poetry.lock') }}
@@ -32,12 +34,10 @@ jobs:
           python -m poetry install
 
       - name: Setup Perl
-        uses: shogo82148/actions-setup-perl@v1.35.0
-        with:
-          perl-version: "5.40.1"
+        uses: shogo82148/actions-setup-perl@a198315ec4e9244f206879ea7b63078003aec8a6 # v1.41.1
 
       - name: Setup Java
-        uses: actions/setup-java@v4.7.1
+        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
         with:
           distribution: 'temurin'
           java-version: '8'
@@ -60,4 +60,4 @@ jobs:
                               locales
 
       - name: Build and run tests
-        run: CI=true && python -m poetry run python -m pytest -sxv tests/
+        run: CI=true && python -m poetry run python -m pytest -sxv tests/core/benchmarks/BugInPy/

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "cache"]
 	path = cache
 	url = https://github.com/ASSERT-KTH/elle-elle-aime-cache.git
+[submodule "benchmarks/BugsInPy"]
+	path = benchmarks/BugsInPy
+	url = https://github.com/soarsmu/BugsInPy.git

Dockerfile.bugsinpy

@@ -0,0 +1,48 @@
+FROM python:3.8-slim-bullseye
+
+# Install core system utilities needed by BugsInPy and Miniconda
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    ca-certificates \
+    git \
+    dos2unix \
+    vim \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Miniconda (Crucial for isolating legacy Python project versions)
+ENV CONDA_DIR=/opt/conda
+RUN curl -sS https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /miniconda.sh && \
+    chmod +x /miniconda.sh && \
+    /miniconda.sh -b -p $CONDA_DIR && \
+    rm /miniconda.sh
+ENV PATH="$CONDA_DIR/bin:${PATH}"
+
+# Initialize Conda for shell interactions
+RUN conda init bash
+
+# Install uv package manager
+ADD https://astral.sh/uv/install.sh /uv-installer.sh
+RUN sh /uv-installer.sh && rm /uv-installer.sh
+
+# Set framework environment variables
+ENV PATH="/root/.local/bin:${PATH}"
+ENV BUGSINPY_HOME="/bugsinpy"
+ENV PATH="$BUGSINPY_HOME/framework/bin:${PATH}"
+
+# Set working directory
+WORKDIR /bugsinpy
+
+# Copy the BugsInPy project files
+COPY benchmarks/BugsInPy /bugsinpy
+
+# Make framework binaries executable and create temp directory
+RUN chmod +x /bugsinpy/framework/bin/* && \
+    mkdir -p /bugsinpy/framework/bin/temp
+
+# Install Python dependencies required by the BugsInPy framework scripts
+RUN pip3 install --no-cache-dir --upgrade pip setuptools wheel && \
+    pip3 install --no-cache-dir six pytest python-toolbox backoff future decorator pandas unidiff ansible
+
+# Set default command
+CMD ["bash"]

elleelleaime/core/benchmarks/BugsInPy/BugsInPy.py

@@ -0,0 +1,117 @@
+from pathlib import Path
+from typing import Optional
+from io import StringIO
+from elleelleaime.core.benchmarks.benchmark import Benchmark
+from elleelleaime.core.benchmarks.BugsInPy.BugsInPybug import BugsInPyBug
+import subprocess
+import logging
+import re
+import pandas as pd
+
+
+class BugsInPy(Benchmark):
+    """
+    The class for representing the BugsInPy benchmark.
+    """
+
+    def __init__(self, path: Path = Path("benchmarks/BugsInPy").absolute()) -> None:
+        super().__init__("BugsInPy", path)
+
+    def get_bin(self, options: str = "") -> Optional[str]:
+        return f'{Path(self.path, "framework/bin/")}'
+
+    def initialize(self) -> None:
+        """
+        Initializes the BugsInPy benchmark object by collecting the list of all projects and bugs.
+        """
+        logging.info("Initializing BugsInPy benchmark...")
+
+        # Get all project names
+        run = subprocess.run(
+            f"docker exec bugsinpy-container ls /bugsinpy/projects",
+            shell=True,
+            capture_output=True,
+            check=True,
+        )
+        project_names = {
+            project_name.decode("utf-8") for project_name in run.stdout.split()
+        }
+        logging.info("Found %3d projects" % len(project_names))
+
+        # Get all bug names for all project_name
+        bugs = {}
+        for project_name in project_names:
+            run = subprocess.run(
+                f"docker exec bugsinpy-container ls /bugsinpy/projects/{project_name}/bugs",
+                shell=True,
+                capture_output=True,
+                check=True,
+            )
+
+            bugs[project_name] = set()
+            for bug_id in run.stdout.split():
+                try:
+                    bug_id_str = bug_id.decode("utf-8").strip()
+                    # Skip invalid bug IDs (files with extensions, special characters, etc.)
+                    if (
+                        not bug_id_str.isdigit()
+                        or "." in bug_id_str
+                        or "~" in bug_id_str
+                        or "$" in bug_id_str
+                    ):
+                        logging.warning(f"Skipping invalid bug ID: {bug_id_str}")
+                        continue
+                    bug_id_int = int(bug_id_str)
+                    bugs[project_name].add(bug_id_int)
+                except ValueError:
+                    logging.warning(
+                        f"Skipping invalid bug ID: {bug_id.decode('utf-8')}"
+                    )
+
+            logging.info(
+                "Found %3d bugs for project %s"
+                % (len(bugs[project_name]), project_name)
+            )
+
+        # Initialize dataset
+        for project_name in project_names:
+            # Create a DataFrame to store the failing test cases and trigger causes
+            df = pd.DataFrame(columns=["bid", "tests", "errors"])
+
+            for bug_id in bugs[project_name]:
+                # Extract ground truth diff
+                diff_path = (
+                    f"/bugsinpy/projects/{project_name}/bugs/{bug_id}/bug_patch.txt"
+                )
+                try:
+                    run = subprocess.run(
+                        f"docker exec bugsinpy-container cat {diff_path}",
+                        shell=True,
+                        capture_output=True,
+                        check=True,
+                    )
+                    diff = run.stdout.decode("utf-8")
+
+                    # Skip bugs with empty ground truth
+                    if not diff.strip():
+                        logging.warning(
+                            f"Empty ground truth for {project_name}-{bug_id}, skipping..."
+                        )
+                        continue
+
+                except subprocess.CalledProcessError:
+                    logging.warning(
+                        f"Could not read bug_patch.txt for {project_name}-{bug_id}, skipping..."
+                    )
+                    continue
+
+                self.add_bug(
+                    BugsInPyBug(
+                        self,
+                        project_name=project_name,
+                        bug_id=bug_id,
+                        version_id="0",
+                        ground_truth=diff,
+                        failing_tests={},
+                    )
+                )