Modernize CI, adopt Ruff, and improve MongoDB test setup

.coveragerc

@@ -1,10 +1,10 @@
 # .coveragerc to control coverage.py
 [run]
 branch = True
-source = 
+source =
     *mcrit*
     *libs*
-include = 
+include =
     mcrit*
     *libs*
     tests*
@@ -20,7 +20,7 @@ omit =
     *config*
     *numpy*
     *pathlib*
-    
+
 [report]
 omit =
     *pymmh3*

.github/workflows/test.yml

@@ -1,30 +1,137 @@
-name: Test
+name: CI
 
-on: ["push", "pull_request"]
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  testing:
-    name: Testing
+  lint:
+    name: Ruff
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+
+      - name: Install Ruff
+        run: python -m pip install --upgrade pip ruff
+
+      - name: Check formatting
+        run: ruff format --check .
+
+      - name: Lint
+        run: ruff check .
+
+  unit:
+    name: Unit tests (Python ${{ matrix.python-version }})
     runs-on: ubuntu-latest
 
     strategy:
+      fail-fast: false
       matrix:
-        mongodb-version: [5.0]
+        python-version: ["3.11", "3.12"]
 
     steps:
-      - name: Clone repository
-        uses: actions/checkout@v3
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: requirements.txt
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt
+          python -m pip install -e . --no-deps
+
+      - name: Run unit tests
+        run: python -m pytest -m "not mongo and not sleep"
 
-      - name: Install dependencies for the client
-        run:  |
-              pip3 install -r requirements.txt
-              pip install -e .
-        shell: bash
+  integration:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    env:
+      TEST_MONGODB: 127.0.0.1:27017
+    services:
+      mongo:
+        image: mongo:5.0
+        ports:
+          - 27017:27017
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
 
-      - name: Start MongoDB v${{ matrix.mongodb-version }}
-        uses: supercharge/mongodb-github-action@1.2.0
+      - name: Set up Python
+        uses: actions/setup-python@v6
         with:
-          mongodb-version: ${{ matrix.mongodb-version }}
+          cache: pip
+          cache-dependency-path: requirements.txt
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt
+          python -m pip install -e . --no-deps
+
+      - name: Wait for MongoDB
+        run: |
+          python - <<'PY'
+          import os
+          import time
+          import pymongo
+
+          uri = f"mongodb://{os.environ['TEST_MONGODB']}/"
+          deadline = time.time() + 60
+          last_error = None
+          while time.time() < deadline:
+              try:
+                  client = pymongo.MongoClient(uri, serverSelectionTimeoutMS=1000)
+                  client.admin.command("ping")
+                  print("MongoDB is ready.")
+                  break
+              except Exception as exc:
+                  last_error = exc
+                  time.sleep(1)
+          else:
+              raise SystemExit(f"MongoDB did not become ready: {last_error}")
+          PY
+
+      - name: Run integration tests
+        run: python -m pytest
+
+  build:
+    name: Build package
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+
+      - name: Install build tooling
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build twine "packaging>=25.0"
+
+      - name: Build distributions
+        run: python -m build --sdist --wheel
 
-      - name: Test
-        run: make test
+      - name: Validate distributions
+        run: python -m twine check dist/*

.gitignore

@@ -74,4 +74,4 @@ data/*
 diagnosis/*
 experiments/*
 
-profiler
+profiler

.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.9
+    hooks:
+      - id: ruff-check
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer

Makefile

@@ -8,7 +8,7 @@ publish:
 pylint:
 	python -m pylint --rcfile=.pylintrc mcrit
 test:
-	python -m pytest 
+	python -m pytest
 test-nomongo:
 	python -m pytest -m 'not mongo'
 test-nosleep:

README.md

@@ -9,20 +9,20 @@ It is tailored to work with disassembly reports emitted by [SMDA](https://github
 
 ### Dockerized Usage
 
-We highly recommend to use the fully packaged [docker-mcrit](https://github.com/danielplohmann/docker-mcrit) for trivial deployment and usage.  
+We highly recommend to use the fully packaged [docker-mcrit](https://github.com/danielplohmann/docker-mcrit) for trivial deployment and usage.
 First and foremost, this will ensure that you have fully compatible versions across all components, including a database for persistence and a web frontend for convenient interaction.
 
 ### Standalone Usage
 
-Installing MCRIT on its own will require some more steps.  
+Installing MCRIT on its own will require some more steps.
 For the following, we assume Ubuntu as host operating system.
 
 The Python installation requirements are listed in `requirements.txt` and can be installed using:
 
 ```bash
 # install python and MCRIT dependencies
 $ sudo apt install python3 python3-pip
-$ pip install -r requirements.txt 
+$ pip install -r requirements.txt
 ```
 
 By default, MongoDB 5.0 is used as backend, which is also the recommended mode of operation as it provides a persistent data storage.
@@ -80,15 +80,15 @@ Regardless of your choice for installation, once running you can interact with t
 
 ### MCRIT Client
 
-We have created a Python client module that is capable of working with all available endpoints of the server.  
+We have created a Python client module that is capable of working with all available endpoints of the server.
 Documentation for this client module is currently in development.
 
 ### MCRIT CLI
 
 There is also a CLI which is based on this client package, examples:
 
 ```bash
-# query some stats of the data stored in the backend 
+# query some stats of the data stored in the backend
 $ mcrit client status
 {'status': {'db_state': 187, 'storage_type': 'mongodb', 'num_bands': 20, 'num_samples': 137, 'num_families': 14, 'num_functions': 129110, 'num_pichashes': 25385}}
 # submit a malware sample with filename sample_unpacked, using family name "some_family"
@@ -115,7 +115,7 @@ Then simply run the script found at
 
 in IDA.
 
-### Reference Data 
+### Reference Data
 
 In July 2023, we started populating a [Github repository](https://github.com/danielplohmann/mcrit-data) which contains ready-to-use reference data for common compilers and libraries.
 
@@ -146,7 +146,7 @@ In July 2023, we started populating a [Github repository](https://github.com/dan
  * 2024-02-02 v1.3.4:  Mini fix in the IDA plugin to avoid referencing a potentially uninitialized object (THX to @r0ny123!!).
  * 2024-02-01 v1.3.2:  FIX: Non-parallelized matching now outputs the [same data format](https://github.com/danielplohmann/mcrit/pull/63) (THX to @dannyquist!!).
  * 2024-01-30 v1.3.1:  The connection to MongoDB is now fully [configurable](https://github.com/danielplohmann/mcrit/pull/61) (THX to @dannyquist!!).
- * 2024-01-24 v1.3.0:  BREAKING: Milestone release with indexing improvements for PicHash and MinHash. To ensure full backward compatibility, recalculation of all hashes is recommended. Check this [migration guide](https://github.com/danielplohmann/mcrit/blob/main/docs/migration-v1.3.0.md). 
+ * 2024-01-24 v1.3.0:  BREAKING: Milestone release with indexing improvements for PicHash and MinHash. To ensure full backward compatibility, recalculation of all hashes is recommended. Check this [migration guide](https://github.com/danielplohmann/mcrit/blob/main/docs/migration-v1.3.0.md).
  * 2024-01-23 v1.2.26: Pinning lief to 0.13.2 in order to ensure that the pinned SMDA remains compatible.
  * 2024-01-09 v1.2.25: Ensure that we can deliver system status regardless of whether there is a `db_state` and `db_timestamp` or not.
  * 2024-01-05 v1.2.24: Now supporting "query" argument in CLI, as well as compact MatchingResults (without function match info) to reduce file footprint.
@@ -261,7 +261,7 @@ Pull requests welcome! :)
 
     You should have received a copy of the GNU General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
-    
-    Some plug-ins and libraries may have different licenses. 
+
+    Some plug-ins and libraries may have different licenses.
     If so, a license file is provided in the plug-in's folder.
 ```

docs/mcrit-cli.md

@@ -4,7 +4,7 @@ In this document we describe the different ways of interaction enabled by the MC
 
 ## Submit
 
-The `submit` command supports 4 methods of data submission, supported by a number of auxiliary flags.  
+The `submit` command supports 4 methods of data submission, supported by a number of auxiliary flags.
 Here is its documentation:
 
 ```
@@ -41,10 +41,10 @@ options:
 
 ### File
 
-Submit a single file.  
-If the provided path ends with `0x[0-9a-fA-F]{8,16}`, the client will assume that this is a mapped file (no evaluation of PE header) and use the given addr as IMAGEBASE. 
+Submit a single file.
+If the provided path ends with `0x[0-9a-fA-F]{8,16}`, the client will assume that this is a mapped file (no evaluation of PE header) and use the given addr as IMAGEBASE.
 ```bash
-$ mcrit client submit --mode file ~/malpedia/win.wannacryptor/vt-2017-05-05/0345782378ee7a8b48c296a120625fd439ed8699ae857c4f84befeb56e727366_dump_0x00400000 
+$ mcrit client submit --mode file ~/malpedia/win.wannacryptor/vt-2017-05-05/0345782378ee7a8b48c296a120625fd439ed8699ae857c4f84befeb56e727366_dump_0x00400000
  0.906s -> (architecture: intel.32bit, base_addr: 0x00400000): 922 functions
 ```
 
@@ -62,7 +62,7 @@ $ mcrit client submit --mode dir ~/malpedia/win.wannacryptor/vt-2017-05-12/
 
  ### Recursive
 
- Recursively submit all files found in a directory, assuming a structure like 
+ Recursively submit all files found in a directory, assuming a structure like
  ```
  ./family_name/version/version/files
  ```
@@ -71,10 +71,10 @@ $ mcrit client submit --mode dir ~/malpedia/win.wannacryptor/vt-2017-05-12/
 
  ### Malpedia
 
-Dedicated mode to synchronize all data available in the Malpedia repository.  
-If files from Malpedia are already available in MCRIT, they will be skipped on the next execution, making this command suitable to synchronize updated states of Malpedia into MCRIT.  
-Only ELF and PE (win.*) families and only files labeled as `_unpacked` or `_dump_0x...`  will be considered and processed.  
-Requires directory name to be called `malpedia`, sanity checks if there is a `malpedia.bib` file present as in the repo.  
+Dedicated mode to synchronize all data available in the Malpedia repository.
+If files from Malpedia are already available in MCRIT, they will be skipped on the next execution, making this command suitable to synchronize updated states of Malpedia into MCRIT.
+Only ELF and PE (win.*) families and only files labeled as `_unpacked` or `_dump_0x...`  will be considered and processed.
+Requires directory name to be called `malpedia`, sanity checks if there is a `malpedia.bib` file present as in the repo.
 
  ```
  $ mcrit client submit --mode malpedia ~/malpedia
@@ -96,7 +96,7 @@ usage: mcrit client export [-h] [--sample_ids SAMPLE_IDS] filepath
 $ mcrit client export --sample_ids 0 sample_0.mcrit
 wrote export to sample_0.mcrit.
 
-$ head sample_0.mcrit 
+$ head sample_0.mcrit
 {
  "content": {
   "is_compressed": true,
@@ -117,7 +117,7 @@ This command can be used to import previously exported sample(s), e.g.:
 ```bash
 usage: mcrit client import [-h] filepath
 
-$ mcrit client import sample_0.mcrit               
+$ mcrit client import sample_0.mcrit
 {'num_samples_imported': 0, 'num_samples_skipped': 1, 'num_functions_imported': 0, 'num_functions_skipped': 214, 'num_families_imported': 0, 'num_families_skipped': 1}
 
 ```
@@ -129,13 +129,13 @@ This command can be used to search across families, samples, and functions, e.g.
 ```bash
 usage: mcrit client search [-h] search_term
 
-$ mcrit client search wanna                                               
+$ mcrit client search wanna
 Family Search Results
-Famliy 2 (win.wannacry): 
+Famliy 2 (win.wannacry):
 ********************
 Sample Search Results
-Sample 1 (intel, 32 bit) - ca29de1dc8817868c93e54b09f557fe14e40083c0955294df5bd91f52ba469c8_unpacked (win.wannacry): 
-Sample 2 (intel, 32 bit) - 3e6de9e2baacf930949647c399818e7a2caea2626df6a468407854aaa515eed9 (win.wannacry): 
+Sample 1 (intel, 32 bit) - ca29de1dc8817868c93e54b09f557fe14e40083c0955294df5bd91f52ba469c8_unpacked (win.wannacry):
+Sample 2 (intel, 32 bit) - 3e6de9e2baacf930949647c399818e7a2caea2626df6a468407854aaa515eed9 (win.wannacry):
 ********************
 ```
 
@@ -148,4 +148,4 @@ $ mcrit client queue
 64243b27f3876416bffad86e 64243b28cbc77c2df4d8d79f | 2023-03-29T13:20:39.065Z 2023-03-29T13:20:39.114Z 2023-03-29T13:20:40.593Z | updateMinHashesForSample(2) - 1
 64131888fbb4d9d4a029164d 6413188c15e4f20d519b35ba | 2023-03-16T13:24:24.707Z 2023-03-16T13:24:24.755Z 2023-03-16T13:24:28.366Z | addBinarySample(None, ca29de1dc8817868c93e54b09f557fe14e40083c0955294df5bd91f52ba469c8_unpacked, win.wannacry, , False, 0, 32) - 1
 641316eefbb4d9d4a029164a 641316f115e4f20d519b322b | 2023-03-16T13:17:34.834Z 2023-03-16T13:17:34.859Z 2023-03-16T13:17:37.238Z | addBinarySample(None, 766d7d591b9ec1204518723a1e5940fd6ac777f606ed64e731fd91b0b4c3d9fc_dump_0x10000000, win.contopee, , True, 268435456, 32) - 1
-```
+```