Implement novelty_threshold decay in ProximityArchive

HISTORY.md

@@ -15,6 +15,7 @@
 - Add `DDS-CNF` density method to `DensityArchive` ({pr}`691`, {pr}`707`)
 - Add PyTorch to ribs[all] deps ({pr}`692`)
 - Add `NSLCRanker` for Novelty Search with Local Competition ({pr}`690`)
+- Implements `novelty_threshold` decay in `ProximityArchive`({issue}`540`, {pr}`708`)
 
 #### Documentation
 

README.md

@@ -267,6 +267,7 @@ USC. For information on contributing to the repo, see
 - [Efstathios Siatras](https://siatras.com/)
 - [Milo Brontesi](https://github.com/zibasPk)
 - [Stefanos Nikolaidis](https://stefanosnikolaidis.net)
+- [Alejandro Marrero](https://github.com/amarrerod)
 
 We thank [Amy K. Hoover](http://amykhoover.com/) and
 [Julian Togelius](http://julian.togelius.com/) for their contributions deriving

ribs/archives/_proximity_archive.py

@@ -102,6 +102,16 @@ class ProximityArchive(ArchiveBase):
             *subtracted* from all objectives in the archive, e.g., if your objectives go
             as low as -300, pass in -300 so that each objective will be transformed as
             ``objective - (-300)``.
+        iterations_without_imp: Maximum number of iterations without inserting novel
+            solutions, before decreasing the ``novelty_threshold`` by a factor of
+            ``threshold_decay``. This is an optional parameter that by
+            default is set to None, which means that the ``novelty_threshold``
+            is not updated during the evolution.
+        threshold_decay: Decay factor to reduce the ``novelty_treshold`` if there were
+            ``iterations_without_imp`` iterations without inserting novelty solutions.
+            This is a optional float value in the range [0.0, 1.0]. Default is set to
+            None, which means that the ``novelty_threshold`` is not updated during
+            the evolution.
         seed: Value to seed the random number generator. Set to None to avoid a fixed
             seed.
         solution_dtype: Data type of the solutions. Defaults to float64 (numpy's default
@@ -140,6 +150,8 @@ def __init__(
         local_competition: bool = False,
         initial_capacity: Int = 128,
         qd_score_offset: Float = 0.0,
+        iterations_without_imp: Int | None = None,
+        threshold_decay: Float | None = None,
         seed: Int | None = None,
         solution_dtype: DTypeLike = None,
         objective_dtype: DTypeLike = None,
@@ -179,6 +191,24 @@ def __init__(
         solution_dtype, objective_dtype, measures_dtype = parse_all_dtypes(
             dtype, solution_dtype, objective_dtype, measures_dtype, np
         )
+
+        # By default, threshold decay is not defined.
+        self._max_it_without_imp = None
+        if iterations_without_imp is not None:
+            if iterations_without_imp < 0:
+                raise ValueError(
+                    "iterations_without_imp must be either None or a positive integer."
+                )
+            if threshold_decay is None or (
+                threshold_decay < 0.0 or threshold_decay > 1.0
+            ):
+                raise ValueError(
+                    "If iterations_without_imp is not None, threshold decay must be a float in the range [0.0, 1.0]."
+                )
+            self._max_it_without_imp = int(iterations_without_imp)
+            self._threshold_decay = float(threshold_decay)
+            self._its_without_imp = 0
+
         self._store = ArrayStore(
             field_desc={
                 "solution": (self.solution_dim, solution_dtype),
@@ -480,6 +510,30 @@ def _maybe_resize(self, new_size: int) -> None:
             multiplier = 2 ** int(np.ceil(np.log2(new_size / self.capacity)))
             self._store.resize(multiplier * self.capacity)
 
+    def __maybe_update_threshold(self):
+        """Performs threshold decay.
+
+        After :attr:`iterations_without_imp` iterations without inserting
+        any new novel solution into the archive, the :attr:`novelty_threshold`
+        is decreased to continue inserting new solutions.
+        """
+        # If n_novel_enough == 0 it means that whether LC is True or not
+        # we have not inserted any novel solutions into the archive
+        # therefore, the number of iterations without any insertion must be updated
+        self._its_without_imp += 1
+        if self._its_without_imp == self._max_it_without_imp:
+            # Restart the counter and set the new threshold to max(0.0, threshold * decay)
+            self._its_without_imp = 0
+            new_threshold = np.max(
+                [
+                    0.0,
+                    self._novelty_threshold * self._threshold_decay,
+                ]
+            )
+            self._novelty_threshold = np.asarray(
+                new_threshold, dtype=self.dtypes["measures"]
+            )
+
     def add(
         self,
         solution: ArrayLike,
@@ -625,6 +679,9 @@ def add(
                     self._store.data("measures"), **self._kdtree_kwargs
                 )
 
+            if self._max_it_without_imp and n_novel_enough == 0:
+                self.__maybe_update_threshold()
+
             return add_info
 
         else:
@@ -755,6 +812,9 @@ def add(
                     self._store.data("measures"), **self._kdtree_kwargs
                 )
 
+            if self._max_it_without_imp and n_novel_enough == 0:
+                self.__maybe_update_threshold()
+
             return add_info
 
     def add_single(

tests/archives/proximity_archive_test.py

@@ -865,3 +865,28 @@ def test_add_novel_and_not_novel_no_improve_bug():
         objective_batch=[10.0, 10.0],
         measures_batch=[[0, 0], [5.0, 5.0]],
     )
+
+
+def test_add_non_novel_solution_threshold_decay():
+    iterations = 5
+    initial_threshold = 1.0
+    archive = ProximityArchive(
+        solution_dim=3,
+        measure_dim=2,
+        k_neighbors=1,
+        novelty_threshold=initial_threshold,
+        initial_capacity=1,
+        threshold_decay=0.5,
+        iterations_without_imp=iterations,
+    )
+    assert_allclose(archive.novelty_threshold, 1.0)
+    archive.add_single([1, 2, 3], None, [0, 0])
+
+    for _ in range(iterations):
+        # Should not be added since threshold is 1.0.
+        add_info = archive.add_single([1, 2, 3], None, [0.5, 0])
+        assert add_info["status"] == 0
+        assert_allclose(add_info["novelty"], 0.5)
+
+    assert_archive_elites(archive, 1, measures_batch=[[0, 0]])
+    assert_allclose(archive.novelty_threshold, 0.5)