diff --git a/HISTORY.md b/HISTORY.md index 01db6e6db..37727c5df 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,13 @@ # History +## (Forthcoming) + +### Changelog + +#### API + +- Implement `novelty_threshold` decay in `ProximityArchive` ({pr}`709`) + ## 0.11.0 ### Changelog diff --git a/README.md b/README.md index c99f5c28a..1af5b8a22 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,7 @@ USC. For information on contributing to the repo, see - [Ryan Boldi](https://ryanboldi.github.io) - [Efstathios Siatras](https://siatras.com/) - [Milo Brontesi](https://github.com/zibasPk) +- [Alejandro Marrero](https://github.com/amarrerod) - [Stefanos Nikolaidis](https://stefanosnikolaidis.net) We thank [Amy K. Hoover](http://amykhoover.com/) and diff --git a/ribs/archives/_proximity_archive.py b/ribs/archives/_proximity_archive.py index 0f8dcd5f0..a59372e73 100644 --- a/ribs/archives/_proximity_archive.py +++ b/ribs/archives/_proximity_archive.py @@ -102,6 +102,17 @@ class ProximityArchive(ArchiveBase): *subtracted* from all objectives in the archive, e.g., if your objectives go as low as -300, pass in -300 so that each objective will be transformed as ``objective - (-300)``. + threshold_decay_rate: Decay factor to reduce the ``novelty_threshold``. When + :meth:`add` is called ``threshold_decay_itrs`` times in a row without + inserting any novel solutions, the ``novelty_threshold`` is multiplied by + this rate. The default value is None, which indicates that there is no + decay. If passed in, it must be a float value in the range [0.0, 1.0]. + threshold_decay_itrs: See ``threshold_decay_rate`` above. This parameter only + applies if ``threshold_decay_rate`` is set. The default value of 1 indicates + that the ``novelty_threshold`` will be decreased immediately after a call to + :meth:`add` has no solutions that are novel enough. + threshold_decay_min: Minimum value for the ``novelty_threshold`` if threshold + decay is enabled. seed: Value to seed the random number generator. Set to None to avoid a fixed seed. solution_dtype: Data type of the solutions. Defaults to float64 (numpy's default @@ -140,6 +151,9 @@ def __init__( local_competition: bool = False, initial_capacity: Int = 128, qd_score_offset: Float = 0.0, + threshold_decay_rate: Float | None = None, + threshold_decay_itrs: Int = 1, + threshold_decay_min: Float = 0.0, seed: Int | None = None, solution_dtype: DTypeLike = None, objective_dtype: DTypeLike = None, @@ -210,6 +224,31 @@ def __init__( self._stats = None self._stats_reset() + # Set up threshold decay. + if threshold_decay_rate is None: + self._threshold_decay_rate = None + self._threshold_decay_itrs = None + self._threshold_decay_min = None + self._itrs_without_novel = None + else: + if threshold_decay_rate <= 0.0 or threshold_decay_rate > 1.0: + raise ValueError( + "If passed in, threshold_decay_rate must be a float in the range [0.0, 1.0]." + ) + if threshold_decay_itrs <= 0: + raise ValueError( + "threshold_decay_itrs must be either None or a positive integer." + ) + if threshold_decay_min < 0.0: + raise ValueError( + "threshold_decay_min must be a non-negative float value." + ) + + self._threshold_decay_rate = float(threshold_decay_rate) + self._threshold_decay_itrs = int(threshold_decay_itrs) + self._threshold_decay_min = float(threshold_decay_min) + self._itrs_without_novel = 0 + ## Properties inherited from ArchiveBase ## @property @@ -480,6 +519,42 @@ def _maybe_resize(self, new_size: int) -> None: multiplier = 2 ** int(np.ceil(np.log2(new_size / self.capacity))) self._store.resize(multiplier * self.capacity) + def _maybe_update_threshold(self, n_novel_enough: int) -> None: + """Performs threshold decay if needed. + + Args: + n_novel_enough (int): Number of newly novel solutions added to the archive. + """ + # Threshold decay has not been activated, so do nothing. + if self._threshold_decay_rate is None: + return + + if n_novel_enough == 0: + # If n_novel_enough == 0, it means that, whether local_competition is True + # or not, we have not inserted any novel solutions into the archive. Thus, + # the number of iterations without novel solutions is updated. + self._itrs_without_novel += 1 + + if self._itrs_without_novel >= self._threshold_decay_itrs: + # If there have been at least `threshold_decay_itrs` calls to `add` + # without inserting any novel solutions, then we update the threshold to + # max(threshold_decay_min, threshold * decay). + new_threshold = np.max( + [ + self._threshold_decay_min, + self._novelty_threshold * self._threshold_decay_rate, + ] + ) + self._novelty_threshold = np.asarray( + new_threshold, dtype=self.dtypes["measures"] + ) + + # Restart the counter since the threshold was just updated. + self._itrs_without_novel = 0 + else: + # If n_novel_enough is not 0, then we restart the counter. + self._itrs_without_novel = 0 + def add( self, solution: ArrayLike, @@ -625,6 +700,7 @@ def add( self._store.data("measures"), **self._kdtree_kwargs ) + self._maybe_update_threshold(n_novel_enough) return add_info else: @@ -755,6 +831,7 @@ def add( self._store.data("measures"), **self._kdtree_kwargs ) + self._maybe_update_threshold(n_novel_enough) return add_info def add_single( diff --git a/tests/archives/proximity_archive_test.py b/tests/archives/proximity_archive_test.py index a98980a72..13c17bd5e 100644 --- a/tests/archives/proximity_archive_test.py +++ b/tests/archives/proximity_archive_test.py @@ -834,6 +834,7 @@ def test_lc_add_batch_replace_same_cell(): def test_add_novel_and_not_novel_no_improve_bug(): + """See https://github.com/icaros-usc/pyribs/pull/704/.""" archive = ProximityArchive( solution_dim=3, measure_dim=2, @@ -865,3 +866,144 @@ def test_add_novel_and_not_novel_no_improve_bug(): objective_batch=[10.0, 10.0], measures_batch=[[0, 0], [5.0, 5.0]], ) + + +def test_add_non_novel_solution_threshold_decay(): + """novelty_threshold decays when no novel solutions are added.""" + iterations = 5 + initial_threshold = 1.0 + archive = ProximityArchive( + solution_dim=3, + measure_dim=2, + k_neighbors=1, + novelty_threshold=initial_threshold, + initial_capacity=1, + threshold_decay_rate=0.5, + threshold_decay_itrs=iterations, + ) + assert_allclose(archive.novelty_threshold, 1.0) + archive.add_single([1, 2, 3], None, [0, 0]) + + for _ in range(iterations): + assert_allclose(archive.novelty_threshold, 1.0) + # Should not be added since threshold is 1.0. + add_info = archive.add_single([1, 2, 3], None, [0.5, 0]) + assert add_info["status"] == 0 + assert_allclose(add_info["novelty"], 0.5) + + # At this point, novelty_threshold must have been updated to 0.5. + assert_allclose(archive.novelty_threshold, 0.5) + assert_archive_elites(archive, 1, measures_batch=[[0, 0]]) + + # Now when we add the solution, it is accepted. + add_info = archive.add_single([1, 2, 3], None, [0.5, 0]) + assert_archive_elites(archive, 2, measures_batch=[[0, 0], [0.5, 0]]) + + +def test_no_threshold_decay_when_adding_novel_solutions(): + """novelty_threshold remains constant when adding novel solutions.""" + iterations = 5 + initial_threshold = 1.0 + archive = ProximityArchive( + solution_dim=3, + measure_dim=2, + k_neighbors=1, + novelty_threshold=initial_threshold, + initial_capacity=1, + threshold_decay_itrs=iterations, + threshold_decay_rate=0.5, + ) + assert_allclose(archive.novelty_threshold, 1.0) + inner_measures = np.arange(iterations) + new_measures = np.stack([inner_measures, inner_measures], axis=1) + + for m in new_measures: + add_info = archive.add_single([1, 2, 3], None, m) + # The solution was added to the archive. + assert add_info["status"] == 2 + # The threshold must remain the same --> 1.0 + assert_allclose(archive.novelty_threshold, 1.0) + + assert_archive_elites(archive, iterations, measures_batch=new_measures) + assert_allclose(archive.novelty_threshold, 1.0) + + +def test_mix_novel_and_not_novel_for_threshold_decay(): + """novelty_threshold responds correctly when we mix novel and not novel + solutions.""" + iterations = 5 + initial_threshold = 1.0 + archive = ProximityArchive( + solution_dim=3, + measure_dim=2, + k_neighbors=1, + novelty_threshold=initial_threshold, + initial_capacity=1, + threshold_decay_itrs=iterations, + threshold_decay_rate=0.5, + ) + assert_allclose(archive.novelty_threshold, 1.0) + + # Insert an initial solution with measure [0,0] to have a reference. Thus, + # not_novel_measure should never be included. + not_novel_measure = [0.5, 0] + + for i in range(iterations * 2): + to_insert = np.asarray([i, i]) if i % 2 == 0 else not_novel_measure + add_info = archive.add_single([1, 2, 3], None, to_insert) + if i % 2 == 0: + # We should have inserted a novel solution. + assert add_info["status"] == 2 + # The threshold must remain the same --> 1.0 + assert_allclose(archive.novelty_threshold, 1.0) + else: + # This duplicated measure should not be inserted. + assert add_info["status"] == 0 + assert_allclose(archive.novelty_threshold, 1.0) + + assert_allclose(archive.novelty_threshold, 1.0) + + +def test_add_with_threshold_decay(): + """Since the threshold_decay tests above use add_single, this uses add() for + completeness.""" + iterations = 1 + initial_threshold = 1.0 + archive = ProximityArchive( + solution_dim=3, + measure_dim=2, + k_neighbors=1, + novelty_threshold=initial_threshold, + initial_capacity=1, + threshold_decay_itrs=iterations, + threshold_decay_rate=0.5, + ) + assert_allclose(archive.novelty_threshold, 1.0) + + add_info = archive.add([[1, 2, 3]], [0.0], [[0.0, 0.0]]) + assert_equal(add_info["status"], [2]) + assert_allclose(archive.novelty_threshold, 1.0) + + # One solution is novel enough while the other is not. + add_info = archive.add([[4, 5, 6], [7, 8, 9]], [0.0, 0.0], [[0.5, 0.0], [1.0, 0.0]]) + assert_equal(add_info["status"], [0, 2]) + assert_allclose(archive.novelty_threshold, 1.0) + + # Neither solution is novel enough, so the threshold decreases immediately. + add_info = archive.add( + [[10, 11, 12], [13, 14, 15]], [0.0, 0.0], [[1.5, 0.0], [1.75, 0.0]] + ) + assert_equal(add_info["status"], [0, 0]) + assert_allclose(archive.novelty_threshold, 0.5) + + # Now both solutions can be added. + add_info = archive.add( + [[10, 11, 12], [13, 14, 15]], [0.0, 0.0], [[1.5, 0.0], [1.75, 0.0]] + ) + assert_equal(add_info["status"], [2, 2]) + assert_allclose(archive.novelty_threshold, 0.5) + + # Check the elites in the archive. + assert_archive_elites( + archive, 4, measures_batch=[[0.0, 0.0], [1.0, 0.0], [1.5, 0.0], [1.75, 0.0]] + )