Skip to content

Rework base function for sensitivity calculation #311

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
tomaskontrimas wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Rework base function for sensitivity calculation #311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions skyllh/core/utils/analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -703,8 +703,8 @@ def estimate_mean_nsignal_for_ts_quantile(

logger.debug(
'Scanned mu range: [%g , %g]\nPoints to fit: %g\n Doing a linear interpolation.',
np.min(n_sig),
np.max(n_sig),
np.min(n_sig) if len(n_sig) > 0 else ns0,
np.max(n_sig) if len(n_sig) > 0 else ns1,
len(n_sig),
)

Expand Down Expand Up @@ -870,13 +870,14 @@ def estimate_sensitivity(
eps_p=0.005,
mu_range=None,
min_dmu=0.5,
critical_ts=None,
bkg_kwargs=None,
sig_kwargs=None,
ppbar=None,
tl=None,
pathfilename=None,
):
"""Estimates the mean number of signal events that whould have to be
"""Estimates the mean number of signal events that would have to be
injected into the data such that the test-statistic value of p*100% of all
trials are larger than the critical test-statistic value c, which
corresponds to the test-statistic value where h0_ts_quantile*100% of all
Expand Down Expand Up @@ -911,6 +912,10 @@ def estimate_sensitivity(
min_dmu : float
The minimum delta mu to use for calculating the derivative dmu/dp.
The default is ``0.5``.
critical_ts : float | None
The critical test-statistic value that should be overcome by the signal
distribution. If set to None, the null-hypothesis test-statistic
distribution will be used to compute the critical TS value.
bkg_kwargs : dict | None
Additional keyword arguments for the `generate_events` method of the
background generation method class. An usual keyword argument is
Expand Down Expand Up @@ -949,6 +954,7 @@ def estimate_sensitivity(
eps_p=eps_p,
mu_range=mu_range,
min_dmu=min_dmu,
critical_ts=critical_ts,
bkg_kwargs=bkg_kwargs,
sig_kwargs=sig_kwargs,
ppbar=ppbar,
Expand All @@ -968,13 +974,14 @@ def estimate_discovery_potential(
eps_p=0.005,
mu_range=None,
min_dmu=0.5,
critical_ts=None,
bkg_kwargs=None,
sig_kwargs=None,
ppbar=None,
tl=None,
pathfilename=None,
):
"""Estimates the mean number of signal events that whould have to be
"""Estimates the mean number of signal events that would have to be
injected into the data such that the test-statistic value of p*100% of all
trials are larger than the critical test-statistic value c, which
corresponds to the test-statistic value where h0_ts_quantile*100% of all
Expand Down Expand Up @@ -1008,6 +1015,10 @@ def estimate_discovery_potential(
min_dmu : float
The minimum delta mu to use for calculating the derivative dmu/dp.
The default is ``0.5``.
critical_ts : float | None
The critical test-statistic value that should be overcome by the signal
distribution. If set to None, the null-hypothesis test-statistic
distribution will be used to compute the critical TS value.
bkg_kwargs : dict | None
Additional keyword arguments for the `generate_events` method of the
background generation method class. An usual keyword argument is
Expand Down Expand Up @@ -1047,6 +1058,7 @@ def estimate_discovery_potential(
h0_trials=h0_trials,
h0_ts_quantile=h0_ts_quantile,
min_dmu=min_dmu,
critical_ts=critical_ts,
bkg_kwargs=bkg_kwargs,
sig_kwargs=sig_kwargs,
ppbar=ppbar,
Expand Down
26 changes: 26 additions & 0 deletions tests/publicdata_ps/test_time_integrated_ps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from skyllh.core.random import RandomStateService
from skyllh.core.source_model import PointLikeSource
from skyllh.core.timing import TimeLord
from skyllh.core.utils.analysis import estimate_mean_nsignal_for_ts_quantile
from skyllh.datasets.i3 import PublicData_10y_ps

# Setup the logger for this Python module, which has to be done only once
Expand Down Expand Up @@ -69,6 +70,31 @@ def test_unblind(self):
np.testing.assert_allclose(params_dict['ns'], 12.879558, rtol=1e-5)
np.testing.assert_allclose(params_dict['gamma'], 2.122526, rtol=1e-5)

@unittest.skipIf(os.getenv('GITHUB_ACTIONS') == 'true', 'Test designed to run locally, skipped in CI.')
def test_estimate_mean_nsignal_for_ts_quantile(self):
rss = RandomStateService(seed=1)
self.cfg['multiproc']['ncpu'] = 6

with self.tl.task_timer('Call estimate_sensitivity.'):
# Skip h0 trial generation entirely by setting critical_ts value and h0_ts_quantile to None.
(mu, mu_err) = estimate_mean_nsignal_for_ts_quantile(
ana=self.__class__.ana,
rss=rss,
critical_ts=10.0,
h0_ts_quantile=None,
p=0.5,
eps_p=0.1,
mu_range=(1, 2),
tl=self.tl,
)

self.cfg['multiproc']['ncpu'] = None
logger.info(f'{self.tl}')
logger.info(f'mu = {mu}')
logger.info(f'mu_err = {mu_err}')

np.testing.assert_allclose(mu, 6.44, rtol=1e-2)

def test_do_trial(self):
rss = RandomStateService(seed=1)
with self.tl.task_timer('Call do_trial.'):
Expand Down