diff --git a/chainladder/core/common.py b/chainladder/core/common.py index afa35ea7..dd949553 100644 --- a/chainladder/core/common.py +++ b/chainladder/core/common.py @@ -15,18 +15,12 @@ from chainladder.utils.sparse import sp from chainladder.utils.utility_functions import concat -from typing import ( - Callable, - Literal, - TYPE_CHECKING -) +from typing import Callable, Literal, TYPE_CHECKING if TYPE_CHECKING: - from numpy.typing import ArrayLike from chainladder.core.typing import TriangleLike - def _get_full_expectation(cdf_, ultimate_, is_cumulative=True): """Private method that builds full expectation""" full = ultimate_ / cdf_ @@ -95,6 +89,27 @@ def has_zeta(self): @property def cdf_(self): + """Cumulative development factors, ``ldf_`` converted with ``incr_to_cum``. + + Examples + -------- + After fitting a development estimator, ``cdf_`` is the cumulative + product of the selected LDFs, including the tail if one was applied. + + .. testsetup:: + + import chainladder as cl + + .. testcode:: + + import numpy as np + cdf = cl.Development().fit_transform(cl.load_sample('raa')).cdf_ + print(np.round(cdf.values[0, 0, 0, :3], 4).tolist()) + + .. testoutput:: + + [8.9202, 2.974, 1.8318] + """ if not self.has_ldf: x = self.__class__.__name__ raise AttributeError("'" + x + "' object has no attribute 'cdf_'") @@ -103,7 +118,27 @@ def cdf_(self): @property def pct_reported_(self): """Percentage of ultimate reported (or paid) at each development age, - equal to the inverse of the cumulative development factor.""" + equal to the inverse of the cumulative development factor. + + Examples + -------- + At 12 months, RAA volume-weighted development implies about 11% of + ultimate is reported. + + .. testsetup:: + + import chainladder as cl + + .. testcode:: + + import numpy as np + pct = cl.Development().fit_transform(cl.load_sample('raa')).pct_reported_ + print(np.round(pct.values[0, 0, 0, :3], 4).tolist()) + + .. testoutput:: + + [0.1121, 0.3362, 0.5459] + """ if not self.has_ldf: x = self.__class__.__name__ raise AttributeError("'" + x + "' object has no attribute 'pct_reported_'") @@ -115,7 +150,9 @@ def pct_unreported_(self): development age, equal to ``1 - 1 / cdf_``.""" if not self.has_ldf: x = self.__class__.__name__ - raise AttributeError("'" + x + "' object has no attribute 'pct_unreported_'") + raise AttributeError( + "'" + x + "' object has no attribute 'pct_unreported_'" + ) return 1 - 1 / self.cdf_ @property @@ -127,6 +164,27 @@ def cum_zeta_(self): @property def ibnr_(self): + """Outstanding development to ultimate: ``ultimate_`` minus the latest + diagonal (or the origin total, for incremental triangles). + + Examples + -------- + Chainladder IBNR is zero for the oldest origin once that year is fully + developed, and largest for the youngest origin. + + .. testsetup:: + + import chainladder as cl + + .. testcode:: + + ibnr = cl.Chainladder().fit(cl.load_sample('raa')).ibnr_ + print(ibnr.to_frame(origin_as_datetime=False).round(2).iloc[:, 0].tolist()) + + .. testoutput:: + + [nan, 153.95, 617.37, 1636.14, 2746.74, 3649.1, 5435.3, 10907.19, 10649.98, 16339.44] + """ if not hasattr(self, "ultimate_"): x = self.__class__.__name__ raise AttributeError("'" + x + "' object has no attribute 'ibnr_'") @@ -189,25 +247,39 @@ def pipe(self, func, *args, **kwargs): -------- Keep development periods from 48 onward: - >>> import chainladder as cl - >>> raa = cl.load_sample('raa') - >>> raa.pipe(lambda tri: tri.loc[..., 48:]) + .. testsetup:: + + import chainladder as cl + + .. testcode:: + + raa = cl.load_sample('raa') + print(raa.pipe(lambda tri: tri.loc[..., 48:])) + + .. testoutput:: + :options: +NORMALIZE_WHITESPACE + 48 60 72 84 96 108 120 - 1981 11805.0 13539.0 16181.0 18009.0 18608.0 18662.0 18834.0 - 1982 10666.0 13782.0 15599.0 15496.0 16169.0 16704.0 NaN - 1983 16141.0 18735.0 22214.0 22863.0 23466.0 NaN NaN - 1984 21266.0 23425.0 26083.0 27067.0 NaN NaN NaN - 1985 22169.0 25955.0 26180.0 NaN NaN NaN NaN - 1986 12935.0 15852.0 NaN NaN NaN NaN NaN - 1987 12314.0 NaN NaN NaN NaN NaN NaN - 1988 NaN NaN NaN NaN NaN NaN NaN - 1989 NaN NaN NaN NaN NaN NaN NaN - 1990 NaN NaN NaN NaN NaN NaN NaN + 1981 11805.0 13539.0 16181.0 18009.0 18608.0 18662.0 18834.0 + 1982 10666.0 13782.0 15599.0 15496.0 16169.0 16704.0 NaN + 1983 16141.0 18735.0 22214.0 22863.0 23466.0 NaN NaN + 1984 21266.0 23425.0 26083.0 27067.0 NaN NaN NaN + 1985 22169.0 25955.0 26180.0 NaN NaN NaN NaN + 1986 12935.0 15852.0 NaN NaN NaN NaN NaN + 1987 12314.0 NaN NaN NaN NaN NaN NaN + 1988 NaN NaN NaN NaN NaN NaN NaN + 1989 NaN NaN NaN NaN NaN NaN NaN + 1990 NaN NaN NaN NaN NaN NaN NaN """ return func(self, *args, **kwargs) def set_backend( - self, backend: str, inplace: bool = False, deep: bool = False, _warn: bool = True, **kwargs + self, + backend: str, + inplace: bool = False, + deep: bool = False, + _warn: bool = True, + **kwargs, ): """ Converts triangle array_backend. @@ -230,6 +302,27 @@ def set_backend( Returns ------- Triangle with updated array_backend + + Examples + -------- + ``set_backend`` returns a new Triangle unless ``inplace=True``. + + .. testsetup:: + + import chainladder as cl + + .. testcode:: + + raa = cl.load_sample('raa') + print(raa.array_backend) + print(raa.set_backend('sparse').array_backend) + print(raa.array_backend) + + .. testoutput:: + + numpy + sparse + numpy """ # Warn once, at the public entry point, so stacklevel=2 points at the # user's call site rather than an internal recursive call. The _warn @@ -289,13 +382,15 @@ def set_backend( return self else: obj = self.copy() - return obj.set_backend(backend=backend, inplace=True, deep=deep, _warn=False, **kwargs) + return obj.set_backend( + backend=backend, inplace=True, deep=deep, _warn=False, **kwargs + ) @staticmethod def _validate_assumption( - triangle: TriangleLike, - value: str | int | float | list | tuple | set | np.ndarray | dict | Callable, - axis: Literal[0, 1, 2, 3] + triangle: TriangleLike, + value: str | int | float | list | tuple | set | np.ndarray | dict | Callable, + axis: Literal[0, 1, 2, 3], ) -> np.ndarray: """ Used by development estimators to turn user-supplied assumptions into a uniform NumPy array diff --git a/chainladder/core/tests/test_pattern_cum_to_incr.py b/chainladder/core/tests/test_pattern_cum_to_incr.py new file mode 100644 index 00000000..f99e63d0 --- /dev/null +++ b/chainladder/core/tests/test_pattern_cum_to_incr.py @@ -0,0 +1,9 @@ +import chainladder as cl +import numpy as np + + +def test_pattern_cum_to_incr_zero_cells_stay_finite(raa): + cdf = cl.Development().fit(raa).cdf_ + cdf.values[..., 1] = 0 + out = cdf.cum_to_incr() + assert not np.isinf(out.values).any() diff --git a/chainladder/core/triangle.py b/chainladder/core/triangle.py index 69652574..9990cf72 100644 --- a/chainladder/core/triangle.py +++ b/chainladder/core/triangle.py @@ -10,7 +10,12 @@ from chainladder.utils.sparse import sp from chainladder.core.slice import VirtualColumns from chainladder.core.correlation import DevelopmentCorrelation, ValuationCorrelation -from chainladder.utils.utility_functions import concat, num_to_nan, num_to_value, to_period +from chainladder.utils.utility_functions import ( + concat, + num_to_nan, + num_to_value, + to_period, +) from chainladder import options, _warn_dask_parallel_deprecated try: @@ -18,11 +23,7 @@ except ImportError: db = None -from typing import ( - cast, - Optional, - TYPE_CHECKING -) +from typing import cast, Optional, TYPE_CHECKING if TYPE_CHECKING: from pandas import DataFrame, Series @@ -30,7 +31,6 @@ from numpy.typing import ArrayLike from pandas._libs.tslibs.timestamps import Timestamp # noqa from pandas.core.interchange.dataframe_protocol import DataFrame as DataFrameXchg - from sparse import COO class Triangle(TriangleBase): @@ -133,6 +133,11 @@ class Triangle(TriangleBase): Transpose index and columns of object. Only available when Triangle is convertible to DataFrame. + See Also + -------- + Development : Fitted development patterns, including ``ldf_`` and ``cdf_``. + Chainladder : Fitted chainladder results, including ``ultimate_`` and ``ibnr_``. + Examples -------- @@ -442,7 +447,7 @@ def __init__( # If data are present, validate the dimensions. if data is None: return - elif type(data) == dict: + elif isinstance(data, dict): data = pd.DataFrame(data) elif not isinstance(data, pd.DataFrame) and hasattr(data, "__dataframe__"): data = self._interchange_dataframe(data) @@ -456,7 +461,7 @@ def __init__( # Store dimension metadata. self.origin_label: list = origin - + # Handle any ultimate vectors in triangles separately. data, ult = self._split_ult( data=data, @@ -484,7 +489,8 @@ def __init__( if len(development_date.unique()) == 1: # checks if development is not empty, and if ithas any non-yearly values dev_has_no_month = not development or all( - pd.to_numeric(data[col], errors="coerce") + pd + .to_numeric(data[col], errors="coerce") .astype("Int64") .astype(str) .str.fullmatch(r"\d{4}") @@ -498,8 +504,13 @@ def __init__( else: dev_date = pd.to_datetime(development_date.iloc[0]) dev_date_monthly_end = dev_date.to_period("M").to_timestamp(how="e") - period_converted = dev_date_monthly_end.to_period(self.origin_grain).to_timestamp(how="e") - if abs((period_converted - dev_date_monthly_end).total_seconds()) < 1e-6: + period_converted = dev_date_monthly_end.to_period( + self.origin_grain + ).to_timestamp(how="e") + if ( + abs((period_converted - dev_date_monthly_end).total_seconds()) + < 1e-6 + ): self.development_grain = self.origin_grain else: self.development_grain = "M" @@ -510,12 +521,16 @@ def __init__( # Ensure that origin_date values represent the beginning of the period. # i.e., 1990 means the start of 1990. - origin_date: Series = to_period(origin_date,self.origin_grain).dt.to_timestamp(how="s") - + origin_date: Series = to_period(origin_date, self.origin_grain).dt.to_timestamp( + how="s" + ) + # Ensure that development_date values represent the end of the period. # i.e., 1990 means the end of 1990 assuming annual development periods. - development_date: Series = to_period(development_date,self.development_grain).dt.to_timestamp(how="e") - + development_date: Series = to_period( + development_date, self.development_grain + ).dt.to_timestamp(how="e") + # Aggregate dates to the origin/development grains. data_agg: DataFrame = self._aggregate_data( data=data, @@ -524,7 +539,7 @@ def __init__( index=index, columns=columns, ) - + # Fill in missing periods with zeros. date_axes: DataFrame = self._get_date_axes( data_agg["__origin__"], @@ -591,10 +606,12 @@ def __init__( # Coerce malformed triangles to something more predictable. check_origin: np.ndarray = ( - pd.period_range( + pd + .period_range( start=self.odims.min(), end=self.valuation_date, - freq=self.origin_grain.replace("S", "2Q") + ('' if self.origin_grain == "M" else '-' + self.origin_close), + freq=self.origin_grain.replace("S", "2Q") + + ("" if self.origin_grain == "M" else "-" + self.origin_close), ) .to_timestamp() .values @@ -620,30 +637,30 @@ def __init__( ) # Construct Sparse multidimensional array. - self.values: BackendArray = cast("BackendArray", num_to_nan( - sp.COO( - coords, - amts, - prune=True, - has_duplicates=False, - sorted=True, - shape=( - len(self.kdims), - len(self.vdims), - len(self.odims), - len(self.ddims), - ), - ) - )) + self.values: BackendArray = cast( + "BackendArray", + num_to_nan( + sp.COO( + coords, + amts, + prune=True, + has_duplicates=False, + sorted=True, + shape=( + len(self.kdims), + len(self.vdims), + len(self.odims), + len(self.ddims), + ), + ) + ), + ) # Deal with array backend. self.array_backend = "sparse" if array_backend is None: array_backend: str = options.ARRAY_BACKEND if not options.AUTO_SPARSE or array_backend == "cupy": - self.set_backend( - backend=array_backend, - inplace=True - ) + self.set_backend(backend=array_backend, inplace=True) else: self = self._auto_sparse() self._set_slicers() @@ -663,11 +680,7 @@ def __init__( @staticmethod def _split_ult( - data: DataFrame, - index: list, - columns: list, - origin: list, - development: list + data: DataFrame, index: list, columns: list, origin: list, development: list ) -> tuple[DataFrame, Triangle]: """Split ultimate valuation rows from long-format triangle data. @@ -687,7 +700,7 @@ def _split_ult( if ( development and len(development) == 1 - and data[development[0]].dtype.kind == 'M' + and data[development[0]].dtype.kind == "M" ): u = data[data[development[0]] == options.ULT_VAL].copy() if len(u) > 0 and len(u) != len(data): @@ -853,9 +866,9 @@ def development(self): ddims = self.ddims.copy() if self.is_val_tri: formats = {"Y": "%Y", "S": "%YQ%q", "Q": "%YQ%q", "M": "%Y-%m"} - ddims = ddims.to_period(freq=self.development_grain.replace("S", "2Q")).strftime( - formats[self.development_grain] - ) + ddims = ddims.to_period( + freq=self.development_grain.replace("S", "2Q") + ).strftime(formats[self.development_grain]) elif self.is_pattern: offset = self._dstep()["M"][self.development_grain] if self.is_ultimate: @@ -924,7 +937,7 @@ def is_val_tri(self): True """ - return type(self.ddims) == pd.DatetimeIndex + return isinstance(self.ddims, pd.DatetimeIndex) @property def is_full(self) -> bool: @@ -969,7 +982,6 @@ def is_full(self) -> bool: return self.nan_triangle.sum().sum() == np.prod(self.shape[-2:]) - @property def is_pattern(self) -> bool: """ @@ -1033,15 +1045,17 @@ def is_disposal_rate(self) -> bool: def is_disposal_rate(self, is_dr: bool) -> None: self._is_disposal_rate = is_dr - def align_pattern(self, X:Triangle, sample_weight:Triangle|None=None) -> Triangle: - """ + def align_pattern( + self, X: Triangle, sample_weight: Triangle | None = None + ) -> Triangle: + """ Vertically align a selected pattern to origin period latest diagonal. Triangle must be a selected pattern. Parameters ---------- X: Triangle The target triangle to align to - + sample_weight: Triangle, option (default=None) Exposure triangle @@ -1052,7 +1066,9 @@ def align_pattern(self, X:Triangle, sample_weight:Triangle|None=None) -> Triangl """ if not self._pattern: - raise ValueError("Triangle is not a selected pattern, such as .ldf_ or .cdf_") + raise ValueError( + "Triangle is not a selected pattern, such as .ldf_ or .cdf_" + ) valuation = X.valuation_date pattern = self.iloc[..., : X.shape[-1]] a = X.iloc[0, 0] * 0 @@ -1066,9 +1082,9 @@ def align_pattern(self, X:Triangle, sample_weight:Triangle|None=None) -> Triangl pattern = X / X * pattern pattern.valuation_date = valuation return pattern.latest_diagonal - + @property - def is_ultimate(self) -> bool: + def is_ultimate(self) -> bool: """ Indicates whether the Triangle includes an ultimate valuation column. @@ -1370,9 +1386,16 @@ def incr_to_cum(self, inplace=False): else: values = xp.nan_to_num(self.values) nan_triangle = xp.nan_to_num(self.nan_triangle) - l1 = lambda i: values[..., 0 : i + 1] - l2 = lambda i: l1(i) * nan_triangle[..., i : i + 1] - l3 = lambda i: l2(i).sum(3, keepdims=True) + + def l1(i): + return values[..., 0 : i + 1] + + def l2(i): + return l1(i) * nan_triangle[..., i : i + 1] + + def l3(i): + return l2(i).sum(3, keepdims=True) + if db: _warn_dask_parallel_deprecated() bag = db.from_sequence(range(self.shape[-1])) @@ -1431,8 +1454,7 @@ def cum_to_incr(self, inplace=False): if self.is_cumulative or self.is_cumulative is None: if self.is_pattern & (not self.is_disposal_rate): xp = self.get_array_module() - self.values = xp.nan_to_num(self.values) - values = num_to_value(self.values, 1) + self.values = num_to_value(xp.nan_to_num(self.values), 1) diff = self.iloc[..., :-1] / self.iloc[..., 1:].values self = concat( ( @@ -1484,7 +1506,7 @@ def _val_dev(self, sign, inplace=False): ) ddims = np.max([np.max(obj.values.coords[-1]) + 1, ddims]) obj.values.shape = tuple(list(obj.shape[:-1]) + [ddims]) - if options.AUTO_SPARSE == False or backend == "cupy": + if not options.AUTO_SPARSE or backend == "cupy": obj = obj.set_backend(backend) else: obj = obj._auto_sparse() @@ -1794,7 +1816,8 @@ def grain(self, grain="", trailing=False, inplace=False): origin_period_end = "DEC" indices = ( - pd.Series(range(len(self.origin)), index=self.origin) + pd + .Series(range(len(self.origin)), index=self.origin) .resample("-".join([freq, origin_period_end])) .indices ) @@ -1808,17 +1831,21 @@ def grain(self, grain="", trailing=False, inplace=False): d_start = pd.Period( obj.valuation[0], - freq=dgrain_old.replace("S", "2Q") + ('' if dgrain_old == "M" else obj.origin.freqstr[-4:]), + freq=dgrain_old.replace("S", "2Q") + + ("" if dgrain_old == "M" else obj.origin.freqstr[-4:]), ).to_timestamp(how="s") if dgrain_old == "S": - d_start = d_start + pd.DateOffset(months=-3) + d_start = d_start + pd.DateOffset(months=-3) if len(obj.ddims) > 1 and obj.origin.to_timestamp(how="s")[0] != d_start: addl_ts = ( - pd.period_range(obj.odims[0], obj.valuation[0], freq=dgrain_old.replace("S","2Q"))[ - :-1 - ] + pd + .period_range( + obj.odims[0], + obj.valuation[0], + freq=dgrain_old.replace("S", "2Q"), + )[:-1] .to_timestamp() .values ) @@ -1826,7 +1853,7 @@ def grain(self, grain="", trailing=False, inplace=False): addl.ddims = addl_ts obj = concat((addl, obj), axis=-1) obj.values = num_to_nan(obj.values) - + if dgrain_old != dgrain_new and obj.shape[-1] > 1: step = self._dstep()[dgrain_old][dgrain_new] d = np.sort( @@ -1842,7 +1869,7 @@ def grain(self, grain="", trailing=False, inplace=False): obj.ddims = ddims obj.development_grain = dgrain_new - + obj = obj.dev_to_val() if self.is_val_tri else obj.val_to_dev() if inplace: diff --git a/chainladder/development/development.py b/chainladder/development/development.py index 3e520a33..fc16e105 100644 --- a/chainladder/development/development.py +++ b/chainladder/development/development.py @@ -80,11 +80,11 @@ class Development(DevelopmentBase): index will receive its own patterns. .. note :: - + (Order of Drop Operations) - + When multiple drop parameters are used together, the weights are built in this order: - + 1. ``n_periods`` — limit to the most recent origin periods. 2. ``drop`` — remove specific origin/development cells. 3. ``drop_valuation`` — remove entire valuation diagonal in the triangle. @@ -102,6 +102,9 @@ class Development(DevelopmentBase): The estimated loss development patterns cdf_: Triangle The estimated cumulative development patterns + pct_reported_: Triangle + The estimated percent of ultimate reported (or paid) at each + development age sigma_: Triangle Sigma of the ldf regression std_err_: Triangle @@ -396,14 +399,14 @@ def fit(self, X: TriangleLike, y: None = None, sample_weight: None = None): link_ratio: ArrayLike = y / x tw = TriangleWeight( - n_periods = self.n_periods, - drop_high = self.drop_high, - drop_low = self.drop_low, - drop_above = self.drop_above, - drop_below = self.drop_below, - drop_valuation = self.drop_valuation, - preserve = self.preserve, - drop = self.drop + n_periods=self.n_periods, + drop_high=self.drop_high, + drop_low=self.drop_low, + drop_above=self.drop_above, + drop_below=self.drop_below, + drop_valuation=self.drop_valuation, + preserve=self.preserve, + drop=self.drop, ) if hasattr(X, "w_v2_"): diff --git a/docs/_templates/autosummary/class.rst b/docs/_templates/autosummary/class.rst index 1f7dfc43..9b261d0b 100644 --- a/docs/_templates/autosummary/class.rst +++ b/docs/_templates/autosummary/class.rst @@ -2,10 +2,13 @@ .. currentmodule:: {{ module }} +{% set documented_attrs = ['cdf_', 'ibnr_', 'pct_reported_'] %} +{% set hidden_attrs = attributes | reject('in', documented_attrs) | list %} + .. autoclass:: {{ objname }} :members: :undoc-members: - :exclude-members: set_fit_request, set_predict_request, set_score_request, set_transform_request, {{ attributes | join(', ') }} + :exclude-members: set_fit_request, set_predict_request, set_score_request, set_transform_request{% if hidden_attrs %}, {{ hidden_attrs | join(', ') }}{% endif %} {% set inherited = [] %} {% for method in methods %} diff --git a/pyproject.toml b/pyproject.toml index 63325e22..d2b4fd10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,7 +121,6 @@ select = ["E2", "E4", "E7", "E9", "F"] "chainladder/adjustments/tests/test_bootstrap.py" = ["E231"] "chainladder/adjustments/tests/test_disposal.py" = ["E226", "E231", "E241", "E251", "E265", "F841"] "chainladder/adjustments/trend.py" = ["F401"] -"chainladder/core/common.py" = ["F401"] "chainladder/core/correlation.py" = ["E741"] "chainladder/core/display.py" = ["E203", "E252"] "chainladder/core/slice.py" = ["E225"] @@ -130,12 +129,10 @@ select = ["E2", "E4", "E7", "E9", "F"] "chainladder/core/tests/test_display.py" = ["E722"] "chainladder/core/tests/test_grain.py" = ["E265", "F401", "F841"] "chainladder/core/tests/test_slicing.py" = ["E203", "E225"] -"chainladder/core/triangle.py" = ["E222", "E227", "E231", "E252", "E712", "E721", "E731", "F401", "F841"] "chainladder/development/barnzehn.py" = ["E201", "E202", "E231", "E251", "E275"] "chainladder/development/base.py" = ["E712", "F401", "F841"] "chainladder/development/clark.py" = ["E721", "E731"] "chainladder/development/constant.py" = ["E712"] -"chainladder/development/development.py" = ["E251"] "chainladder/development/glm.py" = ["E225", "E231", "E251", "F401"] "chainladder/development/incremental.py" = ["E226", "E231", "E251", "E265", "E721", "F401"] "chainladder/development/learning.py" = ["E225", "E226", "E231", "E265", "E711", "F401"]