generalized trend method and lots of doc improvements

README.md

@@ -39,17 +39,3 @@ Alternatively, install directly from github:
 
 Note: This package requires Python 3.5 and later, numpy 1.12.0 and later,
 pandas 0.23.0 and later, scikit-learn 0.18.0 and later.
-
-## GPU support
-New in version `0.5.0` - `chainladder` now supports CUDA-based GPU computations by way of [CuPY](https://github.com/cupy/cupy).  You can now swap `array_backend` between `numpy` and `cupy` to switch between CPU and GPU-based computations.
-
-Array backends can be set globally:
-```python
-import chainladder as cl
-cl.array_backend('cupy')
-```
-Alternatively, they can be set per `Triangle` instance.
-```python
-cl.Triangle(..., array_backend='cupy')
-```
-**Note** you must have a CUDA-enabled graphics card and [CuPY](https://github.com/cupy/cupy) installed to use the GPU backend.

chainladder/__init__.py

@@ -11,4 +11,4 @@ def array_backend(array_backend='numpy'):
 from chainladder.methods import * # noqa (API Import)
 from chainladder.workflow import * # noqa (API Import)
 
-__version__ = '0.7.0'
+__version__ = '0.7.1'

chainladder/core/triangle.py

@@ -6,7 +6,7 @@
 import numpy as np
 from chainladder.utils.cupy import cp
 import copy
-
+import warnings
 from chainladder.core.base import TriangleBase
 from chainladder.core.correlation import DevelopmentCorrelation, ValuationCorrelation
 
@@ -346,7 +346,7 @@ def _val_dev_chg(self, kind):
         obj = copy.deepcopy(self)
         if hasattr(obj, '_nan_triangle_'):
             del obj._nan_triangle_
-        o_vals = obj._expand_dims(xp.arange(len(obj.origin))[:, xp.newaxis])
+        o_vals = obj._expand_dims(xp.arange(len(obj.origin))[:, None])
         if self.shape[-1] == 1:
             return obj
         if kind == 'val_to_dev':
@@ -371,11 +371,11 @@ def _val_dev_chg(self, kind):
             val = np.unique(np.array(list(zip(val[0], val[1]))), axis=0)
             arr = xp.expand_dims(obj.values[:, :, val[:, 0], val[:, 1]], -1)
             if val[0, 0] != 0:
-                prepend = obj._expand_dims(xp.array([xp.nan]*(val[0, 0]))[:, xp.newaxis])
+                prepend = obj._expand_dims(xp.array([xp.nan]*(val[0, 0]))[:, None])
                 arr = xp.concatenate((prepend, arr), -2)
             if len(obj.origin)-1-val[-1, 0] != 0:
                 append = obj._expand_dims(
-                    xp.array([np.nan]*(len(obj.origin)-1-val[-1, 0]))[:, xp.newaxis])
+                    xp.array([np.nan]*(len(obj.origin)-1-val[-1, 0]))[:, None])
                 arr = xp.concatenate((arr, append), -2)
             if obj.is_cumulative and old_arr is not None:
                 arr = xp.isnan(arr)*xp.nan_to_num(old_arr) + xp.nan_to_num(arr)
@@ -454,7 +454,7 @@ def grain(self, grain='', trailing=False, inplace=False):
             o_bool = np.repeat((o == o_new)[:, np.newaxis],
                                len(obj.ddims), axis=1)
             o_bool = obj._expand_dims(o_bool)
-            new_tri = xp.repeat(xp.nan_to_num(obj.values)[..., xp.newaxis],
+            new_tri = xp.repeat(xp.nan_to_num(obj.values)[..., None],
                                 o_bool.shape[-1], axis=-1)
             new_tri[~np.isfinite(new_tri)] = 0
             new_tri = np.swapaxes(np.sum(new_tri*o_bool, axis=2), -1, -2)
@@ -494,18 +494,23 @@ def grain(self, grain='', trailing=False, inplace=False):
             return obj
 
 
-    def trend(self, trend=0.0, axis='origin', valuation_date=None, ultimate_lag=None):
-        """  Allows for the trending of a Triangle object or an origin vector.
-        This method trends using days and assumes a years is 365.25 days long.
+    def trend(self, trend=0.0, axis='origin', start=None, end=None, ultimate_lag=None, **kwargs):
+        """  Allows for the trending of a Triangle object along either a valuation
+        or origin axis.  This method trends using days and assumes a years is
+        365.25 days long.
 
         Parameters
         ----------
         trend : float
             The annual amount of the trend. Use 1/(1+trend)-1 to detrend.
         axis : str (options: ['origin', 'valuation'])
             The axis on which to apply the trend
-        valuation_date: date
-            The terminal date from which trend should be calculated.
+        start: date
+            The terminal date from which trend should be calculated. If none is
+            provided then the latest date of the triangle is used.
+        end: date
+            The terminal date to which the trend should be calculated. If none is
+            provided then the earliest period of the triangle is used.
         ultimate_lag : int
             If ultimate valuations are in the triangle, you can set the overall
             age of the ultimate to be some lag from the latest non-Ultimate
@@ -516,15 +521,33 @@ def trend(self, trend=0.0, axis='origin', valuation_date=None, ultimate_lag=None
         Triangle
             updated with multiplicative trend applied.
         """
+        if kwargs.get('valuation_date', None):
+            start = kwargs['valuation_date']
+            warnings.warn('valuation_date is deprecated, and will be removed. Use start instead.')
+
+        def val_vector(start, end, valuation):
+            if end < start:
+                val_start = xp.maximum(valuation, start)
+                val_end = xp.maximum(valuation, end)
+            else:
+                val_start = xp.minimum(valuation, start)
+                val_end = xp.minimum(valuation, end)
+            return val_start, val_end
+
         xp = cp.get_array_module(self.values)
-        if not valuation_date:
-            days = np.datetime64(self.valuation_date)
+        if not start:
+            start = np.datetime64(self.valuation_date)
+        else:
+            start = np.datetime64(start)
+        if not end:
+            end = np.datetime64(xp.min(self.valuation))
         else:
-            days = np.datetime64(valuation_date)
+            end = np.datetime64(end)
         if axis == 'origin':
+            val_start, val_end = val_vector(start, end, self.origin.end_time.values)
             trend = xp.array((1 + trend)**-(
-                pd.Series(self.origin.end_time.values-days).dt.days/365.25)
-                )[xp.newaxis, xp.newaxis, ..., xp.newaxis]
+                pd.Series(val_end-val_start).dt.days/365.25)
+                )[None, None, ..., None]
         elif axis == 'valuation':
             valuation  = self.valuation
             if self.is_ultimate and ultimate_lag is not None:
@@ -533,8 +556,9 @@ def trend(self, trend=0.0, axis='origin', valuation_date=None, ultimate_lag=None
                     self.valuation.values.reshape(self.shape[-2:], order='f'))
                 val_df.iloc[:, -1] = val_df.iloc[:, -2] + unit_lag * ultimate_lag
                 valuation = pd.PeriodIndex(val_df.unstack().values)
+            val_start, val_end = val_vector(start, end, valuation)
             trend = (1 + trend)**-(
-                pd.Series(valuation-days)
+                pd.Series(val_end-val_start)
                 .dt.days.values.reshape(self.shape[-2:], order='f')/365.25)
         obj = copy.deepcopy(self)
         obj.values = obj.values*trend

chainladder/development/base.py

@@ -185,10 +185,10 @@ def _drop_valuation(self, X):
         dfill = X.shape[-1]-arr.shape[-1]
         if ofill > 0:
             arr = xp.concatenate((arr, xp.repeat(
-                xp.ones(arr.shape[-1])[xp.newaxis], ofill, 0)), 0)
+                xp.ones(arr.shape[-1])[None], ofill, 0)), 0)
         if dfill > 0:
             arr = xp.concatenate((arr, xp.repeat(
-                xp.ones(arr.shape[-2])[..., xp.newaxis], dfill, -1)), -1)
+                xp.ones(arr.shape[-2])[..., None], dfill, -1)), -1)
         return arr[:, :-1]
 
     def _drop(self, X):
@@ -240,7 +240,7 @@ def fit(self, X, y=None, sample_weight=None):
         val = xp.array([weight_dict.get(item.lower(), 1)
                         for item in average])
         for i in [2, 1, 0]:
-            val = xp.repeat(val[xp.newaxis], tri_array.shape[i], axis=0)
+            val = xp.repeat(val[None], tri_array.shape[i], axis=0)
         val = xp.nan_to_num(val * (y * 0 + 1))
         if xp == cp:
             link_ratio = y / x

chainladder/development/constant.py

@@ -56,7 +56,7 @@ def fit(self, X, y=None, sample_weight=None):
         ldf = xp.array([float(self.patterns[item]) for item in obj.ddims[:-1]])
         if self.style == 'cdf':
             ldf = xp.concatenate((ldf[:-1]/ldf[1:], xp.array([ldf[-1]])))
-        ldf = ldf[xp.newaxis, xp.newaxis, xp.newaxis, ...]
+        ldf = ldf[None, None, None, ...]
         obj.values = obj.values * ldf
         obj.ddims = X.link_ratio.ddims
         obj.valuation = obj._valuation_triangle(obj.ddims)

chainladder/development/incremental.py

@@ -74,8 +74,8 @@ def fit(self, X, y=None, sample_weight=None):
             xp.nan_to_num(
                 x._get_latest_diagonal(compress=False).values[0, 0, ...]*0+1)
         obj.values = (1+self.trend) ** \
-            xp.flip((xp.abs(xp.arange(obj.shape[-2])[xp.newaxis].T -
-                     xp.arange(obj.shape[-2])[xp.newaxis])), 0)*y_*keeps
+            xp.flip((xp.abs(xp.arange(obj.shape[-2])[None].T -
+                     xp.arange(obj.shape[-2])[None])), 0)*y_*keeps
         obj.values = obj.values*(X._expand_dims(1-xp.nan_to_num(x._nan_triangle()))) + \
             xp.nan_to_num((X.cum_to_incr()/sample_weight).values)
         obj.values[obj.values == 0] = xp.nan

chainladder/development/munich.py

@@ -126,13 +126,13 @@ def _get_p_to_i_object(self, obj):
         incurred = obj[[item[1] for item in p_to_i][0]]
         for item in [item[1] for item in p_to_i][1:]:
             incurred[item] = obj[item]
-        paid = paid.values[xp.newaxis]
-        incurred = incurred.values[xp.newaxis]
+        paid = paid.values[None]
+        incurred = incurred.values[None]
         return xp.concatenate((paid, incurred), axis=0)
 
     def _p_to_i_concate(self, obj_p, obj_i):
         xp = cp.get_array_module(obj_p)
-        return xp.concatenate((obj_p[xp.newaxis], obj_i[xp.newaxis]), 0)
+        return xp.concatenate((obj_p[None], obj_i[None]), 0)
 
     def _get_MCL_model(self, X):
         xp = cp.get_array_module(X.values)
@@ -178,7 +178,7 @@ def _get_MCL_lambda(self):
         lambdaP = WeightedRegression(thru_orig=True, axis=-1).fit(
             xp.reshape(self.q_resid_[0][..., :-1, :-1], (k, v, o*d)),
             xp.reshape(self.residual_[0], (k, v, o*d)), w).slope_
-        return self._p_to_i_concate(lambdaP, lambdaI)[..., xp.newaxis]
+        return self._p_to_i_concate(lambdaP, lambdaI)[..., None]
 
     @property
     def munich_full_triangle_(self):

chainladder/methods/benktander.py

@@ -96,7 +96,7 @@ def _get_ultimate_(self, X, sample_weight, obj):
         ult.values = \
             obj.cdf_.values[..., :ult.shape[development]]*(ult.values*0+1)
         cdf = ult.latest_diagonal.values
-        cdf = (1-1/cdf)[xp.newaxis]
+        cdf = (1-1/cdf)[None]
         exponents = xp.arange(self.n_iters+1)
         exponents = xp.reshape(exponents, tuple([len(exponents)]+[1]*4))
         cdf = cdf**exponents

chainladder/methods/bornferg.py

@@ -35,7 +35,7 @@ class BornhuetterFerguson(Benktander):
     Smoothing chainladder ultimates by using them as apriori figures in the
     Bornhuetter Ferguson method.
 
-    >>> raa = cl.load_dataset('RAA')
+    >>> raa = cl.load_sample('RAA')
     >>> cl_ult = cl.Chainladder().fit(raa).ultimate_ # Chainladder Ultimate
     >>> apriori = cl_ult*0+(cl_ult.sum()/10)[0] # Mean Chainladder Ultimate
     >>> cl.BornhuetterFerguson(apriori=1).fit(raa, sample_weight=apriori).ultimate_

chainladder/methods/capecod.py

@@ -73,18 +73,18 @@ def _get_ultimate_(self, X, sample_weight, obj):
         reported_exposure = exposure/cdf
         trend_exponent = len_orig-xp.arange(len_orig)-1
         trend_array = (1+self.trend)**(trend_exponent)
-        trend_array = X._expand_dims(trend_array[..., xp.newaxis])
+        trend_array = X._expand_dims(trend_array[..., None])
         decay_matrix = self.decay ** xp.abs(
-            xp.arange(len_orig)[xp.newaxis].T -
-            xp.arange(len_orig)[xp.newaxis])
+            xp.arange(len_orig)[None].T -
+            xp.arange(len_orig)[None])
         decay_matrix = X._expand_dims(decay_matrix)
         weighted_exposure = \
             xp.swapaxes(reported_exposure, development, origin)*decay_matrix
         trended_ultimate = (latest*trend_array)/reported_exposure
         trended_ultimate = xp.swapaxes(trended_ultimate, development, origin)
         apriori = xp.sum(weighted_exposure*trended_ultimate, development) / \
             xp.sum(weighted_exposure, development)
-        ult.values = apriori[..., xp.newaxis]
+        ult.values = apriori[..., None]
         ult.ddims = np.array([None])
         apriori_ = copy.copy(ult)
         detrended_ultimate = apriori_.values/trend_array

chainladder/methods/chainladder.py

@@ -34,7 +34,7 @@ class Chainladder(MethodBase):
     --------
     Comparing the ultimates for a company using company LDFs vs industry LDFs
 
-    >>> clrd = cl.load_dataset('clrd')['CumPaidLoss']
+    >>> clrd = cl.load_sample('clrd')['CumPaidLoss']
     >>> clrd = clrd[clrd['LOB'] == 'wkcomp']
     >>> industry = clrd.sum()
     >>> allstate_industry_cl = cl.Chainladder().fit(industry).predict(clrd).ultimate_.loc['Allstate Ins Co Grp']

chainladder/methods/mack.py

@@ -100,7 +100,7 @@ def total_process_risk_(self):
     def _mack_recursion(self, est):
         obj = copy.copy(self.X_)
         xp = cp.get_array_module(obj.values)
-        nans = self.X_._nan_triangle()[xp.newaxis, xp.newaxis]
+        nans = self.X_._nan_triangle()[None, None]
         nans = nans * xp.ones(self.X_.shape)
         nans = xp.concatenate(
             (nans, xp.ones((*self.X_.shape[:3], 1))*xp.nan), 3)

chainladder/utils/tests/test_exhibits.py

@@ -4,7 +4,7 @@
 
 
 def test_simple_exhibit():
-    raa = cl.load_dataset('raa')
+    raa = cl.load_sample('raa')
 
     col = cl.Column(
         cl.DataFrame(raa, margin=(0,0,1,0)),

chainladder/utils/tests/test_utilities.py

@@ -14,7 +14,7 @@ def test_vertical_line():
     assert abs(olf.loc['2017'].iloc[0] - ((1-184/365)*.2+1)) < .00001
 
 def test_triangle_json_io():
-    clrd = cl.load_dataset('clrd')
+    clrd = cl.load_sample('clrd')
     xp = cp.get_array_module(clrd.values)
     clrd2 = cl.read_json(clrd.to_json())
     xp.testing.assert_array_equal(clrd.values, clrd2.values)
@@ -25,8 +25,8 @@ def test_triangle_json_io():
     assert np.all(clrd.valuation == clrd2.valuation)
 
 def test_json_for_val():
-    x = cl.load_dataset('raa').dev_to_val().to_json()
-    assert cl.read_json(x) == cl.load_dataset('raa').dev_to_val()
+    x = cl.load_sample('raa').dev_to_val().to_json()
+    assert cl.read_json(x) == cl.load_sample('raa').dev_to_val()
 
 def test_estimator_json_io():
     assert cl.read_json(cl.Development().to_json()).get_params() == \
@@ -42,6 +42,6 @@ def test_pipeline_json_io():
             for item in pipe2.get_params()['steps']}
 
 def test_concat():
-    tri = cl.load_dataset('clrd').groupby('LOB').sum()
+    tri = cl.load_sample('clrd').groupby('LOB').sum()
     assert cl.concat([tri.loc['wkcomp'], tri.loc['comauto']], axis=0) == \
            tri.loc[['wkcomp', 'comauto']]

docs/auto_examples/plot_mack.ipynb

@@ -26,7 +26,7 @@
       },
       "outputs": [],
       "source": [
-        "import pandas as pd\nimport chainladder as cl\n\n# Load the data\ndata = cl.load_sample('raa')\n\n# Compute Mack Chainladder ultimates and Std Err using 'simple' average\nmack = cl.MackChainladder()\ndev = cl.Development(average='volume')\nmack.fit(dev.fit_transform(data))\n\n# Plotting\nplot_data = mack.summary_.to_frame()\ng = plot_data[['Latest', 'IBNR']].plot(\n    kind='bar', stacked=True, ylim=(0, None), grid=True,\n    yerr=pd.DataFrame({'latest': plot_data['Mack Std Err']*0,\n                       'IBNR': plot_data['Mack Std Err']}),\n    title='Mack Chainladder Ultimate').set(\n    xlabel='Accident Year', ylabel='Loss');"
+        "import pandas as pd\nimport chainladder as cl\n\n# Load the data\ndata = cl.load_sample('raa')\n\n# Compute Mack Chainladder ultimates and Std Err using 'volume' average\nmack = cl.MackChainladder()\ndev = cl.Development(average='volume')\nmack.fit(dev.fit_transform(data))\n\n# Plotting\nplot_data = mack.summary_.to_frame()\ng = plot_data[['Latest', 'IBNR']].plot(\n    kind='bar', stacked=True, ylim=(0, None), grid=True,\n    yerr=pd.DataFrame({'latest': plot_data['Mack Std Err']*0,\n                       'IBNR': plot_data['Mack Std Err']}),\n    title='Mack Chainladder Ultimate').set(\n    xlabel='Accident Year', ylabel='Loss');"
       ]
     }
   ],

docs/auto_examples/plot_mack.py

@@ -11,7 +11,7 @@
 # Load the data
 data = cl.load_sample('raa')
 
-# Compute Mack Chainladder ultimates and Std Err using 'simple' average
+# Compute Mack Chainladder ultimates and Std Err using 'volume' average
 mack = cl.MackChainladder()
 dev = cl.Development(average='volume')
 mack.fit(dev.fit_transform(data))

docs/auto_examples/plot_mack.py.md5

@@ -1 +1 @@
-3804a12e26dd770db8c2de1329d025ea
+86dc57b9e0477273d17b585407ff738a

docs/auto_examples/plot_mack.rst

@@ -34,7 +34,7 @@ This example demonstrates how you can can use the Mack Chainladder method.
     # Load the data
     data = cl.load_sample('raa')
 
-    # Compute Mack Chainladder ultimates and Std Err using 'simple' average
+    # Compute Mack Chainladder ultimates and Std Err using 'volume' average
     mack = cl.MackChainladder()
     dev = cl.Development(average='volume')
     mack.fit(dev.fit_transform(data))
@@ -51,7 +51,7 @@ This example demonstrates how you can can use the Mack Chainladder method.
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 0 minutes  0.470 seconds)
+   **Total running time of the script:** ( 0 minutes  0.569 seconds)
 
 
 .. _sphx_glr_download_auto_examples_plot_mack.py:

docs/auto_examples/sg_execution_times.rst

@@ -5,17 +5,19 @@
 
 Computation times
 =================
-**00:00.586** total execution time for **auto_examples** files:
+**00:00.569** total execution time for **auto_examples** files:
 
 +-----------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_auto_examples_plot_bondy_sensitivity.py` (``plot_bondy_sensitivity.py``)       | 00:00.586 | 0.0 MB |
+| :ref:`sphx_glr_auto_examples_plot_mack.py` (``plot_mack.py``)                                 | 00:00.569 | 0.0 MB |
 +-----------------------------------------------------------------------------------------------+-----------+--------+
 | :ref:`sphx_glr_auto_examples_plot_ave_analysis.py` (``plot_ave_analysis.py``)                 | 00:00.000 | 0.0 MB |
 +-----------------------------------------------------------------------------------------------+-----------+--------+
 | :ref:`sphx_glr_auto_examples_plot_benktander.py` (``plot_benktander.py``)                     | 00:00.000 | 0.0 MB |
 +-----------------------------------------------------------------------------------------------+-----------+--------+
 | :ref:`sphx_glr_auto_examples_plot_bf_apriori_from_cl.py` (``plot_bf_apriori_from_cl.py``)     | 00:00.000 | 0.0 MB |
 +-----------------------------------------------------------------------------------------------+-----------+--------+
+| :ref:`sphx_glr_auto_examples_plot_bondy_sensitivity.py` (``plot_bondy_sensitivity.py``)       | 00:00.000 | 0.0 MB |
++-----------------------------------------------------------------------------------------------+-----------+--------+
 | :ref:`sphx_glr_auto_examples_plot_bootstrap.py` (``plot_bootstrap.py``)                       | 00:00.000 | 0.0 MB |
 +-----------------------------------------------------------------------------------------------+-----------+--------+
 | :ref:`sphx_glr_auto_examples_plot_bootstrap_comparison.py` (``plot_bootstrap_comparison.py``) | 00:00.000 | 0.0 MB |
@@ -38,8 +40,6 @@ Computation times
 +-----------------------------------------------------------------------------------------------+-----------+--------+
 | :ref:`sphx_glr_auto_examples_plot_industry_to_company.py` (``plot_industry_to_company.py``)   | 00:00.000 | 0.0 MB |
 +-----------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_auto_examples_plot_mack.py` (``plot_mack.py``)                                 | 00:00.000 | 0.0 MB |
-+-----------------------------------------------------------------------------------------------+-----------+--------+
 | :ref:`sphx_glr_auto_examples_plot_munich.py` (``plot_munich.py``)                             | 00:00.000 | 0.0 MB |
 +-----------------------------------------------------------------------------------------------+-----------+--------+
 | :ref:`sphx_glr_auto_examples_plot_munich_resid.py` (``plot_munich_resid.py``)                 | 00:00.000 | 0.0 MB |