Note
Click here to download the full example code
Benchmarking
You can easily compare predictive performance of multiple algorithms such as
Silverkite and Prophet using the
BenchmarkForecastConfig class.
In this tutorial we describe the step-by-step process of defining, running and monitoring a benchmark.
We also demonstrate how to use the class functions to compute and plot errors for multiple models.
12 from dataclasses import replace
13
14 import plotly
15 import plotly.graph_objects as go
16
17 from greykite.common.evaluation import EvaluationMetricEnum
18 from greykite.framework.benchmark.benchmark_class import BenchmarkForecastConfig
19 from greykite.framework.benchmark.data_loader_ts import DataLoaderTS
20 from greykite.framework.templates.autogen.forecast_config import ComputationParam
21 from greykite.framework.templates.autogen.forecast_config import EvaluationMetricParam
22 from greykite.framework.templates.autogen.forecast_config import EvaluationPeriodParam
23 from greykite.framework.templates.autogen.forecast_config import MetadataParam
24 from greykite.framework.templates.autogen.forecast_config import ForecastConfig
25 from greykite.framework.templates.autogen.forecast_config import ModelComponentsParam
26 from greykite.framework.templates.model_templates import ModelTemplateEnum
27 from greykite.sklearn.cross_validation import RollingTimeSeriesSplit
Load the data
First load your dataset into a pandas dataframe. We will use the peyton-manning dataset as a running example.
Define the Configs
We specify the models we want to benchmark via the configs parameter.
In this example we will benchmark 1 Prophet and 2 different Silverkite models.
We first define the common components of the models
such as MetadataParam and EvaluationMetricParam, and then update the configuration to specify
individual models.
49 ## Define common components of the configs
50 # Specifies dataset information
51 metadata = MetadataParam(
52 time_col="ts", # name of the time column
53 value_col="y", # name of the value column
54 freq="D" # "H" for hourly, "D" for daily, "W" for weekly, etc.
55 )
56
57 # Defines number of periods to forecast into the future
58 forecast_horizon = 7
59
60 # Specifies intended coverage of the prediction interval
61 coverage = 0.95
62
63 # Defines the metrics to evaluate the forecasts
64 # We use Mean Absolute Percent Error (MAPE) in this tutorial
65 evaluation_metric = EvaluationMetricParam(
66 cv_selection_metric=EvaluationMetricEnum.MeanAbsolutePercentError.name,
67 cv_report_metrics=None
68 )
69
70 # Defines the cross-validation config within pipeline
71 evaluation_period = EvaluationPeriodParam(
72 cv_max_splits=1, # Benchmarking n_splits is defined in tscv, here we don't need split to choose parameter sets
73 periods_between_train_test=0,
74 )
75
76 # Defines parameters related to grid-search computation
77 computation = ComputationParam(
78 hyperparameter_budget=None,
79 n_jobs=-1, # to debug, change to 1 for more informative error messages
80 verbose=3)
81
82 # Defines common components across all the configs
83 # ``model_template`` and ``model_components_param`` changes between configs
84 common_config = ForecastConfig(
85 metadata_param=metadata,
86 forecast_horizon=forecast_horizon,
87 coverage=coverage,
88 evaluation_metric_param=evaluation_metric,
89 evaluation_period_param=evaluation_period,
90 computation_param=computation,
91 )
Now we update common_config to specify the individual models.
96 # # The following code defines a ``Prophet`` configuration.
97 # # Defines ``Prophet`` model template with custom seasonality
98 # model_components = ModelComponentsParam(
99 # seasonality={
100 # "seasonality_mode": ["additive"],
101 # "yearly_seasonality": ["auto"],
102 # "weekly_seasonality": [True],
103 # },
104 # growth={
105 # "growth_term": ["linear"]
106 # }
107 # )
108 # param_update = dict(
109 # model_template=ModelTemplateEnum.PROPHET.name,
110 # model_components_param=model_components
111 # )
112 # Prophet = replace(common_config, **param_update)
113
114 # Defines ``Silverkite`` model template with automatic autoregression
115 # and changepoint detection
116 model_components = ModelComponentsParam(
117 changepoints={
118 "changepoints_dict": {
119 "method": "auto",
120 }
121 },
122 autoregression={
123 "autoreg_dict": "auto"
124 }
125 )
126 param_update = dict(
127 model_template=ModelTemplateEnum.SILVERKITE.name,
128 model_components_param=model_components
129 )
130 Silverkite_1 = replace(common_config, **param_update)
131
132 # Defines ``Silverkite`` model template via string encoding
133 param_update = dict(
134 model_template="DAILY_SEAS_NMQM_GR_LINEAR_CP_NM_HOL_SP2_FEASET_AUTO_ALGO_RIDGE_AR_AUTO_DSI_AUTO_WSI_AUTO",
135 model_components_param=None
136 )
137 Silverkite_2 = replace(common_config, **param_update)
138
139 # Define the list of configs to benchmark
140 # The dictionary keys will be used to store the benchmark results
141 configs = {
142 # "Prophet": Prophet,
143 "SK_1": Silverkite_1,
144 "SK_2": Silverkite_2,
145 }
Define the Cross-Validation (CV)
In time-series forecasting we use a Rolling Window CV.
You can easily define it by using
RollingTimeSeriesSplit class.
The CV parameters depend on the data frequency,
forecast horizon as well as the speed of the models.
See Benchmarking documentation for guidance on how
to choose CV parameters for your use case.
158 # Define the benchmark folds
159 # CV parameters are changed for illustration purpose
160 tscv = RollingTimeSeriesSplit(
161 forecast_horizon=forecast_horizon,
162 min_train_periods=2 * 365,
163 expanding_window=True,
164 use_most_recent_splits=True,
165 periods_between_splits=5,
166 periods_between_train_test=0,
167 max_splits=4) # reduced to 4 from 16 for faster runtime
168
169 # Print the train, test split for benchmark folds
170 for split_num, (train, test) in enumerate(tscv.split(X=df)):
171 print(split_num, train, test)
Out:
0 [ 0 1 2 ... 2939 2940 2941] [2942 2943 2944 2945 2946 2947 2948]
1 [ 0 1 2 ... 2944 2945 2946] [2947 2948 2949 2950 2951 2952 2953]
2 [ 0 1 2 ... 2949 2950 2951] [2952 2953 2954 2955 2956 2957 2958]
3 [ 0 1 2 ... 2954 2955 2956] [2957 2958 2959 2960 2961 2962 2963]
Run the Benchmark
To start the benchmarking procedure execute its run method.
If you get an error message at this point, then there is a compatibility issue between your benchmark inputs. Check Debugging the Benchmark section for instructions on how to derive valid inputs.
Out:
0%| | 0/2 [00:00<?, ?it/s]
Benchmarking 'SK_1' : 0%| | 0/2 [00:00<?, ?it/s]
0%| | 0/4 [00:00<?, ?it/s]
Split '0' : 0%| | 0/4 [00:00<?, ?it/s]Fitting 1 folds for each of 1 candidates, totalling 1 fits
Split '0' : 25%|############################################################################################################################################################################################2 | 1/4 [00:57<02:53, 57.87s/it]
Split '1' : 25%|############################################################################################################################################################################################2 | 1/4 [00:57<02:53, 57.87s/it]Fitting 1 folds for each of 1 candidates, totalling 1 fits
Split '1' : 50%|########################################################################################################################################################################################################################################################################################################################################################################################5 | 2/4 [01:40<01:37, 48.84s/it]
Split '2' : 50%|########################################################################################################################################################################################################################################################################################################################################################################################5 | 2/4 [01:40<01:37, 48.84s/it]Fitting 1 folds for each of 1 candidates, totalling 1 fits
Split '2' : 75%|####################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################7 | 3/4 [02:07<00:38, 38.74s/it]
Split '3' : 75%|####################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################7 | 3/4 [02:07<00:38, 38.74s/it]Fitting 1 folds for each of 1 candidates, totalling 1 fits
Split '3' : 100%|#################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################| 4/4 [02:35<00:00, 34.46s/it]
Split '3' : 100%|#################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################| 4/4 [02:35<00:00, 38.75s/it]
Benchmarking 'SK_1' : 50%|################################################################################################################################################################################################################################################################################################################################################################################### | 1/2 [02:35<02:35, 155.01s/it]
Benchmarking 'SK_2' : 50%|################################################################################################################################################################################################################################################################################################################################################################################### | 1/2 [02:35<02:35, 155.01s/it]
0%| | 0/4 [00:00<?, ?it/s]
Split '0' : 0%| | 0/4 [00:00<?, ?it/s]Fitting 1 folds for each of 1 candidates, totalling 1 fits
Split '0' : 25%|############################################################################################################################################################################################2 | 1/4 [00:23<01:10, 23.66s/it]
Split '1' : 25%|############################################################################################################################################################################################2 | 1/4 [00:23<01:10, 23.66s/it]Fitting 1 folds for each of 1 candidates, totalling 1 fits
Split '1' : 50%|########################################################################################################################################################################################################################################################################################################################################################################################5 | 2/4 [00:49<00:49, 24.67s/it]
Split '2' : 50%|########################################################################################################################################################################################################################################################################################################################################################################################5 | 2/4 [00:49<00:49, 24.67s/it]Fitting 1 folds for each of 1 candidates, totalling 1 fits
Split '2' : 75%|####################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################7 | 3/4 [01:13<00:24, 24.80s/it]
Split '3' : 75%|####################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################7 | 3/4 [01:13<00:24, 24.80s/it]Fitting 1 folds for each of 1 candidates, totalling 1 fits
Split '3' : 100%|#################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################| 4/4 [01:39<00:00, 25.23s/it]
Split '3' : 100%|#################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################| 4/4 [01:39<00:00, 24.97s/it]
Benchmarking 'SK_2' : 100%|######################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################| 2/2 [04:14<00:00, 122.59s/it]
Benchmarking 'SK_2' : 100%|######################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################| 2/2 [04:14<00:00, 127.45s/it]
Monitor the Benchmark
During benchmarking a couple of color coded progress bars are displayed to inform the user of the
advancement of the entire process. The first bar displays config level information, while
the second bar displays split level information for the current config.
See example in Benchmarking documentation.
On the left side of the progress bar, it shows which config/ split is currently being
benchmarked and progress within that level as a percentage.
On the right side, the user can see how many configs/ splits have been benchmarked
and how many are remaining. Additionally, this bar also displays elapsed time and remaining runtime
for the corresponding level.
Benchmark Output
The output of a successful benchmark procedure is stored as a nested dictionary under the class attribute
result. For details on the structure of this tree check
Benchmarking documentation.
You can extract any specific information by navigating this tree. For example, you can
check the summary and component plot of any config.
Out:
================================ Model Summary =================================
Number of observations: 2942, Number of features: 264
Method: Ridge regression
Number of nonzero features: 264
Regularization parameter: 3.29
Residuals:
Min 1Q Median 3Q Max
-2.261 -0.2428 -0.0543 0.1658 3.23
Pred_col Estimate Std. Err Pr(>)_boot sig. code 95%CI
Intercept 6.117 0.1259 <2e-16 *** (5.888, 6.385)
events_Chinese New Year 0.04014 0.1835 0.820 (-0.3388, 0.3678)
events_Chinese New Year-1 -0.1147 0.1711 0.494 (-0.4387, 0.2301)
events_Chinese New Year-2 0.09564 0.1712 0.548 (-0.2092, 0.4332)
events_Chinese New Year+1 0.06808 0.1651 0.678 (-0.2474, 0.3861)
events_Chinese New Year+2 0.1144 0.1278 0.368 (-0.1242, 0.3708)
events_Christmas Day -0.3427 0.1288 0.012 * (-0.624, -0.1219)
events_Christmas Day-1 -0.1638 0.1385 0.244 (-0.453, 0.08302)
events_Christmas Day-2 -0.02716 0.1897 0.894 (-0.4239, 0.2603)
events_Christmas Day+1 -0.2164 0.1392 0.110 (-0.5029, 0.03143)
events_Christmas Day+2 0.1633 0.1003 0.106 (-0.02661, 0.3586)
events_Easter...hern Ireland] -0.1713 0.08345 0.036 * (-0.3138, 0.02502)
events_Easter...rn Ireland]-1 -0.1137 0.04976 0.022 * (-0.209, -0.001971)
events_Easter...rn Ireland]-2 -0.07558 0.04274 0.074 . (-0.1583, 0.005487)
events_Easter...rn Ireland]+1 -0.1081 0.06038 0.066 . (-0.2142, 0.01316)
events_Easter...rn Ireland]+2 -0.03451 0.04534 0.444 (-0.1338, 0.04539)
events_Good Friday -0.1708 0.05605 0.004 ** (-0.2674, -0.04553)
events_Good Friday-1 -0.1334 0.05423 0.016 * (-0.2357, -0.03233)
events_Good Friday-2 -0.07375 0.05857 0.198 (-0.1901, 0.04067)
events_Good Friday+1 -0.07558 0.04274 0.074 . (-0.1583, 0.005487)
events_Good Friday+2 -0.1137 0.04976 0.022 * (-0.209, -0.001971)
events_Independence Day 0.00592 0.05384 0.920 (-0.09812, 0.1098)
events_Independence Day-1 -0.02185 0.05678 0.708 (-0.136, 0.08871)
events_Independence Day-2 -0.04206 0.04507 0.356 (-0.1246, 0.04548)
events_Independence Day+1 -0.06111 0.05451 0.242 (-0.1571, 0.04793)
events_Independence Day+2 -0.06432 0.06008 0.300 (-0.181, 0.04431)
events_Labor Day -0.2295 0.08023 0.006 ** (-0.3759, -0.06909)
events_Labor Day-1 -0.09774 0.07759 0.202 (-0.2338, 0.06428)
events_Labor Day-2 -0.03347 0.06654 0.600 (-0.1549, 0.1055)
events_Labor Day+1 -0.1579 0.06375 0.016 * (-0.2736, -0.02953)
events_Labor Day+2 -0.1835 0.05528 0.002 ** (-0.2835, -0.0662)
events_Memorial Day -0.3278 0.06202 <2e-16 *** (-0.4147, -0.1866)
events_Memorial Day-1 -0.2063 0.06519 0.002 ** (-0.3198, -0.07707)
events_Memorial Day-2 -0.04419 0.07685 0.554 (-0.1652, 0.1313)
events_Memorial Day+1 -0.1079 0.04557 0.016 * (-0.1923, -0.01332)
events_Memorial Day+2 0.08969 0.06939 0.186 (-0.04154, 0.2252)
events_New Years Day 0.01326 0.09121 0.908 (-0.1527, 0.1963)
events_New Years Day-1 0.1042 0.1059 0.312 (-0.09886, 0.3278)
events_New Years Day-2 0.2211 0.1348 0.098 . (-0.02452, 0.5019)
events_New Years Day+1 0.266 0.1133 0.018 * (0.03937, 0.4778)
events_New Years Day+2 0.2402 0.1259 0.052 . (-0.007329, 0.4823)
events_Other 0.01332 0.03159 0.670 (-0.04481, 0.07651)
events_Other-1 0.01768 0.03154 0.598 (-0.03733, 0.07869)
events_Other-2 0.03747 0.02848 0.198 (-0.01313, 0.09258)
events_Other+1 0.03224 0.03033 0.302 (-0.02461, 0.09273)
events_Other+2 0.0264 0.02829 0.376 (-0.02604, 0.08574)
events_Thanksgiving -0.2122 0.0691 0.002 ** (-0.3442, -0.0738)
events_Thanksgiving-1 -0.3417 0.06162 <2e-16 *** (-0.4428, -0.2095)
events_Thanksgiving-2 -0.2734 0.06319 <2e-16 *** (-0.375, -0.1428)
events_Thanksgiving+1 -0.142 0.0742 0.062 . (-0.2824, 0.008333)
events_Thanksgiving+2 -0.1877 0.0567 0.002 ** (-0.2921, -0.0758)
events_Veterans Day -0.02697 0.08415 0.756 (-0.183, 0.1423)
events_Veterans Day-1 -0.05888 0.06737 0.396 (-0.1787, 0.07702)
events_Veterans Day-2 -0.03315 0.08703 0.712 (-0.1902, 0.1554)
events_Veterans Day+1 -0.02047 0.06905 0.758 (-0.1458, 0.1183)
events_Veterans Day+2 -0.02765 0.06929 0.670 (-0.1598, 0.112)
str_dow_2-Tue 0.01169 0.01875 0.498 (-0.02444, 0.0469)
str_dow_3-Wed -0.02174 0.01697 0.210 (-0.05594, 0.01123)
str_dow_4-Thu -0.0307 0.01615 0.060 . (-0.06155, 0.001807)
str_dow_5-Fri -0.03898 0.01763 0.030 * (-0.0756, -0.007238)
str_dow_6-Sat -0.1091 0.01785 <2e-16 *** (-0.1412, -0.07499)
str_dow_7-Sun 0.01308 0.02137 0.522 (-0.02542, 0.05457)
ct1 0.005591 0.08469 0.952 (-0.1611, 0.1865)
is_weekend:ct1 -0.005726 0.04654 0.900 (-0.09644, 0.09078)
str_dow_2-Tue:ct1 0.01478 0.04659 0.756 (-0.07396, 0.1091)
str_dow_3-Wed:ct1 -0.01108 0.03589 0.786 (-0.08372, 0.06032)
str_dow_4-Thu:ct1 -0.01126 0.03635 0.782 (-0.08019, 0.05917)
str_dow_5-Fri:ct1 0.03008 0.03576 0.452 (-0.03716, 0.09325)
str_dow_6-Sat:ct1 0.01223 0.03345 0.710 (-0.05154, 0.07372)
str_dow_7-Sun:ct1 -0.01795 0.04926 0.678 (-0.1156, 0.08238)
cp0_2008_09_29_00 0.2107 0.02847 <2e-16 *** (0.1604, 0.271)
is_weekend:cp0_2008_09_29_00 0.04814 0.01976 0.012 * (0.009078, 0.0863)
str_dow_2-Tue...2008_09_29_00 0.033 0.02646 0.216 (-0.01809, 0.08614)
str_dow_3-Wed...2008_09_29_00 0.01978 0.01869 0.318 (-0.01522, 0.05429)
str_dow_4-Thu...2008_09_29_00 0.01765 0.01881 0.336 (-0.01685, 0.05664)
str_dow_5-Fri...2008_09_29_00 0.03991 0.02243 0.078 . (-0.004134, 0.08518)
str_dow_6-Sat...2008_09_29_00 0.02829 0.01957 0.150 (-0.01081, 0.0658)
str_dow_7-Sun...2008_09_29_00 0.01987 0.02217 0.370 (-0.02145, 0.06417)
cp1_2008_11_03_00 0.2332 0.02764 <2e-16 *** (0.1777, 0.2905)
is_weekend:cp1_2008_11_03_00 0.05369 0.01953 0.004 ** (0.01686, 0.09175)
str_dow_2-Tue...2008_11_03_00 0.03584 0.02477 0.152 (-0.01251, 0.08404)
str_dow_3-Wed...2008_11_03_00 0.025 0.01752 0.156 (-0.009325, 0.05675)
str_dow_4-Thu...2008_11_03_00 0.02347 0.01718 0.180 (-0.009257, 0.05974)
str_dow_5-Fri...2008_11_03_00 0.04009 0.02129 0.066 . (-0.001134, 0.08238)
str_dow_6-Sat...2008_11_03_00 0.03256 0.01825 0.090 . (-0.0043, 0.06761)
str_dow_7-Sun...2008_11_03_00 0.02114 0.0212 0.318 (-0.01547, 0.06224)
cp2_2008_12_15_00 0.2379 0.03076 <2e-16 *** (0.1757, 0.2974)
is_weekend:cp2_2008_12_15_00 0.0537 0.02216 0.008 ** (0.01342, 0.096)
str_dow_2-Tue...2008_12_15_00 0.03843 0.02335 0.108 (-0.004397, 0.08338)
str_dow_3-Wed...2008_12_15_00 0.0245 0.01723 0.170 (-0.009808, 0.05567)
str_dow_4-Thu...2008_12_15_00 0.02529 0.01657 0.134 (-0.007915, 0.05958)
str_dow_5-Fri...2008_12_15_00 0.03431 0.0213 0.108 (-0.007262, 0.07546)
str_dow_6-Sat...2008_12_15_00 0.03146 0.01826 0.096 . (-0.008099, 0.06613)
str_dow_7-Sun...2008_12_15_00 0.02225 0.02239 0.318 (-0.0197, 0.06764)
cp3_2009_01_12_00 0.2351 0.0345 <2e-16 *** (0.1657, 0.304)
is_weekend:cp3_2009_01_12_00 0.05406 0.02476 0.022 * (0.008274, 0.1013)
str_dow_2-Tue...2009_01_12_00 0.03763 0.02332 0.106 (-0.005504, 0.08255)
str_dow_3-Wed...2009_01_12_00 0.02275 0.01784 0.200 (-0.01284, 0.05643)
str_dow_4-Thu...2009_01_12_00 0.02356 0.0172 0.170 (-0.008863, 0.05731)
str_dow_5-Fri...2009_01_12_00 0.0306 0.02142 0.160 (-0.01028, 0.06955)
str_dow_6-Sat...2009_01_12_00 0.02925 0.01895 0.140 (-0.01067, 0.06593)
str_dow_7-Sun...2009_01_12_00 0.02482 0.02394 0.300 (-0.01947, 0.07081)
cp4_2009_12_21_00 -0.1748 0.06495 0.008 ** (-0.2951, -0.05357)
is_weekend:cp4_2009_12_21_00 -0.06254 0.04794 0.206 (-0.1512, 0.02875)
str_dow_2-Tue...2009_12_21_00 -0.05231 0.04728 0.280 (-0.1507, 0.03208)
str_dow_3-Wed...2009_12_21_00 -0.01102 0.03577 0.772 (-0.07561, 0.05675)
str_dow_4-Thu...2009_12_21_00 -0.00327 0.03206 0.900 (-0.06467, 0.06291)
str_dow_5-Fri...2009_12_21_00 -0.05109 0.03985 0.224 (-0.1282, 0.02668)
str_dow_6-Sat...2009_12_21_00 -0.03847 0.03844 0.322 (-0.1131, 0.03785)
str_dow_7-Sun...2009_12_21_00 -0.02408 0.05144 0.624 (-0.1355, 0.07083)
cp5_2010_01_25_00 -0.1655 0.07128 0.018 * (-0.2995, -0.04255)
is_weekend:cp5_2010_01_25_00 -0.05777 0.05213 0.282 (-0.1545, 0.04503)
str_dow_2-Tue...2010_01_25_00 -0.05328 0.05168 0.328 (-0.1575, 0.03708)
str_dow_3-Wed...2010_01_25_00 -0.01018 0.03843 0.774 (-0.08274, 0.06868)
str_dow_4-Thu...2010_01_25_00 -0.003794 0.03361 0.902 (-0.06758, 0.06255)
str_dow_5-Fri...2010_01_25_00 -0.05383 0.04134 0.210 (-0.1336, 0.02403)
str_dow_6-Sat...2010_01_25_00 -0.04081 0.04145 0.334 (-0.1195, 0.04199)
str_dow_7-Sun...2010_01_25_00 -0.01698 0.05609 0.748 (-0.1326, 0.08734)
cp6_2011_02_14_00 0.2477 0.08593 0.004 ** (0.05456, 0.3983)
is_weekend:cp6_2011_02_14_00 0.08774 0.05915 0.142 (-0.02624, 0.2048)
str_dow_2-Tue...2011_02_14_00 -0.002566 0.06019 0.968 (-0.1257, 0.1154)
str_dow_3-Wed...2011_02_14_00 0.04493 0.04522 0.318 (-0.04644, 0.1269)
str_dow_4-Thu...2011_02_14_00 0.03484 0.04465 0.434 (-0.05458, 0.1175)
str_dow_5-Fri...2011_02_14_00 0.01418 0.05587 0.794 (-0.1, 0.1192)
str_dow_6-Sat...2011_02_14_00 0.02888 0.05039 0.572 (-0.07099, 0.1284)
str_dow_7-Sun...2011_02_14_00 0.05888 0.05771 0.308 (-0.0522, 0.174)
cp7_2012_02_27_00 -0.4776 0.07077 <2e-16 *** (-0.6127, -0.3409)
is_weekend:cp7_2012_02_27_00 -0.1035 0.05101 0.040 * (-0.2045, -0.01063)
str_dow_2-Tue...2012_02_27_00 -0.08778 0.05872 0.132 (-0.2029, 0.02487)
str_dow_3-Wed...2012_02_27_00 -0.07799 0.0603 0.192 (-0.201, 0.03047)
str_dow_4-Thu...2012_02_27_00 -0.07495 0.05867 0.216 (-0.1884, 0.03343)
str_dow_5-Fri...2012_02_27_00 -0.04902 0.06162 0.454 (-0.1645, 0.05631)
str_dow_6-Sat...2012_02_27_00 -0.04071 0.04996 0.432 (-0.1425, 0.04972)
str_dow_7-Sun...2012_02_27_00 -0.06282 0.05264 0.240 (-0.1654, 0.03782)
cp8_2012_04_09_00 -0.4308 0.06382 <2e-16 *** (-0.5496, -0.3003)
is_weekend:cp8_2012_04_09_00 -0.1019 0.04673 0.030 * (-0.1964, -0.0168)
str_dow_2-Tue...2012_04_09_00 -0.07723 0.05868 0.178 (-0.193, 0.0295)
str_dow_3-Wed...2012_04_09_00 -0.06449 0.05707 0.258 (-0.1803, 0.04103)
str_dow_4-Thu...2012_04_09_00 -0.06451 0.05552 0.246 (-0.1691, 0.04146)
str_dow_5-Fri...2012_04_09_00 -0.05378 0.05756 0.374 (-0.1674, 0.05061)
str_dow_6-Sat...2012_04_09_00 -0.0419 0.04742 0.376 (-0.143, 0.04544)
str_dow_7-Sun...2012_04_09_00 -0.06008 0.0506 0.246 (-0.1568, 0.03537)
cp9_2013_03_04_00 0.2743 0.08289 <2e-16 *** (0.09249, 0.4221)
is_weekend:cp9_2013_03_04_00 0.004187 0.06363 0.946 (-0.1276, 0.1193)
str_dow_2-Tue...2013_03_04_00 0.08094 0.06058 0.218 (-0.0392, 0.1938)
str_dow_3-Wed...2013_03_04_00 0.07103 0.04381 0.098 . (-0.01345, 0.1574)
str_dow_4-Thu...2013_03_04_00 0.06013 0.04951 0.222 (-0.02911, 0.1621)
str_dow_5-Fri...2013_03_04_00 0.03473 0.05249 0.524 (-0.06791, 0.1341)
str_dow_6-Sat...2013_03_04_00 0.008695 0.04835 0.876 (-0.08678, 0.1037)
str_dow_7-Sun...2013_03_04_00 -0.004499 0.06508 0.942 (-0.1354, 0.1149)
cp10_2014_01_27_00 -0.2157 0.09256 0.026 * (-0.3864, -0.01473)
is_weekend:cp10_2014_01_27_00 -0.05859 0.06778 0.402 (-0.1892, 0.07865)
str_dow_2-Tue...2014_01_27_00 0.04794 0.1097 0.638 (-0.1632, 0.2727)
str_dow_3-Wed...2014_01_27_00 -0.05173 0.06367 0.430 (-0.1533, 0.08083)
str_dow_4-Thu...2014_01_27_00 -0.03651 0.06857 0.610 (-0.1633, 0.1057)
str_dow_5-Fri...2014_01_27_00 -0.01177 0.07899 0.874 (-0.1738, 0.1335)
str_dow_6-Sat...2014_01_27_00 -0.05403 0.05476 0.326 (-0.1645, 0.04236)
str_dow_7-Sun...2014_01_27_00 -0.00464 0.08675 0.956 (-0.1603, 0.1654)
ct1:sin1_tow_weekly -0.00771 0.03362 0.804 (-0.07019, 0.06189)
ct1:cos1_tow_weekly -0.016 0.05807 0.808 (-0.1368, 0.08941)
ct1:sin2_tow_weekly 0.0381 0.04156 0.364 (-0.04593, 0.116)
ct1:cos2_tow_weekly 8.192e-05 0.05405 0.996 (-0.1121, 0.1023)
cp0_2008_09_2...n1_tow_weekly -0.003941 0.01563 0.788 (-0.03231, 0.02748)
cp0_2008_09_2...s1_tow_weekly 0.01203 0.02761 0.658 (-0.03961, 0.06558)
cp0_2008_09_2...n2_tow_weekly 0.01739 0.01937 0.360 (-0.02154, 0.05446)
cp0_2008_09_2...s2_tow_weekly 0.01747 0.02685 0.510 (-0.03519, 0.07168)
cp1_2008_11_0...n1_tow_weekly -0.001587 0.01496 0.900 (-0.02994, 0.02813)
cp1_2008_11_0...s1_tow_weekly 0.01101 0.02509 0.676 (-0.0342, 0.06112)
cp1_2008_11_0...n2_tow_weekly 0.01571 0.01745 0.362 (-0.01858, 0.04857)
cp1_2008_11_0...s2_tow_weekly 0.01604 0.02442 0.512 (-0.02905, 0.06515)
cp2_2008_12_1...n1_tow_weekly 0.0009993 0.01564 0.950 (-0.02891, 0.03064)
cp2_2008_12_1...s1_tow_weekly 0.01771 0.02346 0.462 (-0.02825, 0.06233)
cp2_2008_12_1...n2_tow_weekly 0.01327 0.01697 0.430 (-0.02053, 0.04617)
cp2_2008_12_1...s2_tow_weekly 0.01849 0.02265 0.406 (-0.02643, 0.06415)
cp3_2009_01_1...n1_tow_weekly 0.0003183 0.01682 0.986 (-0.03388, 0.0327)
cp3_2009_01_1...s1_tow_weekly 0.02388 0.02332 0.314 (-0.02138, 0.06858)
cp3_2009_01_1...n2_tow_weekly 0.01069 0.01771 0.538 (-0.02248, 0.04423)
cp3_2009_01_1...s2_tow_weekly 0.02092 0.02238 0.336 (-0.02289, 0.06578)
cp4_2009_12_2...n1_tow_weekly 0.01304 0.03542 0.740 (-0.06083, 0.07643)
cp4_2009_12_2...s1_tow_weekly 0.009213 0.05055 0.856 (-0.08737, 0.1082)
cp4_2009_12_2...n2_tow_weekly -0.03939 0.043 0.368 (-0.1185, 0.04771)
cp4_2009_12_2...s2_tow_weekly 0.01732 0.04895 0.718 (-0.08445, 0.1129)
cp5_2010_01_2...n1_tow_weekly 0.01188 0.03814 0.746 (-0.0648, 0.08129)
cp5_2010_01_2...s1_tow_weekly 0.01714 0.05413 0.758 (-0.08674, 0.1211)
cp5_2010_01_2...n2_tow_weekly -0.04501 0.04616 0.320 (-0.1314, 0.04569)
cp5_2010_01_2...s2_tow_weekly 0.02044 0.05212 0.686 (-0.0897, 0.123)
cp6_2011_02_1...n1_tow_weekly -0.01209 0.04592 0.830 (-0.1024, 0.07264)
cp6_2011_02_1...s1_tow_weekly 0.02295 0.06274 0.704 (-0.09557, 0.1536)
cp6_2011_02_1...n2_tow_weekly -0.04252 0.04954 0.374 (-0.1326, 0.06059)
cp6_2011_02_1...s2_tow_weekly 0.01084 0.05648 0.854 (-0.1074, 0.1234)
cp7_2012_02_2...n1_tow_weekly -0.03441 0.04707 0.460 (-0.1254, 0.06382)
cp7_2012_02_2...s1_tow_weekly -0.02186 0.06953 0.778 (-0.1479, 0.1231)
cp7_2012_02_2...n2_tow_weekly 0.006063 0.04812 0.892 (-0.08722, 0.1053)
cp7_2012_02_2...s2_tow_weekly -0.01174 0.06825 0.868 (-0.1413, 0.1174)
cp8_2012_04_0...n1_tow_weekly -0.02055 0.04419 0.636 (-0.1022, 0.066)
cp8_2012_04_0...s1_tow_weekly -0.0134 0.06555 0.846 (-0.1326, 0.1203)
cp8_2012_04_0...n2_tow_weekly 0.0006358 0.04643 0.988 (-0.08849, 0.09423)
cp8_2012_04_0...s2_tow_weekly -0.009023 0.0648 0.896 (-0.1417, 0.1177)
cp9_2013_03_0...n1_tow_weekly 0.07116 0.04459 0.116 (-0.008223, 0.1631)
cp9_2013_03_0...s1_tow_weekly -0.01631 0.06307 0.828 (-0.1504, 0.09138)
cp9_2013_03_0...n2_tow_weekly 0.01882 0.05323 0.776 (-0.07851, 0.1185)
cp9_2013_03_0...s2_tow_weekly -0.002872 0.05613 0.958 (-0.106, 0.108)
cp10_2014_01_...n1_tow_weekly 0.01698 0.05709 0.752 (-0.1041, 0.1244)
cp10_2014_01_...s1_tow_weekly -0.006531 0.1179 0.948 (-0.2422, 0.2188)
cp10_2014_01_...n2_tow_weekly 0.03562 0.06827 0.610 (-0.1023, 0.1621)
cp10_2014_01_...s2_tow_weekly -0.02681 0.1131 0.818 (-0.2509, 0.1966)
sin1_tow_weekly 0.04498 0.01909 0.024 * (0.006723, 0.0835)
cos1_tow_weekly 0.1489 0.0296 <2e-16 *** (0.0879, 0.2059)
sin2_tow_weekly -0.02346 0.01956 0.238 (-0.06228, 0.01219)
cos2_tow_weekly 0.1287 0.02921 <2e-16 *** (0.07371, 0.1819)
sin3_tow_weekly -0.03121 0.01579 0.058 . (-0.06212, 0.001176)
cos3_tow_weekly 0.04595 0.028 0.092 . (-0.01199, 0.09604)
sin1_tom_monthly 0.1879 0.05566 0.002 ** (0.08051, 0.2957)
cos1_tom_monthly 0.378 0.05769 <2e-16 *** (0.2511, 0.48)
sin2_tom_monthly 0.0567 0.02193 0.010 * (0.01474, 0.09976)
cos2_tom_monthly -0.07404 0.02503 <2e-16 *** (-0.1247, -0.02589)
sin3_tom_monthly -0.03113 0.02562 0.232 (-0.07983, 0.01629)
cos3_tom_monthly -0.01881 0.0255 0.458 (-0.07084, 0.02661)
sin4_tom_monthly -0.005921 0.02365 0.812 (-0.04855, 0.04431)
cos4_tom_monthly -0.02279 0.02378 0.344 (-0.0677, 0.02153)
sin1_toq_quarterly 0.08543 0.02059 <2e-16 *** (0.04563, 0.1245)
cos1_toq_quarterly -0.009899 0.02115 0.632 (-0.0526, 0.03108)
sin2_toq_quarterly 0.1593 0.03544 <2e-16 *** (0.08745, 0.2231)
cos2_toq_quarterly -0.1946 0.03252 <2e-16 *** (-0.2491, -0.1165)
sin3_toq_quarterly 0.04661 0.03828 0.220 (-0.02802, 0.1279)
cos3_toq_quarterly -0.06365 0.04403 0.148 (-0.143, 0.02286)
sin4_toq_quarterly 0.007778 0.0243 0.772 (-0.03848, 0.05473)
cos4_toq_quarterly -0.0794 0.02558 0.002 ** (-0.1266, -0.02891)
sin1_ct1_yearly -0.2681 0.02724 <2e-16 *** (-0.3229, -0.219)
cos1_ct1_yearly 0.6065 0.04651 <2e-16 *** (0.5125, 0.6917)
sin2_ct1_yearly 0.09868 0.02637 <2e-16 *** (0.04972, 0.1491)
cos2_ct1_yearly -0.08284 0.02474 <2e-16 *** (-0.1344, -0.03822)
sin3_ct1_yearly 0.3063 0.02671 <2e-16 *** (0.2518, 0.3577)
cos3_ct1_yearly 0.09213 0.02642 <2e-16 *** (0.04323, 0.1417)
sin4_ct1_yearly 0.08335 0.02249 <2e-16 *** (0.04025, 0.1269)
cos4_ct1_yearly -0.05211 0.01723 <2e-16 *** (-0.08763, -0.01951)
sin5_ct1_yearly -0.1368 0.02495 <2e-16 *** (-0.1852, -0.08438)
cos5_ct1_yearly -0.106 0.02432 <2e-16 *** (-0.1538, -0.05853)
sin6_ct1_yearly -0.1869 0.02245 <2e-16 *** (-0.2283, -0.1441)
cos6_ct1_yearly -0.1542 0.0255 <2e-16 *** (-0.2041, -0.09786)
sin7_ct1_yearly -0.1481 0.02397 <2e-16 *** (-0.1911, -0.09762)
cos7_ct1_yearly 0.03018 0.02425 0.216 (-0.0201, 0.07687)
sin8_ct1_yearly 0.1709 0.03597 <2e-16 *** (0.09029, 0.2319)
cos8_ct1_yearly 0.09833 0.03161 <2e-16 *** (0.04192, 0.163)
sin9_ct1_yearly 0.02573 0.02579 0.320 (-0.02358, 0.07872)
cos9_ct1_yearly -0.04296 0.0242 0.086 . (-0.08663, 0.006201)
sin10_ct1_yearly -0.1238 0.02407 <2e-16 *** (-0.1696, -0.07749)
cos10_ct1_yearly -0.1818 0.02623 <2e-16 *** (-0.2319, -0.132)
sin11_ct1_yearly -0.06817 0.02529 0.008 ** (-0.1121, -0.01718)
cos11_ct1_yearly -0.05662 0.02443 0.018 * (-0.1032, -0.00735)
sin12_ct1_yearly 0.323 0.06821 <2e-16 *** (0.1793, 0.4599)
cos12_ct1_yearly -0.07043 0.05822 0.252 (-0.1783, 0.04757)
sin13_ct1_yearly -0.04137 0.02746 0.142 (-0.09384, 0.01584)
cos13_ct1_yearly 0.1126 0.02677 <2e-16 *** (0.0626, 0.1674)
sin14_ct1_yearly 0.0695 0.02426 <2e-16 *** (0.02285, 0.112)
cos14_ct1_yearly 0.007866 0.02604 0.766 (-0.04626, 0.05587)
sin15_ct1_yearly 0.06504 0.02515 0.012 * (0.01859, 0.1151)
cos15_ct1_yearly -0.05512 0.02622 0.044 * (-0.109, -0.005108)
y_lag7 0.7971 0.1521 <2e-16 *** (0.5028, 1.075)
y_lag8 0.4709 0.1404 0.004 ** (0.2342, 0.7868)
y_lag9 -0.00889 0.1236 0.948 (-0.2474, 0.2192)
y_avglag_7_14_21 1.504 0.155 <2e-16 *** (1.195, 1.789)
y_avglag_7_to_13 0.433 0.1383 <2e-16 *** (0.1584, 0.6971)
y_avglag_14_to_20 0.1228 0.1034 0.242 (-0.07069, 0.3194)
Signif. Code: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
Multiple R-squared: 0.7276, Adjusted R-squared: 0.717
F-statistic: 66.69 on 110 and 2830 DF, p-value: 1.110e-16
Model AIC: 18935.0, model BIC: 19600.0
WARNING: the condition number is large, 2.56e+04. This might indicate that there are strong multicollinearity or other numerical problems.
WARNING: the F-ratio and its p-value on regularized methods might be misleading, they are provided only for reference purposes.
Compare forecasts
To obtain forecasts run the extract_forecasts method. You only need to run this once.
226 bm.extract_forecasts()
This method does two things.
For every
config, it gathers forecast results across rolling windows and stores it as a dataframe inrolling_forecast_dfunder theconfigkey. This helps in comparing forecasts and prediction accuracy across splits for theconfig.
235 # Forecast across rolling windows for SK_1
236 forecast_sk_1 = bm.result["SK_1"]["rolling_forecast_df"]
237 forecast_sk_1.head()
Concatenates
rolling_forecast_dffor all theconfigsand stores it as a dataframe in the class attributeforecasts. This helps in comparing forecasts and prediction accuracies acrossconfigs.
243 # Forecasts across configs
244 bm.forecasts.head()
For any config you can plot forecasts across splits. This allows you to quickly check if there is
any particular time window where the test performance drops. The forecasts for adjacent folds will
overlap if the time windows of the corresponding folds overlap.
251 fig = bm.plot_forecasts_by_config(config_name="SK_1")
252 plotly.io.show(fig)
The importance of this function becomes more significant when assessing a models performance over a longer period e.g. a year or multiple years. You can quickly catch if models test performance drops during weekends, specific months or holiday seasons.
You can also compare forecasts from multiple configs by forecast_step which is
defined as any number between 1 and forecast_horizon. This is useful in forecasts with longer
forecast horizons to check if the forecast volatility changes over time.
263 fig = bm.plot_forecasts_by_step(forecast_step=3)
264 plotly.io.show(fig)
Compare Errors
You can compare the predictive performance of your models via multiple evaluation metrics.
In this example we will use MAPE and RMSE, but you can use any metric from EvaluationMetricEnum.
272 metric_dict = {
273 "MAPE": EvaluationMetricEnum.MeanAbsolutePercentError,
274 "RMSE": EvaluationMetricEnum.RootMeanSquaredError
275 }
Non Grouping Errors
To compare evaluation metrics without any grouping use get_evaluation_metrics.
The output shows metric values by config and split. We can group by config_name to get
metric values aggregated across all folds.
284 # Compute evaluation metrics
285 evaluation_metrics_df = bm.get_evaluation_metrics(metric_dict=metric_dict)
286 # Aggregate by model across splits
287 error_df = evaluation_metrics_df.drop(columns=["split_num"]).groupby("config_name").mean()
288 error_df
292 # Visualize
293 fig = bm.plot_evaluation_metrics(metric_dict)
294 plotly.io.show(fig)
Train MAPE is high because some values in training dataset are close to 0.
You can also compare the predictive accuracy across splits for any model from configs.
This allows you to check if the model performance varies significantly across time periods.
302 # Compute evaluation metrics for a single config
303 evaluation_metrics_df = bm.get_evaluation_metrics(metric_dict=metric_dict, config_names=["SK_1"])
304 # Aggregate by split number
305 error_df = evaluation_metrics_df.groupby("split_num").mean()
306 error_df.head()
310 # Visualize
311 title = "Average evaluation metric across rolling windows"
312 data = []
313 # Each row (index) is a config. Adds each row to the bar plot.
314 for index in error_df.index:
315 data.append(
316 go.Bar(
317 name=index,
318 x=error_df.columns,
319 y=error_df.loc[index].values
320 )
321 )
322 layout = go.Layout(
323 xaxis=dict(title=None),
324 yaxis=dict(title="Metric Value"),
325 title=title,
326 title_x=0.5,
327 showlegend=True,
328 barmode="group",
329 )
330 fig = go.Figure(data=data, layout=layout)
331 plotly.io.show(fig)
Grouping Errors
To compare evaluation metrics with grouping use get_grouping_evaluation_metrics.
This allows you to group the error values by time features such as day of week, month etc.
339 # Compute grouped evaluation metrics
340 grouped_evaluation_df = bm.get_grouping_evaluation_metrics(
341 metric_dict=metric_dict,
342 which="test",
343 groupby_time_feature="str_dow")
344 # Aggregate by split number
345 error_df = grouped_evaluation_df.groupby(["str_dow", "config_name"]).mean()
346 error_df
350 # Visualize
351 fig = bm.plot_grouping_evaluation_metrics(
352 metric_dict=metric_dict,
353 which="test",
354 groupby_time_feature="str_dow")
355 plotly.io.show(fig)
As you can see all the models have higher MAPE and RMSE during weekends. That means adding
is_weekend indicator to the models will help.
Compare runtimes
You can compare and visualize runtimes of the models using the following codes.
365 # Compute runtimes
366 runtime_df = bm.get_runtimes()
367 # Aggregate across splits
368 runtimes_df = runtime_df.drop(columns=["split_num"]).groupby("config_name").mean()
369 runtimes_df
373 # Visualize
374 fig = bm.plot_runtimes()
375 plotly.io.show(fig)
You can see Silverkite models run almost 3 times faster compared to Prophet.
Debugging the Benchmark
When the run method is called, the input configs are first assessed of
their suitability for a cohesive benchmarking procedure via the validate method.
This is done prior to passing the configs to the forecasting pipeline to save wasted
computing time for the user.
Though not necessary, the user is encouraged to use validate for debugging.
The validate method runs a series of checks to ensure that
The
configsare compatible among themselves. For example, it checks if all theconfigshave the sameforecast horizon.The
configsare compatible with the CV schema. For example,forecast_horizonandperiods_between_train_testparameters ofconfigsare matched against that of thetscv.
Note that the validate method does not guarantee that the models will execute properly
while in the pipeline. It is a good idea to do a test run on a smaller data and/ or smaller
number of splits before running the full procedure.
In the event of a mismatch a ValueError is raised with informative error messages
to help the user in debugging. Some examples are provided below.
Error due to incompatible model components in config
406 # some_regressor is not part of Prophet's model components
407 model_components=ModelComponentsParam(
408 regressors={
409 "some_regressor": ["regressor1", "regressor2", "regressor_categ"]
410 }
411 )
412 invalid_prophet = replace(Silverkite_1, model_components_param=model_components)
413 invalid_configs = {"invalid_prophet": invalid_prophet}
414 bm = BenchmarkForecastConfig(df=df, configs=invalid_configs, tscv=tscv)
415 try:
416 bm.validate()
417 except ValueError as err:
418 print(err)
Out:
Unexpected key(s) found: {'some_regressor'}. The valid keys are: dict_keys(['regressor_cols'])
Error due to wrong template name
424 # model template name is not part of TemplateEnum, thus invalid
425 unknown_template = replace(Silverkite_1, model_template="SOME_TEMPLATE")
426 invalid_configs = {"unknown_template": unknown_template}
427 bm = BenchmarkForecastConfig(df=df, configs=invalid_configs, tscv=tscv)
428 try:
429 bm.validate()
430 except ValueError as err:
431 print(err)
Out:
Model Template 'SOME_TEMPLATE' is not recognized! Must be one of: SILVERKITE, SILVERKITE_DAILY_1_CONFIG_1, SILVERKITE_DAILY_1_CONFIG_2, SILVERKITE_DAILY_1_CONFIG_3, SILVERKITE_DAILY_1, SILVERKITE_DAILY_90, SILVERKITE_WEEKLY, SILVERKITE_MONTHLY, SILVERKITE_HOURLY_1, SILVERKITE_HOURLY_24, SILVERKITE_HOURLY_168, SILVERKITE_HOURLY_336, SILVERKITE_EMPTY, SK, PROPHET, AUTO_ARIMA, SILVERKITE_TWO_STAGE, MULTISTAGE_EMPTY, AUTO, LAG_BASED, SILVERKITE_WOW or satisfy the `SimpleSilverkiteTemplate` rules.
Error due to different forecast horizons in configs
437 # the configs are valid by themselves, however incompatible for
438 # benchmarking as these have different forecast horizons
439 Silverkite_forecast_horizon_30 = replace(Silverkite_1, forecast_horizon=30)
440 invalid_configs = {
441 "Silverkite": Silverkite_1,
442 "Silverkite_30": Silverkite_forecast_horizon_30
443 }
444 bm = BenchmarkForecastConfig(df=df, configs=invalid_configs, tscv=tscv)
445 try:
446 bm.validate()
447 except ValueError as err:
448 print(err)
Out:
Silverkite_30's 'forecast_horizon' (30) does not match that of 'tscv' (7).
Error due to different forecast horizons in config and tscv
454 # Error due to different forecast horizons in config and tscv
455 tscv = RollingTimeSeriesSplit(forecast_horizon=15)
456 bm = BenchmarkForecastConfig(df=df, configs=configs, tscv=tscv)
457 try:
458 bm.validate()
459 except ValueError as err:
460 print(err)
Out:
SK_1's 'forecast_horizon' (7) does not match that of 'tscv' (15).
Total running time of the script: ( 4 minutes 27.289 seconds)