Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Major] simplify learning rate finder and use log-mean of default and smooth suggestion #1630

Merged
merged 7 commits into from
Aug 14, 2024
Merged

[Major] simplify learning rate finder and use log-mean of default and smooth suggestion #1630

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8f6f2b6
simplify learning rate finder
ourownstory Aug 13, 2024
e942e5e
set learning rate to logarithmic mean of smooth and default.
ourownstory Aug 13, 2024
687b6d8
move logic to util
ourownstory Aug 13, 2024
f796efa
fix logic
ourownstory Aug 13, 2024
52db2f7
Update configure.py removing f
ourownstory Aug 14, 2024
201a661
Update configure.py
ourownstory Aug 14, 2024
0b9715c
Update utils.py
ourownstory Aug 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions neuralprophet/configure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@
Set the lr_finder_args.
This is the range of learning rates to test.
"""
num_training = 150 + int(np.log10(100 + dataset_size) * 25)
num_training = 100 + int(np.log10(dataset_size) * 20)
if num_batches < num_training:
log.warning(
f"Learning rate finder: The number of batches ({num_batches}) is too small than the required number \
Expand All @@ -217,7 +217,7 @@
# num_training = num_batches
self.lr_finder_args.update(
{
"min_lr": 1e-6,
"min_lr": 1e-7,
"max_lr": 10,
"num_training": num_training,
"early_stop_threshold": None,
Expand Down Expand Up @@ -304,7 +304,7 @@
log.error("Invalid growth for global_local mode '{}'. Set to 'global'".format(self.trend_global_local))
self.trend_global_local = "global"

if self.trend_local_reg < 0:

Check failure on line 307 in neuralprophet/configure.py

View workflow job for this annotation

GitHub Actions / pyright 

Operator "<" not supported for "None" (reportOptionalOperand)
log.error("Invalid negative trend_local_reg '{}'. Set to False".format(self.trend_local_reg))
self.trend_local_reg = False

Expand Down Expand Up @@ -353,13 +353,13 @@
log.error("Invalid global_local mode '{}'. Set to 'global'".format(self.global_local))
self.global_local = "global"

self.periods = OrderedDict(

Check failure on line 356 in neuralprophet/configure.py

View workflow job for this annotation

GitHub Actions / pyright 

No overloads for "__init__" match the provided arguments (reportCallIssue)
{

Check failure on line 357 in neuralprophet/configure.py

View workflow job for this annotation

GitHub Actions / pyright 

Argument of type "dict[str, Season]" cannot be assigned to parameter "iterable" of type "Iterable[list[bytes]]" in function "__init__" (reportArgumentType)
"yearly": Season(
resolution=6,
period=365.25,
arg=self.yearly_arg,
global_local=(

Check failure on line 362 in neuralprophet/configure.py

View workflow job for this annotation

GitHub Actions / pyright 

Argument of type "SeasonGlobalLocalMode | Literal['auto']" cannot be assigned to parameter "global_local" of type "SeasonGlobalLocalMode" in function "__init__" (reportArgumentType)
self.yearly_global_local
if self.yearly_global_local in ["global", "local"]
else self.global_local
Expand All @@ -370,7 +370,7 @@
resolution=3,
period=7,
arg=self.weekly_arg,
global_local=(

Check failure on line 373 in neuralprophet/configure.py

View workflow job for this annotation

GitHub Actions / pyright 

Argument of type "SeasonGlobalLocalMode | Literal['auto']" cannot be assigned to parameter "global_local" of type "SeasonGlobalLocalMode" in function "__init__" (reportArgumentType)
self.weekly_global_local
if self.weekly_global_local in ["global", "local"]
else self.global_local
Expand All @@ -381,7 +381,7 @@
resolution=6,
period=1,
arg=self.daily_arg,
global_local=(

Check failure on line 384 in neuralprophet/configure.py

View workflow job for this annotation

GitHub Actions / pyright 

Argument of type "SeasonGlobalLocalMode | Literal['auto']" cannot be assigned to parameter "global_local" of type "SeasonGlobalLocalMode" in function "__init__" (reportArgumentType)
self.daily_global_local if self.daily_global_local in ["global", "local"] else self.global_local
),
condition_name=None,
Expand All @@ -389,7 +389,7 @@
}
)

assert self.seasonality_local_reg >= 0, "Invalid seasonality_local_reg '{}'.".format(self.seasonality_local_reg)

Check failure on line 392 in neuralprophet/configure.py

View workflow job for this annotation

GitHub Actions / pyright 

Operator ">=" not supported for "None" (reportOptionalOperand)

if self.seasonality_local_reg is True:
log.warning("seasonality_local_reg = True. Default seasonality_local_reg value set to 1")
Expand All @@ -407,7 +407,7 @@
resolution=resolution,
period=period,
arg=arg,
global_local=global_local if global_local in ["global", "local"] else self.global_local,

Check failure on line 410 in neuralprophet/configure.py

View workflow job for this annotation

GitHub Actions / pyright 

Argument of type "str" cannot be assigned to parameter "global_local" of type "SeasonGlobalLocalMode" in function "__init__"
  Type "str" is incompatible with type "SeasonGlobalLocalMode"
    "str" is incompatible with type "Literal['global']"
    "str" is incompatible with type "Literal['local']"
    "str" is incompatible with type "Literal['glocal']" (reportArgumentType)
condition_name=condition_name,
)

Expand Down Expand Up @@ -483,7 +483,7 @@
regressors: OrderedDict = field(init=False) # contains RegressorConfig objects

def __post_init__(self):
self.regressors = None

Check failure on line 486 in neuralprophet/configure.py

View workflow job for this annotation

GitHub Actions / pyright 

Cannot assign to attribute "regressors" for class "ConfigFutureRegressors*"
  "None" is incompatible with "OrderedDict[Unknown, Unknown]" (reportAttributeAccessIssue)


@dataclass
Expand Down
8 changes: 3 additions & 5 deletions neuralprophet/forecaster.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2805,13 +2805,12 @@ def _train(
lr_finder = tuner.lr_find(
model=self.model,
train_dataloaders=train_loader,
val_dataloaders=val_loader,
# val_dataloaders=val_loader, # not be used, but may lead to Lightning bug if not provided
**self.config_train.lr_finder_args,
)
# Estimate the optimal learning rate from the loss curve
assert lr_finder is not None
_, _, lr_suggestion = utils.smooth_loss_and_suggest(lr_finder.results)
self.model.learning_rate = lr_suggestion
_, _, self.model.learning_rate = utils.smooth_loss_and_suggest(lr_finder)
start = time.time()
self.trainer.fit(
self.model,
Expand All @@ -2832,8 +2831,7 @@ def _train(
)
assert lr_finder is not None
# Estimate the optimal learning rate from the loss curve
_, _, lr_suggestion = utils.smooth_loss_and_suggest(lr_finder.results)
self.model.learning_rate = lr_suggestion
_, _, self.model.learning_rate = utils.smooth_loss_and_suggest(lr_finder)
start = time.time()
self.trainer.fit(
self.model,
Expand Down
18 changes: 15 additions & 3 deletions neuralprophet/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -751,7 +751,7 @@ def set_log_level(log_level: str = "INFO", include_handlers: bool = False):
set_logger_level(logging.getLogger("NP"), log_level, include_handlers)


def smooth_loss_and_suggest(lr_finder_results, window=10):
def smooth_loss_and_suggest(lr_finder, window=10):
"""
Smooth loss using a Hamming filter.

Expand All @@ -769,10 +769,12 @@ def smooth_loss_and_suggest(lr_finder_results, window=10):
suggested_lr: float
Suggested learning rate based on gradient
"""
lr_finder_results = lr_finder.results
lr = lr_finder_results["lr"]
loss = lr_finder_results["loss"]
# Derive window size from num lr searches, ensure window is divisible by 2
half_window = math.ceil(round(len(loss) * 0.1) / 2)
# half_window = math.ceil(round(len(loss) * 0.1) / 2)
half_window = math.ceil(window / 2)
# Pad sequence and initialialize hamming filter
loss = np.pad(np.array(loss), pad_width=half_window, mode="edge")
window = np.hamming(half_window * 2)
Expand All @@ -798,7 +800,17 @@ def smooth_loss_and_suggest(lr_finder_results, window=10):
"samples or manually set the learning rate."
)
raise
return (loss, lr, suggestion)
suggestion_default = lr_finder.suggestion(skip_begin=10, skip_end=3)
if suggestion is not None and suggestion_default is not None:
log_suggestion_smooth = np.log(suggestion)
log_suggestion_default = np.log(suggestion_default)
lr_suggestion = np.exp((log_suggestion_smooth + log_suggestion_default) / 2)
elif suggestion is None and suggestion_default is None:
log.error("Automatic learning rate test failed. Please set manually the learning rate.")
raise
else:
lr_suggestion = suggestion if suggestion is not None else suggestion_default
return (loss, lr, lr_suggestion)


def _smooth_loss(loss, beta=0.9):
Expand Down
Loading