Lines Matching full:optimizer
27 from .optimizer import Optimizer
71 def _format_param(name: str, optimizer: Optimizer, param): argument
78 if len(param) != len(optimizer.param_groups):
80 f"{name} must have the same length as optimizer.param_groups. "
81 f"{name} has {len(param)} values, param_groups has {len(optimizer.param_groups)}."
84 param = [param] * len(optimizer.param_groups)
95 self, optimizer: Optimizer, last_epoch=-1, verbose="deprecated" argument
97 # Attach optimizer
98 if not isinstance(optimizer, Optimizer):
99 raise TypeError(f"{type(optimizer).__name__} is not an Optimizer")
100 self.optimizer = optimizer
104 for group in optimizer.param_groups:
110 for i, group in enumerate(optimizer.param_groups):
114 f"in param_groups[{i}] when resuming an optimizer"
117 group["initial_lr"] for group in optimizer.param_groups
123 # `optimizer.step()`
124 def patch_track_step_called(opt: Optimizer): argument
130 opt_ref = ref(self.optimizer)
144 patch_track_step_called(self.optimizer)
157 is not the optimizer.
160 key: value for key, value in self.__dict__.items() if key != "optimizer"
213 if not hasattr(self.optimizer.step, "_wrapped_by_lr_sched"):
215 … "Seems like `optimizer.step()` has been overridden after learning rate scheduler "
216 "initialization. Please, make sure to call `optimizer.step()` before "
222 # Just check if there were two first lr_scheduler.step() calls before optimizer.step()
223 elif not getattr(self.optimizer, "_opt_called", False):
225 "Detected call of `lr_scheduler.step()` before `optimizer.step()`. "
227 "`optimizer.step()` before `lr_scheduler.step()`. Failure to do this "
247 for i, data in enumerate(zip(self.optimizer.param_groups, values)):
255 group["lr"] for group in self.optimizer.param_groups
294 optimizer (Optimizer): Wrapped optimizer.
297 functions, one for each group in optimizer.param_groups.
308 >>> # Assuming optimizer has two groups.
311 >>> scheduler = LambdaLR(optimizer, lr_lambda=[lambda1, lambda2])
320 optimizer: Optimizer, argument
325 self.optimizer = optimizer
329 self.lr_lambdas = [lr_lambda] * len(optimizer.param_groups)
331 if len(lr_lambda) != len(optimizer.param_groups):
333 f"Expected {len(optimizer.param_groups)} lr_lambdas, but got {len(lr_lambda)}"
336 super().__init__(optimizer, last_epoch, verbose)
342 is not the optimizer.
346 … saving or loading the scheduler, please make sure to also save or load the state of the optimizer.
351 if key not in ("optimizer", "lr_lambdas")
364 … saving or loading the scheduler, please make sure to also save or load the state of the optimizer.
396 optimizer (Optimizer): Wrapped optimizer.
399 functions, one for each group in optimizer.param_groups.
411 >>> scheduler = MultiplicativeLR(optimizer, lr_lambda=lmbda)
420 optimizer: Optimizer, argument
425 self.optimizer = optimizer
429 self.lr_lambdas = [lr_lambda] * len(optimizer.param_groups)
431 if len(lr_lambda) != len(optimizer.param_groups):
433 f"Expected {len(optimizer.param_groups)} lr_lambdas, but got {len(lr_lambda)}"
436 super().__init__(optimizer, last_epoch, verbose)
442 is not the optimizer.
449 if key not in ("optimizer", "lr_lambdas")
483 for lmbda, group in zip(self.lr_lambdas, self.optimizer.param_groups)
486 return [group["lr"] for group in self.optimizer.param_groups]
496 optimizer (Optimizer): Wrapped optimizer.
510 >>> # Assuming optimizer uses lr = 0.05 for all groups
515 >>> scheduler = StepLR(optimizer, step_size=30, gamma=0.1)
524 optimizer: Optimizer, argument
532 super().__init__(optimizer, last_epoch, verbose)
539 return [group["lr"] for group in self.optimizer.param_groups]
540 return [group["lr"] * self.gamma for group in self.optimizer.param_groups]
556 optimizer (Optimizer): Wrapped optimizer.
570 >>> # Assuming optimizer uses lr = 0.05 for all groups
574 >>> scheduler = MultiStepLR(optimizer, milestones=[30,80], gamma=0.1)
583 optimizer: Optimizer, argument
591 super().__init__(optimizer, last_epoch, verbose)
598 return [group["lr"] for group in self.optimizer.param_groups]
601 for group in self.optimizer.param_groups
621 optimizer (Optimizer): Wrapped optimizer.
635 >>> # Assuming optimizer uses lr = 0.05 for all groups
641 >>> scheduler = ConstantLR(optimizer, factor=0.5, total_iters=4)
650 optimizer: Optimizer, argument
663 super().__init__(optimizer, last_epoch, verbose)
670 return [group["lr"] * self.factor for group in self.optimizer.param_groups]
673 return [group["lr"] for group in self.optimizer.param_groups]
676 group["lr"] * (1.0 / self.factor) for group in self.optimizer.param_groups
695 optimizer (Optimizer): Wrapped optimizer.
713 >>> # Assuming optimizer uses lr = 0.05 for all groups
719 >>> scheduler = LinearLR(optimizer, start_factor=0.5, total_iters=4)
728 optimizer: Optimizer, argument
748 super().__init__(optimizer, last_epoch, verbose)
756 group["lr"] * self.start_factor for group in self.optimizer.param_groups
760 return [group["lr"] for group in self.optimizer.param_groups]
772 for group in self.optimizer.param_groups
794 optimizer (Optimizer): Wrapped optimizer.
806 self, optimizer: Optimizer, gamma: float, last_epoch=-1, verbose="deprecated" argument
809 super().__init__(optimizer, last_epoch, verbose)
816 return [group["lr"] for group in self.optimizer.param_groups]
817 return [group["lr"] * self.gamma for group in self.optimizer.param_groups]
830 optimizer (Optimizer): Wrapped optimizer.
842 >>> # Assuming optimizer uses lr = 1. for all groups
848 >>> scheduler1 = ConstantLR(optimizer, factor=0.1, total_iters=2)
849 >>> scheduler2 = ExponentialLR(optimizer, gamma=0.9)
850 >>> scheduler = SequentialLR(optimizer, schedulers=[scheduler1, scheduler2], milestones=[2])
859 optimizer: Optimizer, argument
871 if not hasattr(scheduler, "optimizer"):
873 … f"{self.__class__.__name__} at index {scheduler_idx} should have `optimizer` as its attribute."
881 if optimizer != scheduler.optimizer:
883 … f"{self.__class__.__name__} expects all schedulers to belong to the same optimizer, but "
884 …"got scheduler {scheduler.__class__.__name__} at index {scheduler_idx} has {scheduler.optimizer}, "
885 f"which is different from {optimizer.__class__.__name__}."
898 self.optimizer = optimizer
901 for group in self.optimizer.param_groups:
929 is not the optimizer.
935 if key not in ("optimizer", "_schedulers")
967 optimizer (Optimizer): Wrapped optimizer.
979 >>> # Assuming optimizer uses lr = 0.001 for all groups
985 >>> scheduler = PolynomialLR(optimizer, total_iters=4, power=1.0)
994 optimizer: Optimizer, argument
1002 super().__init__(optimizer, last_epoch, verbose)
1009 return [group["lr"] for group in self.optimizer.param_groups]
1015 return [group["lr"] * decay_factor for group in self.optimizer.param_groups]
1058 optimizer (Optimizer): Wrapped optimizer.
1075 optimizer: Optimizer, argument
1083 super().__init__(optimizer, last_epoch, verbose)
1090 return [group["lr"] for group in self.optimizer.param_groups]
1097 for base_lr, group in zip(self.base_lrs, self.optimizer.param_groups)
1103 for base_lr, group in zip(self.base_lrs, self.optimizer.param_groups)
1110 for group in self.optimizer.param_groups
1131 optimizer (Optimizer, optional): Wrapped optimizer. Default: None.
1135 >>> # Assuming optimizer uses lr = 1. for all groups
1141 >>> scheduler1 = ConstantLR(optimizer, factor=0.1, total_iters=2)
1142 >>> scheduler2 = ExponentialLR(optimizer, gamma=0.9)
1143 >>> scheduler = ChainedScheduler([scheduler1, scheduler2], optimizer=optimizer)
1151 self, schedulers: Sequence[LRScheduler], optimizer: Optional[Optimizer] = None argument
1158 optimizer = optimizer or schedulers[0].optimizer
1160 if not hasattr(scheduler, "optimizer"):
1162 … f"{self.__class__.__name__} at index {scheduler_idx} should have `optimizer` as its attribute."
1170 if optimizer != scheduler.optimizer:
1172 … f"{self.__class__.__name__} expects all schedulers to belong to the same optimizer, but "
1173 …"got scheduler {scheduler.__class__.__name__} at index {scheduler_idx} has {scheduler.optimizer}, "
1174 f"which is different from {optimizer.__class__.__name__}."
1177 self.optimizer = optimizer
1179 group["lr"] for group in self._schedulers[-1].optimizer.param_groups
1187 group["lr"] for group in self._schedulers[-1].optimizer.param_groups
1194 is not the optimizer.
1200 if key not in ("optimizer", "_schedulers")
1235 optimizer (Optimizer): Wrapped optimizer.
1278 >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
1279 >>> scheduler = ReduceLROnPlateau(optimizer, 'min')
1289 optimizer: Optimizer, argument
1304 # Attach optimizer
1305 if not isinstance(optimizer, Optimizer):
1306 raise TypeError(f"{type(optimizer).__name__} is not an Optimizer")
1307 self.optimizer = optimizer
1310 if len(min_lr) != len(optimizer.param_groups):
1312 f"expected {len(optimizer.param_groups)} min_lrs, got {len(min_lr)}"
1316 self.min_lrs = [min_lr] * len(optimizer.param_groups)
1331 self._last_lr = [group["lr"] for group in self.optimizer.param_groups]
1368 self._last_lr = [group["lr"] for group in self.optimizer.param_groups]
1371 for i, param_group in enumerate(self.optimizer.param_groups):
1413 key: value for key, value in self.__dict__.items() if key != "optimizer"
1445 optimizer (Optimizer): Wrapped optimizer.
1510 >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
1511 >>> scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=0.01, max_lr=0.1)
1525 optimizer: Optimizer, argument
1540 # Attach optimizer
1541 if not isinstance(optimizer, Optimizer):
1542 raise TypeError(f"{type(optimizer).__name__} is not an Optimizer")
1543 self.optimizer = optimizer
1545 base_lrs = _format_param("base_lr", optimizer, base_lr)
1547 for lr, group in zip(base_lrs, optimizer.param_groups):
1554 self.max_lrs = _format_param("max_lr", optimizer, max_lr)
1577 "momentum" not in optimizer.defaults
1578 and "betas" not in optimizer.defaults
1581 "optimizer must support momentum or beta1 with `cycle_momentum` option enabled"
1584 self.use_beta1 = "betas" in self.optimizer.defaults
1586 "base_momentum", optimizer, base_momentum
1588 self.max_momentums = _format_param("max_momentum", optimizer, max_momentum)
1591 self.max_momentums, self.base_momentums, optimizer.param_groups
1600 super().__init__(optimizer, last_epoch, verbose)
1641 updating the optimizer's momentum.
1674 for param_group, momentum in zip(self.optimizer.param_groups, momentums):
1723 optimizer (Optimizer): Wrapped optimizer.
1741 optimizer: Optimizer, argument
1761 super().__init__(optimizer, last_epoch, verbose)
1780 >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult)
1785 >>> optimizer.zero_grad()
1789 >>> optimizer.step()
1796 >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult)
1833 for i, data in enumerate(zip(self.optimizer.param_groups, self.get_lr())):
1837 self._last_lr = [group["lr"] for group in self.optimizer.param_groups]
1879 optimizer (Optimizer): Wrapped optimizer.
1942 >>> optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9)
1943 …>>> scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=0.01, steps_per_epoch=len(da…
1947 >>> optimizer.step()
1957 optimizer: Optimizer, argument
1973 # Validate optimizer
1974 if not isinstance(optimizer, Optimizer):
1975 raise TypeError(f"{type(optimizer).__name__} is not an Optimizer")
1976 self.optimizer = optimizer
2056 max_lrs = _format_param("max_lr", self.optimizer, max_lr)
2058 for idx, group in enumerate(self.optimizer.param_groups):
2067 "momentum" not in self.optimizer.defaults
2068 and "betas" not in self.optimizer.defaults
2071 "optimizer must support momentum or beta1 with `cycle_momentum` option enabled"
2073 self.use_beta1 = "betas" in self.optimizer.defaults
2074 max_momentums = _format_param("max_momentum", optimizer, max_momentum)
2075 base_momentums = _format_param("base_momentum", optimizer, base_momentum)
2078 max_momentums, base_momentums, optimizer.param_groups
2087 super().__init__(optimizer, last_epoch, verbose)
2124 for group in self.optimizer.param_groups: