一文搞定PyTorch中优化器optimizer的所有属性和方法

class SGD(Optimizer):
    def __init__(self, params, lr=required, momentum=0, dampening=0,
                 weight_decay=0, nesterov=False):
        if lr is not required and lr < 0.0:
            raise ValueError("Invalid learning rate:{}".format(lr))
        if momentum < 0.0:
            raise ValueError("Invalid momentum value:{}".format(momentum))
        if weight_decay < 0.0:
            raise ValueError("Invalid weight_decay value:{}".format(weight_decay))

        defaults=dict(lr=lr, momentum=momentum, dampening=dampening,
                        weight_decay=weight_decay, nesterov=nesterov)
        if nesterov and (momentum <=0 or dampening !=0):
            raise ValueError("Nesterov momentum requires a momentum and zero dampening")
        super(SGD, self).__init__(params, defaults)

class Optimizer(object):
    Args:
        params (iterable): an iterable of :class:`torch.Tensor` s or
            :class:`dict` s. Specifies what Tensors should be optimized.
        defaults: (dict): a dict containing default values of optimization
            options (used when a parameter group doesn't specify them).
    """

    def __init__(self, params, defaults):
        torch._C._log_api_usage_once("python.optimizer")
        self.defaults=defaults

        self._hook_for_profile()

        if isinstance(params, torch.Tensor):
            raise TypeError("params argument given to the optimizer should be "
                            "an iterable of Tensors or dicts, but got " +
                            torch.typename(params))

        self.state=defaultdict(dict)
        self.param_groups=[]

        param_groups=list(params)          #   1
        if len(param_groups)==0:
            raise ValueError("optimizer got an empty parameter list")
        if not isinstance(param_groups[0], dict):
            param_groups=[{'params': param_groups}]#  2

        for param_group in param_groups:   # 3
            self.add_param_group(param_group)

optimizer=optim.SGD(
***************************
    net.parameters(),               # params的一种形式
***************************
    lr=LR, momentum=0.9
)

    def add_param_group(self, param_group):
        r"""Add a param group to the :class:`Optimizer` s `param_groups`.

        This can be useful when fine tuning a pre-trained network as frozen layers can be made
        trainable and added to the :class:`Optimizer` as training progresses.

        Args:
            param_group (dict): Specifies what Tensors should be optimized along with group
            specific optimization options.
        """
        assert isinstance(param_group, dict), "param group must be a dict"        #  1

        params=param_group['params']# 2
        if isinstance(params, torch.Tensor):
            param_group['params']=[params]
        elif isinstance(params, set):
            raise TypeError('optimizer parameters need to be organized in ordered collections, but '
                            'the ordering of tensors in sets will change between runs. Please use a list instead.')
        else:
            param_group['params']=list(params)  # 3

        for param in param_group['params']:       # 4
            if not isinstance(param, torch.Tensor):
                raise TypeError("optimizer can only optimize Tensors, "
                                "but one of the params is " + torch.typename(param))
            if not param.is_leaf:
                raise ValueError("can't optimize a non-leaf Tensor")

        for name, default in self.defaults.items():    # 5
            if default is required and name not in param_group:
                raise ValueError("parameter group didn't specify a value of required optimization parameter " +
                                 name)
            else:
                param_group.setdefault(name, default)

        params=param_group['params']
        if len(params) !=len(set(params)):
            warnings.warn("optimizer contains a parameter group with duplicate parameters; "
                          "in future, this will cause an error; "
                          "see github.com/pytorch/pytorch/issues/40967 for more information", stacklevel=3)

        param_set=set()         # 6
        for group in self.param_groups:
            param_set.update(set(group['params']))

        if not param_set.isdisjoint(set(param_group['params'])):     # 7
            raise ValueError("some parameters appear in more than one parameter group")

        self.param_groups.append(param_group) # 8

defaults=dict(lr=lr, momentum=momentum, dampening=dampening,
                        weight_decay=weight_decay, nesterov=nesterov)

构建SGD实例后打印一下defaults这个变量的结果：
{'lr': 0.01,
 'momentum': 0.9,
 'dampening': 0,
 'weight_decay': 0,
 'nesterov': False}

class _RequiredParameter(object):
    """Singleton class representing a required parameter for an Optimizer."""
    def __repr__(self):
        return "<required parameter>"

required=_RequiredParameter()

fcParamsId=list(map(id, resnet18_ft.fc.parameters()))     # 返回的是parameters的 内存地址
features_params=filter(lambda p: id(p) not in fcParamsId, resnet18_ft.parameters())

optimizer=optim.SGD(
*************************************************************
[{'params': features_params, 'lr': LR * 0.1},                         #  这个列表是params的另一种形式
{'params': resnet18_ft.fc.parameters()}], 
***************************************************************
    'lr': LR, momentum=0.9
)

def step(self, closure=None):
    ......
    for group in self.param_groups:
        ......
        for p in group['params']:
            ......
    F.sgd(params_with_grad,
                  d_p_list,
                  momentum_buffer_list,
                  weight_decay=weight_decay,
                  momentum=momentum,
                  lr=lr,
                  dampening=dampening,
                  nesterov=nesterov)