|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | +# |
| 4 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +# you may not use this file except in compliance with the License. |
| 6 | +# You may obtain a copy of the License at |
| 7 | +# |
| 8 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +# |
| 10 | +# Unless required by applicable law or agreed to in writing, software |
| 11 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +# See the License for the specific language governing permissions and |
| 14 | +# limitations under the License. |
| 15 | + |
| 16 | +from __future__ import annotations |
| 17 | + |
| 18 | +import logging |
| 19 | +from typing import List |
| 20 | + |
| 21 | +from opacus.grad_sample.functorch import ft_compute_per_sample_gradient |
| 22 | +from opacus.grad_sample.grad_sample_module_fast_gradient_clipping import ( |
| 23 | + GradSampleModuleFastGradientClipping, |
| 24 | +) |
| 25 | +from opacus.utils.module_utils import requires_grad, trainable_parameters |
| 26 | + |
| 27 | +import torch |
| 28 | +import torch.nn as nn |
| 29 | + |
| 30 | + |
| 31 | +logger = logging.getLogger(__name__) |
| 32 | +logger.disabled = True |
| 33 | + |
| 34 | + |
| 35 | +class GradSampleModuleFastGradientClippingFSDP(GradSampleModuleFastGradientClipping): |
| 36 | + """ |
| 37 | + Hooks-based implementation of GradSampleModule with Fast Gradient and Ghost Clipping |
| 38 | +
|
| 39 | + Computes norms of gradients without gradient instantiation |
| 40 | + """ |
| 41 | + |
| 42 | + def __init__( |
| 43 | + self, |
| 44 | + m: nn.Module, |
| 45 | + *, |
| 46 | + batch_first=True, |
| 47 | + loss_reduction="mean", |
| 48 | + strict: bool = True, |
| 49 | + max_grad_norm=1, |
| 50 | + ): |
| 51 | + """ |
| 52 | +
|
| 53 | + Args: |
| 54 | + m: nn.Module to be wrapped |
| 55 | + batch_first: Flag to indicate if the input tensor to the corresponding module |
| 56 | + has the first dimension representing the batch. If set to True, dimensions on |
| 57 | + input tensor are expected be ``[batch_size, ...]``, otherwise |
| 58 | + ``[K, batch_size, ...]`` |
| 59 | + loss_reduction: Indicates if the loss reduction (for aggregating the gradients) |
| 60 | + is a sum or a mean operation. Can take values "sum" or "mean" |
| 61 | + max_grad_norm: The value at which gradients are to be clipped. |
| 62 | + strict: If set to True, the input module will be validated to make sure that |
| 63 | + it does not have buffers in all its submodules. |
| 64 | +
|
| 65 | + Raises: |
| 66 | + NotImplementedError |
| 67 | + If ``strict`` is set to ``True`` and module ``m`` (or any of its |
| 68 | + submodules) includes a buffer. |
| 69 | + """ |
| 70 | + |
| 71 | + super().__init__( |
| 72 | + m, |
| 73 | + batch_first=batch_first, |
| 74 | + loss_reduction=loss_reduction, |
| 75 | + strict=strict, |
| 76 | + force_functorch=False, |
| 77 | + max_grad_norm=max_grad_norm, |
| 78 | + use_ghost_clipping=True, |
| 79 | + ) |
| 80 | + |
| 81 | + self.sampler_classes = list(self.GRAD_SAMPLERS.keys()) + list( |
| 82 | + self.NORM_SAMPLERS.keys() |
| 83 | + ) |
| 84 | + self.module_types = {} |
| 85 | + for m in self.iterate_submodules(m): |
| 86 | + if type(m) not in self.module_types: |
| 87 | + module_type = next( |
| 88 | + (i for i in self.sampler_classes if isinstance(m, i)), |
| 89 | + type(m), |
| 90 | + ) |
| 91 | + self.module_types[type(m)] = module_type |
| 92 | + |
| 93 | + def get_clipping_coef(self) -> torch.Tensor: |
| 94 | + """Get per-example gradient scaling factor for clipping.""" |
| 95 | + norm_sample = self.get_norm_sample() |
| 96 | + return (self.max_grad_norm / (norm_sample + 1e-6)).clamp(max=1.0) |
| 97 | + |
| 98 | + def get_norm_sample(self) -> torch.Tensor: |
| 99 | + """Get per-example gradient norms.""" |
| 100 | + norm_sample = torch.stack( |
| 101 | + [ |
| 102 | + per_param_norm |
| 103 | + for module in self.iterate_submodules(self._module) |
| 104 | + for per_param_norm in module.norm_sample |
| 105 | + ], |
| 106 | + dim=0, |
| 107 | + ).norm(2, dim=0) |
| 108 | + |
| 109 | + self.per_sample_gradient_norms = norm_sample |
| 110 | + return norm_sample |
| 111 | + |
| 112 | + def capture_activations_hook( |
| 113 | + self, |
| 114 | + module: nn.Module, |
| 115 | + forward_input: List[torch.Tensor], |
| 116 | + _forward_output: torch.Tensor, |
| 117 | + ): |
| 118 | + if ( |
| 119 | + not requires_grad(module) |
| 120 | + or not module.training |
| 121 | + or not torch.is_grad_enabled() |
| 122 | + or not self.hooks_enabled |
| 123 | + ): |
| 124 | + return |
| 125 | + |
| 126 | + if not hasattr(module, "activations"): |
| 127 | + module.activations = [] |
| 128 | + module.activations.append([t.detach() for t in forward_input]) # pyre-ignore |
| 129 | + |
| 130 | + if not hasattr(module, "forward_counter"): |
| 131 | + module.forward_counter = 0 |
| 132 | + |
| 133 | + module.forward_counter += 1 |
| 134 | + if self.use_ghost_clipping and module.forward_counter > 1: |
| 135 | + raise NotImplementedError("Parameter tying is not supported with FSDP") |
| 136 | + |
| 137 | + def capture_backprops_hook( |
| 138 | + self, |
| 139 | + module: nn.Module, |
| 140 | + _forward_input: torch.Tensor, |
| 141 | + forward_output: torch.Tensor, |
| 142 | + loss_reduction: str, |
| 143 | + batch_first: bool, |
| 144 | + ): |
| 145 | + """ |
| 146 | + Computes norms of per sample gradient given the current backprops and activations |
| 147 | + stored by the associated forward hook. Computed per sample gradient norms are |
| 148 | + stored in ``norm_sample`` field in each parameter. |
| 149 | +
|
| 150 | + Args: |
| 151 | + module: nn.Module, |
| 152 | + _forward_input: torch.Tensor, |
| 153 | + forward_output: torch.Tensor, |
| 154 | + loss_reduction: str, |
| 155 | + batch_first: bool, |
| 156 | + """ |
| 157 | + if not self.hooks_enabled: |
| 158 | + return |
| 159 | + |
| 160 | + backprops = forward_output[0].detach() |
| 161 | + |
| 162 | + activations, backprops = self.rearrange_grad_samples( |
| 163 | + module=module, |
| 164 | + backprops=backprops, |
| 165 | + loss_reduction=loss_reduction, |
| 166 | + batch_first=batch_first, |
| 167 | + ) |
| 168 | + |
| 169 | + if not hasattr(module, "norm_sample"): |
| 170 | + # currently, we don't support freezing and unfreezing params in between training. Making this a dictionary and mapping with param names might fix this. |
| 171 | + module.norm_sample = [] |
| 172 | + for _, param in trainable_parameters(module): |
| 173 | + module.norm_sample.append( |
| 174 | + torch.zeros( |
| 175 | + torch.Size([module.max_batch_len, 1]), |
| 176 | + device=param.device, |
| 177 | + dtype=param.dtype, |
| 178 | + ) |
| 179 | + ) |
| 180 | + |
| 181 | + module_type = self.module_types[type(module)] |
| 182 | + module.forward_counter -= 1 |
| 183 | + if self.use_ghost_clipping and module_type in self.NORM_SAMPLERS: |
| 184 | + norm_sampler_fn = self.NORM_SAMPLERS[module_type] |
| 185 | + norm_samples = norm_sampler_fn(module, activations, backprops) |
| 186 | + |
| 187 | + for idx, (_, ns) in enumerate( |
| 188 | + (item for item in norm_samples.items() if item[0].requires_grad) |
| 189 | + ): |
| 190 | + module.norm_sample[idx] = ns |
| 191 | + else: |
| 192 | + if not self.force_functorch and module_type in self.GRAD_SAMPLERS: |
| 193 | + grad_sampler_fn = self.GRAD_SAMPLERS[module_type] |
| 194 | + else: |
| 195 | + grad_sampler_fn = ft_compute_per_sample_gradient |
| 196 | + |
| 197 | + grad_samples = grad_sampler_fn(module, activations, backprops) |
| 198 | + |
| 199 | + for idx, (_, gs) in enumerate((item for item in grad_samples.items())): |
| 200 | + module.norm_sample[idx] = gs.reshape(len(gs), -1).norm(2, dim=-1) |
| 201 | + del grad_samples |
| 202 | + |
| 203 | + if len(module.activations) == 0: |
| 204 | + if hasattr(module, "max_batch_len"): |
| 205 | + del module.max_batch_len |
| 206 | + |
| 207 | + @property |
| 208 | + def per_sample_gradient_norms(self) -> torch.Tensor: |
| 209 | + """Returns per sample gradient norms. Note that these are not privatized and should only be used for debugging purposes or in non-private settings""" |
| 210 | + if self._per_sample_gradient_norms is not None: |
| 211 | + return self._per_sample_gradient_norms |
| 212 | + else: |
| 213 | + raise AttributeError( |
| 214 | + "per_sample_gradient_norms is not set. Please call forward and backward on the model before accessing this property." |
| 215 | + ) |
| 216 | + |
| 217 | + @per_sample_gradient_norms.setter |
| 218 | + def per_sample_gradient_norms(self, value): |
| 219 | + self._per_sample_gradient_norms = value |
0 commit comments