We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 355d9d8 commit 3769092Copy full SHA for 3769092
deepspeed/runtime/zero/parameter_offload.py
@@ -492,7 +492,8 @@ def _run_after_backward_function(sub_module):
492
493
def pre_sub_module_forward_function(self, sub_module):
494
see_memory_usage(f"Before sub module function {sub_module.__class__.__name__}", force=False)
495
- prev_grad_state = torch.is_grad_enabled() # we don't want to enable grad for sub modules fetching, yet the subfunction need to know if grad is enabled
+ prev_grad_state = torch.is_grad_enabled(
496
+ ) # we don't want to enable grad for sub modules fetching, yet the subfunction need to know if grad is enabled
497
torch.set_grad_enabled(False)
498
global FWD_MODULE_STACK
499
FWD_MODULE_STACK.append(sub_module)
0 commit comments