Skip to content

Improve engine's cleanup #7813

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sfc-gh-truwase merged 1 commit into from
Jan 25, 2026
Merged

Improve engine's cleanup #7813

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions deepspeed/runtime/engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -565,17 +565,23 @@ def broadcast_and_check(args, bcast_rank, bcast_group):
with_kwargs=True)

def __del__(self):
self.destroy()
try:
self.destroy()
except Exception as exc:
# Avoid destructor-time exceptions for partially initialized engines.
logger.debug("DeepSpeedEngine.__del__ cleanup skipped: %s", exc, exc_info=True)

def destroy(self):
if self.optimizer is not None and hasattr(self.optimizer, 'destroy'):
self.optimizer.destroy()
optimizer = getattr(self, "optimizer", None)
if optimizer is not None and hasattr(optimizer, 'destroy'):
optimizer.destroy()
if self.is_deepcompile_active():
get_deepcompile_handle().cleanup()
debug_clear_module_and_param_names()

if self.checkpoint_engine is not None and self.checkpoint_engine.is_decoupled():
self.checkpoint_engine.cleanup()
checkpoint_engine = getattr(self, "checkpoint_engine", None)
if checkpoint_engine is not None and checkpoint_engine.is_decoupled():
checkpoint_engine.cleanup()

def _get_model_parameters(self):
if self.autotuning_profile_model_info():
Expand Down Expand Up @@ -4376,7 +4382,7 @@ def is_deepcompile_enabled(self) -> bool:
return self._config.compile_config.deepcompile

def is_deepcompile_active(self) -> bool:
return self._deepcompile_active
return getattr(self, "_deepcompile_active", False)

@property
def is_compiled(self) -> bool:
Expand Down