From 3875b33a87164b6bfb1393383f7297c3173e9b8e Mon Sep 17 00:00:00 2001 From: Tizen Ye Date: Sat, 20 Aug 2022 01:09:06 +0800 Subject: [PATCH 1/2] fix #2240: wrong time unit in flops_profiler --- deepspeed/moe/sharded_moe.py | 8 ++++---- deepspeed/runtime/engine.py | 14 ++++++-------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/deepspeed/moe/sharded_moe.py b/deepspeed/moe/sharded_moe.py index 727b2baec0a6..211b2127b026 100644 --- a/deepspeed/moe/sharded_moe.py +++ b/deepspeed/moe/sharded_moe.py @@ -431,7 +431,7 @@ def forward( if self.wall_clock_breakdown: self.timers('TopKGate').stop() - self.gate_time = self.timers('TopKGate').elapsed(reset=False) * 1000 + self.gate_time = self.timers('TopKGate').elapsed(reset=False) return gate_output @@ -535,7 +535,7 @@ def forward(self, *input: Tensor, **kwargs: Any) -> Tensor: if self.wall_clock_breakdown: self.timers('falltoall').stop() - self.time_falltoall = self.timers('falltoall').elapsed(reset=False) * 1000 + self.time_falltoall = self.timers('falltoall').elapsed(reset=False) # Re-shape after all-to-all: ecm -> gecm dispatched_input = dispatched_input.reshape(self.ep_size, @@ -552,7 +552,7 @@ def forward(self, *input: Tensor, **kwargs: Any) -> Tensor: if self.wall_clock_breakdown: self.timers('salltoall').stop() - self.time_salltoall = self.timers('salltoall').elapsed(reset=False) * 1000 + self.time_salltoall = self.timers('salltoall').elapsed(reset=False) # Re-shape back: gecm -> ecm expert_output = expert_output.reshape(self.ep_size * self.num_local_experts, @@ -576,6 +576,6 @@ def forward(self, *input: Tensor, **kwargs: Any) -> Tensor: if self.wall_clock_breakdown: self.timers('moe').stop() - self.time_moe = self.timers('moe').elapsed(reset=False) * 1000 + self.time_moe = self.timers('moe').elapsed(reset=False) return a diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index 13d58ff11438..a7f83ecc6b8f 100644 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -2049,7 +2049,7 @@ def step(self, lr_kwargs=None): if self.has_moe_layers: fwd_time = self.timers(FORWARD_GLOBAL_TIMER).elapsed( - reset=False) * 1000 + reset=False) self.print_forward_breakdown(fwd_time=fwd_time) self.timers.log(self.engine_timers.global_timers) @@ -2093,29 +2093,27 @@ def _write_monitor(self): self.summary_events = [ ( f"Train/Samples/elapsed_time_ms_forward", - self.timers(FORWARD_GLOBAL_TIMER).elapsed(reset=False) * 1000.0, + self.timers(FORWARD_GLOBAL_TIMER).elapsed(reset=False), self.global_samples, ), ( f"Train/Samples/elapsed_time_ms_backward", - self.timers(BACKWARD_GLOBAL_TIMER).elapsed(reset=False) * 1000.0, + self.timers(BACKWARD_GLOBAL_TIMER).elapsed(reset=False), self.global_samples, ), ( f"Train/Samples/elapsed_time_ms_backward_inner", - self.timers(BACKWARD_INNER_GLOBAL_TIMER).elapsed(reset=False) * - 1000.0, + self.timers(BACKWARD_INNER_GLOBAL_TIMER).elapsed(reset=False), self.global_samples, ), ( f"Train/Samples/elapsed_time_ms_backward_allreduce", - self.timers(BACKWARD_REDUCE_GLOBAL_TIMER).elapsed(reset=False) * - 1000.0, + self.timers(BACKWARD_REDUCE_GLOBAL_TIMER).elapsed(reset=False), self.global_samples, ), ( f"Train/Samples/elapsed_time_ms_step", - self.timers(STEP_GLOBAL_TIMER).elapsed(reset=False) * 1000.0, + self.timers(STEP_GLOBAL_TIMER).elapsed(reset=False), self.global_samples, ), ] From 535c0b69e0715adf6f6928818c83a711fb3105e5 Mon Sep 17 00:00:00 2001 From: Tizen Ye Date: Sat, 20 Aug 2022 11:02:22 +0800 Subject: [PATCH 2/2] fix #2240: make yapf happy --- deepspeed/runtime/engine.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index a7f83ecc6b8f..7d883f74f0f4 100644 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -2048,8 +2048,7 @@ def step(self, lr_kwargs=None): self._write_monitor() if self.has_moe_layers: - fwd_time = self.timers(FORWARD_GLOBAL_TIMER).elapsed( - reset=False) + fwd_time = self.timers(FORWARD_GLOBAL_TIMER).elapsed(reset=False) self.print_forward_breakdown(fwd_time=fwd_time) self.timers.log(self.engine_timers.global_timers)