File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change 2323from paddle .framework import (
2424 core ,
2525)
26- from paddle .io import (
27- DataLoader ,
28- )
2926
3027from deepmd .common import (
3128 symlink_prefix_files ,
5855 DEFAULT_PRECISION ,
5956 DEVICE ,
6057 JIT ,
61- NUM_WORKERS ,
6258 SAMPLER_RECORD ,
6359 enable_prim ,
6460)
@@ -180,8 +176,12 @@ def get_dataloader_and_buffer(_data, _params):
180176 # )
181177 # _data_buffered = BufferedIterator(iter(_dataloader))
182178 # return _dataloader, _data_buffered
183-
184- from itertools import chain , cycle
179+
180+ from itertools import (
181+ chain ,
182+ cycle ,
183+ )
184+
185185 all_dataloaders = []
186186 self .all_dlen = 0
187187 for dataloader in _data .dataloaders :
@@ -192,7 +192,9 @@ def get_dataloader_and_buffer(_data, _params):
192192 self .all_dlen += dlen
193193 all_dataloaders .append (shard_dataloader )
194194 _shard_dataloader = cycle (chain (* all_dataloaders ))
195- _data_buffered = BufferedIterator (iter (_shard_dataloader ),self .all_dlen )
195+ _data_buffered = BufferedIterator (
196+ iter (_shard_dataloader ), self .all_dlen
197+ )
196198 return _shard_dataloader , _data_buffered
197199
198200 training_dataloader , training_data_buffered = get_dataloader_and_buffer (
Original file line number Diff line number Diff line change @@ -182,8 +182,7 @@ def construct_dataset(system):
182182 system_batch_sampler = BatchSampler (
183183 system ,
184184 shuffle = (
185- (not (dist .is_available () and dist .is_initialized ()))
186- and shuffle
185+ (not (dist .is_available () and dist .is_initialized ())) and shuffle
187186 ),
188187 batch_size = int (batch_size ),
189188 )
You can’t perform that action at this time.
0 commit comments