sometimes only one worker returns

training_functions.py

@@ -207,7 +207,7 @@ class train_segmentation:
                 fut = fut.compute()
                 im = fut.data
                 self.client.restart(wait_for_workers=False)
-                if not len(self.client.cluster.workers)>0:
+                if not len(self.client.cluster.workers)>1:
                         self.client = reboot_client(self.client, memory_limit=self.memlim, n_workers=self.n_workers)
                         
             # if imfirst is not None and type(imfirst) is not np.ndarray:
@@ -375,7 +375,7 @@ class train_segmentation:
                     feat_stack = fut.data
                     self.client.restart(wait_for_workers=False)
                     
-            if not len(self.client.cluster.workers)>0:
+            if not len(self.client.cluster.workers)>1:
                     self.client = reboot_client(self.client, memory_limit=self.memlim, n_workers=self.n_workers)
                     # TODO client reboot if workers can't return
             if type(feat_stack_t_idp) is not np.ndarray:
@@ -384,7 +384,7 @@ class train_segmentation:
                     fut = fut.compute()
                     feat_stack_t_idp = fut.data
                     self.client.restart(wait_for_workers=False)
-            if not len(self.client.cluster.workers)>0:
+            if not len(self.client.cluster.workers)>1:
                     self.client = reboot_client(self.client, memory_limit=self.memlim, n_workers=self.n_workers)
                     
             feat_stack = np.concatenate([feat_stack, feat_stack_t_idp], axis = 2)