Disable tensorboard for non-root nodes

lib/core/train.py

@@ -52,7 +52,7 @@ class SolverWrapper(object):
 
         # Plan the metrics
         self.metrics = collections.OrderedDict()
-        if cfg.ENABLE_TENSOR_BOARD:
+        if cfg.ENABLE_TENSOR_BOARD and logger.is_root():
             from dragon.tools.tensorboard import TensorBoard
             self.board = TensorBoard(log_dir=coordinator.experiment_dir + '/logs')
 

tools/mpi_train.py

@@ -70,6 +70,8 @@ if __name__ == '__main__':
             'Excepted staring of {} processes, got {}.'
             .format(cfg.NUM_GPUS, world_size)
         )
+
+    # Setup the logging modules
     logger.set_root_logger(world_rank == 0)
 
     # Select the GPU depending on the rank of process