Skip to content

SeetaResearch / SeetaDet

Go to a project
Commit 4bcab266 by Ting PAN
Options

Plan the queueing of testing images

1 parent f4ecc7c7
Inline Side-by-side
Showing with 76 additions and 76 deletions
CHANGES
------------------------------------------------------------------------
The list of most significant changes made over time in SeetaDet.
SeetaDet 0.4.1 (20200421)
Dragon Minimum Required (Version 0.3.0.dev20200421)
Changes:
- Plan the queueing of testing images instead of reading them all.
Preview Features:
- None
Bugs fixed:
- None
------------------------------------------------------------------------
SeetaDet 0.4.0 (20200408)
Dragon Minimum Required (Version 0.3.0.dev20200408)
......
README.md
......@@ -14,7 +14,7 @@ The torch-style codes help us to simplify the hierarchical pipeline of modern de
## Requirements
seeta-dragon >= 0.3.0.dev20200408
seeta-dragon >= 0.3.0.dev20200421
## Installation
......
configs/retinanet/coco_retinanet_400_R-50-FPN.yml → configs/retinanet/coco_retinanet_416_R-50-FPN.yml
......@@ -22,10 +22,10 @@ MODEL:
NUM_CLASSES: 81
SOLVER:
BASE_LR: 0.01
DECAY_STEPS: [60000, 80000]
MAX_STEPS: 90000
DECAY_STEPS: [120000, 160000]
MAX_STEPS: 180000
SNAPSHOT_EVERY: 5000
SNAPSHOT_PREFIX: coco_retinanet_400
SNAPSHOT_PREFIX: coco_retinanet_416
FPN:
RPN_MIN_LEVEL: 3
RPN_MAX_LEVEL: 7
......@@ -34,15 +34,13 @@ TRAIN:
DATASET: '/data/coco_2014_trainval35k'
USE_DIFF: False # Do not use crowd objects
USE_COLOR_JITTER: True
IMS_PER_BATCH: 8
SCALES: [400]
MAX_SIZE: 666
RANDOM_SCALES: [0.75, 1.0]
IMS_PER_BATCH: 16
SCALES: [416]
RANDOM_SCALES: [0.25, 1.0]
TEST:
DATASET: '/data/coco_2014_minival'
JSON_FILE: '/data/instances_minival2014.json'
PROTOCOL: 'coco'
IMS_PER_BATCH: 1
SCALES: [400]
MAX_SIZE: 666
SCALES: [416]
NMS: 0.5
\ No newline at end of file
configs/retinanet/voc_retinanet_320_AirNet-FPN.yml
......@@ -26,7 +26,7 @@ TRAIN:
USE_COLOR_JITTER: True
IMS_PER_BATCH: 32
SCALES: [320]
RANDOM_SCALES: [0.5, 1.0]
RANDOM_SCALES: [0.25, 1.0]
TEST:
DATASET: '/data/voc_2007_test'
PROTOCOL: 'voc2007' # 'voc2007', 'voc2010', 'coco'
......
configs/retinanet/voc_retinanet_320_R-50-FPN.yml
......@@ -27,7 +27,7 @@ TRAIN:
USE_COLOR_JITTER: True
IMS_PER_BATCH: 32
SCALES: [320]
RANDOM_SCALES: [0.5, 2.0]
RANDOM_SCALES: [0.25, 2.0]
TEST:
DATASET: '/data/voc_2007_test'
PROTOCOL: 'voc2007' # 'voc2007', 'voc2010', 'coco'
......
seetadet/core/test_engine.py
......@@ -23,7 +23,7 @@ from seetadet.utils import time_util
from seetadet.utils.vis import vis_one_image
def run_test_net(checkpoint, server, devices):
def run_test_net(checkpoint, server, devices, read_every=1000):
classes = server.classes
num_images = server.num_images
num_classes = server.num_classes
......@@ -60,17 +60,21 @@ def run_test_net(checkpoint, server, devices):
for process in workers:
process.start()
for i in range(num_images):
num_sends = 0
for count in range(num_images):
if count >= num_sends:
num_to_send = min(read_every, num_images - num_sends)
for i in range(count, count + num_to_send):
image_id, raw_image = server.get_image()
queues[i % num_workers].put((i, raw_image))
# Hold the image until the visualization
if cfg.VIS or cfg.VIS_ON_FILE:
vis_image_dict[i] = (image_id, raw_image)
num_sends += num_to_send
if num_sends == num_images:
for i in range(num_workers):
queues[i].put((-1, None))
for count in range(num_images):
i, time_diffs, results = queues[-1].get()
# Unpack the diverse results
......
seetadet/datasets/kpl_record.py
......@@ -36,7 +36,7 @@ class KPLRecordDataset(Dataset):
def dump_detections(self, all_boxes, output_dir):
dataset = self.cls(self.source)
for file in ('data.data', 'data.index', 'data.meta'):
for file in ('root.data', 'root.index', 'root.meta'):
file = os.path.join(output_dir, file)
if os.path.exists(file):
os.remove(file)
......
tools/export.py
......@@ -13,14 +13,13 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import sys
sys.path.insert(0, '..')
import argparse
import dragon.vm.torch as torch
import pprint
sys.path.insert(0, '..')
from seetadet import onnx as _
from seetadet.core.config import cfg
from seetadet.core.coordinator import Coordinator
......@@ -30,12 +29,14 @@ from seetadet.utils import logger
def parse_args():
"""Parse input arguments"""
parser = argparse.ArgumentParser(description='Export a Detection Network')
parser = argparse.ArgumentParser(
description='Export a detection network into the onnx model')
parser.add_argument('--cfg', dest='cfg_file',
help='optional config file', default=None, type=str)
help='optional config file',
default=None, type=str)
parser.add_argument('--exp_dir', dest='exp_dir',
help='experiment dir',
default=None, type=str)
default='', type=str)
parser.add_argument('--input_shape', dest='input_shape',
help='The shape of dummy input',
default=(1, 224, 224, 3), type=tuple)
......@@ -50,16 +51,7 @@ def parse_args():
if __name__ == '__main__':
args = parse_args()
if args.exp_dir is None or \
not os.path.exists(args.exp_dir):
raise ValueError(
'Excepted a existing experiment dir. \nGot {}.'
.format(os.path.abspath(args.exp_dir))
)
logger.info('Called with args:')
logger.info(args)
logger.info('Called with args:\n' + str(args))
coordinator = Coordinator(args.cfg_file, exp_dir=args.exp_dir)
logger.info('Using config:\n' + pprint.pformat(cfg))
......
tools/mpi_train.py
......@@ -13,14 +13,13 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import sys
sys.path.insert(0, '..')
import argparse
import dragon
import numpy
sys.path.insert(0, '..')
from seetadet.core.config import cfg
from seetadet.core.coordinator import Coordinator
from seetadet.core.train import train_net
......@@ -30,13 +29,14 @@ from seetadet.utils import logger
def parse_args():
"""Parse input arguments."""
parser = argparse.ArgumentParser(description='Train a Fast R-CNN network')
parser = argparse.ArgumentParser(
description='Train a detection network with mpi utilities')
parser.add_argument('--cfg', dest='cfg_file',
help='config file',
default=None, type=str)
parser.add_argument('--exp_dir', dest='exp_dir',
help='experiment dir',
default=None, type=str)
default='', type=str)
if len(sys.argv) == 1:
parser.print_help()
......@@ -49,13 +49,6 @@ def parse_args():
if __name__ == '__main__':
args = parse_args()
if args.exp_dir is None or \
not os.path.exists(args.exp_dir):
raise ValueError(
'Excepted a existing experiment dir. \nGot {}.'
.format(os.path.abspath(args.exp_dir))
)
coordinator = Coordinator(args.cfg_file, exp_dir=args.exp_dir)
checkpoint, start_iter = coordinator.checkpoint(wait=False)
......
tools/test.py
......@@ -13,13 +13,12 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import sys
sys.path.insert(0, '..')
import argparse
import pprint
sys.path.insert(0, '..')
from seetadet.core import test_engine
from seetadet.core.config import cfg
from seetadet.core.coordinator import Coordinator
......@@ -30,7 +29,8 @@ from seetadet.utils import logger
def parse_args():
"""Parse input arguments"""
parser = argparse.ArgumentParser(description='Test a Detection Network')
parser = argparse.ArgumentParser(
description='Test a detection network with a specified checkpoint')
parser.add_argument('--gpus', dest='gpus',
help='index of GPUs to use',
default=None, nargs='+', type=int)
......@@ -39,12 +39,15 @@ def parse_args():
default=None, type=str)
parser.add_argument('--exp_dir', dest='exp_dir',
help='experiment dir',
default=None, type=str)
default='', type=str)
parser.add_argument('--output_dir', dest='output_dir',
help='output dir',
default=None, type=str)
parser.add_argument('--iter', dest='iter', help='global step',
default=None, type=int)
parser.add_argument('--read_every', dest='read_every',
help='read every n images for testing',
default=1000, type=int)
parser.add_argument('--dump', dest='dump',
help='dump the result back to record?',
action='store_true')
......@@ -62,16 +65,7 @@ def parse_args():
if __name__ == '__main__':
args = parse_args()
if args.exp_dir is None or \
not os.path.exists(args.exp_dir):
raise ValueError(
'Excepted a existing experiment dir. \nGot {}.'
.format(os.path.abspath(args.exp_dir))
)
logger.info('Called with args:')
logger.info(args)
logger.info('Called with args:\n' + str(args))
coordinator = Coordinator(args.cfg_file, args.exp_dir)
logger.info('Using config:\n' + pprint.pformat(cfg))
......@@ -79,7 +73,10 @@ if __name__ == '__main__':
# Load the checkpoint and test engine
checkpoint, _ = coordinator.checkpoint(args.iter, wait=args.wait)
if checkpoint is None:
raise RuntimeError('The checkpoint of global step {} does not exist.'.format(args.iter))
raise RuntimeError(
'The checkpoint of step {} does not exist.'
.format(args.iter)
)
# Inspect the dataset
dataset = get_dataset(cfg.TEST.DATASET)
......@@ -93,4 +90,4 @@ if __name__ == '__main__':
# Bind the server and run the test
server = TestServer(coordinator.results_dir(checkpoint))
test_engine.run_test_net(checkpoint, server, args.gpus)
test_engine.run_test_net(checkpoint, server, args.gpus, args.read_every)
tools/test_all.py
......@@ -15,23 +15,25 @@ from __future__ import print_function
import os
import sys
sys.path.insert(0, '..')
import argparse
import numpy
sys.path.insert(0, '..')
from seetadet.core.coordinator import Coordinator
from seetadet.utils import logger
def parse_args():
"""Parse input arguments"""
parser = argparse.ArgumentParser(description='Test a Detection Network')
parser = argparse.ArgumentParser(
description='Test a detection network with all checkpoints')
parser.add_argument('--cfg', dest='cfg_file',
help='optional config file', default=None, type=str)
help='optional config file',
default=None, type=str)
parser.add_argument('--exp_dir', dest='exp_dir',
help='experiment dir',
default=None, type=str)
default='', type=str)
if len(sys.argv) == 1:
parser.print_help()
......@@ -52,13 +54,7 @@ def test(cfg_file, exp_dir, global_step):
if __name__ == '__main__':
args = parse_args()
if args.exp_dir is None or \
not os.path.exists(args.exp_dir):
raise ValueError(
'Excepted a existing experiment dir. \nGot {}.'
.format(os.path.abspath(args.exp_dir))
)
logger.info('Called with args:\n' + str(args))
coordinator = Coordinator(args.cfg_file, exp_dir=args.exp_dir)
......@@ -66,7 +62,7 @@ if __name__ == '__main__':
files = os.listdir(coordinator.checkpoints_dir())
for file in files:
step = int(file.split('_iter_')[-1].split(b'.')[0])
step = int(file.split('_iter_')[-1].split('.')[0])
global_steps.append(step)
order = numpy.argsort(-numpy.array(global_steps))
......
tools/train.py
......@@ -13,15 +13,15 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import sys
sys.path.insert(0, '..')
import os.path as osp
import argparse
import dragon
import numpy
import pprint
sys.path.insert(0, '..')
from seetadet.core.config import cfg
from seetadet.core.coordinator import Coordinator
from seetadet.core.train import train_net
......@@ -31,7 +31,8 @@ from seetadet.utils import logger
def parse_args():
"""Parse input arguments."""
parser = argparse.ArgumentParser(description='Train a Detection Network')
parser = argparse.ArgumentParser(
description='Train a detection network')
parser.add_argument('--cfg', dest='cfg_file',
help='optional config file',
default=None, type=str)
......@@ -61,7 +62,7 @@ def mpi_train(cfg_file, exp_dir):
import subprocess
args = 'mpirun --allow-run-as-root -n {} --bind-to none '.format(cfg.NUM_GPUS)
args += '{} {} '.format(sys.executable, 'mpi_train.py')
args += '--cfg {} --exp_dir {} '.format(osp.abspath(cfg_file), exp_dir)
args += '--cfg {} --exp_dir {} '.format(os.path.abspath(cfg_file), exp_dir)
return subprocess.call(args, shell=True)
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!