Skip to content

SeetaResearch / Dragon

Go to a project
Commit 990c496b by Ting PAN
Options

python3 support

1 parent b6182abc
Inline Side-by-side
Showing with 520 additions and 285 deletions
Dragon/CMakeLists.txt
...@@ -8,6 +8,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8.0) ...@@ -8,6 +8,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8.0)
# ---------------- User Config ---------------- # ---------------- User Config ----------------
# set optional libraries # set optional libraries
option(WITH_PYTHON3 "Set ON to use PYTHON3 otherwise PYTHON2" OFF)
option(WITH_CUDA "Set ON to use CUDA" ON) option(WITH_CUDA "Set ON to use CUDA" ON)
option(WITH_CUDNN "Set ON to use CUDNN" OFF) option(WITH_CUDNN "Set ON to use CUDNN" OFF)
option(WITH_BLAS "Set ON to use BLAS" OFF) option(WITH_BLAS "Set ON to use BLAS" OFF)
...@@ -19,10 +20,11 @@ option(WITH_CUDA_FP16 "Set ON to use FP16" ON) ...@@ -19,10 +20,11 @@ option(WITH_CUDA_FP16 "Set ON to use FP16" ON)
# set your 3rdparty # set your 3rdparty
set(3RDPARTY_DIR ${PROJECT_SOURCE_DIR}/../3rdparty) set(3RDPARTY_DIR ${PROJECT_SOURCE_DIR}/../3rdparty)
# set your py27 # set your python environment
set(PYTHON_DIR /usr/include/python2.7) # prefer set(PYTHON_DIR /usr/include/python2.7) # prefer
#set(ANACONDA_DIR /xxx/anaconda) # optional #set(PYTHON_DIR /usr/include/python3.x) # optional, set specific version
set(NUMPY_DIR /xxx/numpy) # require #set(ANACONDA_DIR /xxx/anaconda) # optional, set specific version below if using py3
set(NUMPY_DIR /xxx/numpy) # require£¬ root folder of numpy package
# set CUDA compiling architecture # set CUDA compiling architecture
set(CUDA_ARCH -gencode arch=compute_20,code=sm_20 set(CUDA_ARCH -gencode arch=compute_20,code=sm_20
...@@ -78,7 +80,10 @@ include_directories(${3RDPARTY_DIR}/include/mpi) ...@@ -78,7 +80,10 @@ include_directories(${3RDPARTY_DIR}/include/mpi)
include_directories(${CUDA_INCLUDE_DIRS}) include_directories(${CUDA_INCLUDE_DIRS})
include_directories(${PROJECT_SOURCE_DIR}/src) include_directories(${PROJECT_SOURCE_DIR}/src)
include_directories(${NUMPY_DIR}/core/include) include_directories(${NUMPY_DIR}/core/include)
include_directories(${NUMPY_DIR})
include_directories(${NUMPY_DIR}/numpy)
include_directories(${ANACONDA_DIR}/include/python2.7) include_directories(${ANACONDA_DIR}/include/python2.7)
include_directories(${ANACONDA_DIR}/include/python3.x)
include_directories(${PYTHON_DIR}) include_directories(${PYTHON_DIR})
include_directories(${ANACONDA_DIR}/include) include_directories(${ANACONDA_DIR}/include)
...@@ -91,50 +96,54 @@ link_directories(/usr/local/cuda/lib64) ...@@ -91,50 +96,54 @@ link_directories(/usr/local/cuda/lib64)
set(CMAKE_INSTALL_PREFIX ${PROJECT_SOURCE_DIR} CACHE STRING "set install prefix" FORCE) set(CMAKE_INSTALL_PREFIX ${PROJECT_SOURCE_DIR} CACHE STRING "set install prefix" FORCE)
# ---[ defines # ---[ defines
if (WITH_PYTHON3)
ADD_DEFINITIONS(-DWITH_PYTHON3)
message(STATUS "Use PYTHON3 [Optional]")
endif()
if (WITH_CUDA) if (WITH_CUDA)
ADD_DEFINITIONS(-DWITH_CUDA) ADD_DEFINITIONS(-DWITH_CUDA)
message(STATUS "Use CUDA [Optional]") message(STATUS "Use CUDA [Optional]")
endif() endif()
if (WITH_CUDNN) if (WITH_CUDNN)
ADD_DEFINITIONS(-DWITH_CUDNN) ADD_DEFINITIONS(-DWITH_CUDNN)
message(STATUS "Use CUDNN [Optional]") message(STATUS "Use CUDNN [Optional]")
endif() endif()
if (WITH_BLAS) if (WITH_BLAS)
ADD_DEFINITIONS(-DWITH_BLAS) ADD_DEFINITIONS(-DWITH_BLAS)
message(STATUS "Use BLAS [Optional]") message(STATUS "Use BLAS [Optional]")
else() else()
message(STATUS "Unuse BLAS [Optional]" message(STATUS "Unuse BLAS [Optional]"
"\n -- > GEMM/GEMV is disabled" "\n -- > GEMM/GEMV is disabled"
"\n -- > prefer not to run as CPU Mode") "\n -- > prefer not to run as CPU Mode")
endif() endif()
if (WITH_SSE) if (WITH_SSE)
ADD_DEFINITIONS(-DWITH_SSE) ADD_DEFINITIONS(-DWITH_SSE)
message(STATUS "Use SSE [Optional]") message(STATUS "Use SSE [Optional]")
if(UNIX) if(UNIX)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.1") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.1")
endif() endif()
endif() endif()
if (WITH_MPI) if (WITH_MPI)
ADD_DEFINITIONS(-DWITH_MPI) ADD_DEFINITIONS(-DWITH_MPI)
message(STATUS "Use MPI [Optional]") message(STATUS "Use MPI [Optional]")
endif() endif()
if (WITH_MPI_CUDA) if (WITH_MPI_CUDA)
ADD_DEFINITIONS(-DWITH_CUDA_AWARE) ADD_DEFINITIONS(-DWITH_CUDA_AWARE)
message(STATUS "Use MPI-CUDA [Optional]") message(STATUS "Use MPI-CUDA [Optional]")
endif() endif()
if (WITH_CUDA_FP16) if (WITH_CUDA_FP16)
ADD_DEFINITIONS(-DWITH_CUDA_FP16) ADD_DEFINITIONS(-DWITH_CUDA_FP16)
message(STATUS "Use CUDA FP16 [Optional]") message(STATUS "Use CUDA FP16 [Optional]")
endif() endif()
# ---[ Flags # ---[ Flags
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} ${CUDA_ARCH}") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} ${CUDA_ARCH}")
if(WIN32) if(WIN32)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP")
endif() endif()
if(UNIX) if(UNIX)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -O2 -m64 -fpermissive -std=c++11") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -O2 -m64 -fpermissive -std=c++11")
endif() endif()
# ---[ Warnings # ---[ Warnings
...@@ -143,4 +152,4 @@ endif() ...@@ -143,4 +152,4 @@ endif()
add_subdirectory(modules/python) add_subdirectory(modules/python)
# ---[ Utils # ---[ Utils
file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/../lib) file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/../lib)
\ No newline at end of file
Dragon/include/core/context_cuda.h
...@@ -28,7 +28,7 @@ namespace dragon { ...@@ -28,7 +28,7 @@ namespace dragon {
* it seems not necessary to create handles for different threads * it seems not necessary to create handles for different threads
*************************************************************************/ *************************************************************************/
class CUDAObject{ class CUDAObject {
public: public:
CUDAObject(): cur_gpu(0) { CUDAObject(): cur_gpu(0) {
for (int i = 0; i < MAX_GPUS; i++) { for (int i = 0; i < MAX_GPUS; i++) {
......
Dragon/include/core/workspace.h
...@@ -51,21 +51,21 @@ class Workspace{ ...@@ -51,21 +51,21 @@ class Workspace{
return tensor_map_[query].get(); return tensor_map_[query].get();
} }
inline Tensor* GetTensor(const string& name){ inline Tensor* GetTensor(const string& name) {
string query = GetTensorName(name); string query = GetTensorName(name);
CHECK(HasTensor(query)) CHECK(HasTensor(query))
<< "Tensor(" << name << ") does not exist."; << "Tensor(" << name << ") does not exist.";
return tensor_map_[query].get(); return tensor_map_[query].get();
} }
inline void LockTensor(const string& name){ inline void LockTensor(const string& name) {
string query = GetTensorName(name); string query = GetTensorName(name);
if (!lock_map_.count(query)) if (!lock_map_.count(query))
lock_map_[query] = unique_ptr<mutex>(new mutex); lock_map_[query] = unique_ptr<mutex>(new mutex);
lock_map_[query]->lock(); lock_map_[query]->lock();
} }
inline void UnlockTensor(const string& name){ inline void UnlockTensor(const string& name) {
string query = GetTensorName(name); string query = GetTensorName(name);
if (!lock_map_.count(query)) if (!lock_map_.count(query))
lock_map_[query] = unique_ptr<mutex>(new mutex); lock_map_[query] = unique_ptr<mutex>(new mutex);
......
Dragon/include/operators/common/python_op.h
...@@ -11,6 +11,10 @@ ...@@ -11,6 +11,10 @@
#include "core/operator.h" #include "core/operator.h"
#ifdef WITH_PYTHON3
#define PyBytes_FromStringAndSize PyUnicode_FromStringAndSize
#endif
namespace dragon { namespace dragon {
template <class Context> template <class Context>
......
Dragon/modules/python/dragon.cc
...@@ -403,7 +403,7 @@ PyMethodDef* GetAllMethods() { ...@@ -403,7 +403,7 @@ PyMethodDef* GetAllMethods() {
return g_python_methods; return g_python_methods;
} }
static void import_array_wrapper() { import_array(); } static void import_array_wrapper() { import_array1(); }
void common_init() { void common_init() {
import_array_wrapper(); import_array_wrapper();
...@@ -414,10 +414,24 @@ void common_init() { ...@@ -414,10 +414,24 @@ void common_init() {
initialized = true; initialized = true;
} }
#ifdef WITH_PYTHON3
static struct PyModuleDef libdragon = { PyModuleDef_HEAD_INIT,
"libdragon", "", -1,
GetAllMethods() };
PyMODINIT_FUNC PyInit_libdragon(void) {
PyObject* module = PyModule_Create(&libdragon);
if (module == nullptr) return nullptr;
common_init();
return module;
}
#else // WITH_PYTHON2
PyMODINIT_FUNC initlibdragon(void) { PyMODINIT_FUNC initlibdragon(void) {
PyObject* moudle = Py_InitModule("libdragon", GetAllMethods()); PyObject* moudle = Py_InitModule("libdragon", GetAllMethods());
if (moudle == nullptr) return; if (moudle == nullptr) return;
common_init(); common_init();
} }
#endif
} }
\ No newline at end of file
Dragon/modules/python/dragon.h
...@@ -19,6 +19,10 @@ ...@@ -19,6 +19,10 @@
#include "core/operator_gradient.h" #include "core/operator_gradient.h"
#include "core/workspace.h" #include "core/workspace.h"
#ifdef WITH_PYTHON3
#define PyString_AsString PyUnicode_AsUTF8
#endif
using namespace dragon; using namespace dragon;
inline std::string PyBytesToStdString(PyObject* pystring) { inline std::string PyBytesToStdString(PyObject* pystring) {
......
Dragon/modules/python/py_mpi.h
...@@ -13,6 +13,11 @@ ...@@ -13,6 +13,11 @@
#ifdef WITH_MPI #ifdef WITH_MPI
#include <mpi/mpi.h> #include <mpi/mpi.h>
#ifdef WITH_PYTHON3
#define PyInt_FromLong PyLong_FromLong
#define _PyInt_AsInt _PyLong_AsInt
#endif
using namespace dragon; using namespace dragon;
inline PyObject* MPIInitCC(PyObject* self, PyObject* args) { inline PyObject* MPIInitCC(PyObject* self, PyObject* args) {
......
Dragon/python/__init__.py
...@@ -16,5 +16,4 @@ except ImportError as e: ...@@ -16,5 +16,4 @@ except ImportError as e:
from dragon.core.scope import TensorScope as name_scope from dragon.core.scope import TensorScope as name_scope
from dragon.core.scope import PhaseScope as phase_scope from dragon.core.scope import PhaseScope as phase_scope
from dragon.core.scope import DeviceScope as device_scope from dragon.core.scope import DeviceScope as device_scope
\ No newline at end of file
Dragon/python/config.py
...@@ -4,8 +4,12 @@ ...@@ -4,8 +4,12 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
import numpy as np from dragon.__init__ import *
from __init__ import *
import logging
logger = logging.getLogger('dragon')
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler(sys.stdout))
option = {} option = {}
...@@ -58,6 +62,14 @@ def SetLoggingLevel(level): ...@@ -58,6 +62,14 @@ def SetLoggingLevel(level):
""" """
SetLogLevelCC(level) SetLogLevelCC(level)
global logger
logger.setLevel({
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'FATAL': logging.CRITICAL
}[level])
......
Dragon/python/core/gradient_maker.py
...@@ -4,18 +4,21 @@ ...@@ -4,18 +4,21 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
import dragon.config as config
import dragon.protos.dragon_pb2 as pb
from collections import defaultdict from collections import defaultdict
from dragon.core.utils import MakeOperatorDef
from dragon.__init__ import * from dragon.__init__ import *
import dragon.protos.dragon_pb2 as pb
import dragon.config as config from .scope import GetOperatorName
from scope import GetOperatorName
from dragon.utils import MakeOperatorDef
class GraphGradientMaker(object): class GraphGradientMaker(object):
@classmethod @classmethod
def CreateGradientForOp(cls, op_def, g_output): def CreateGradientForOp(cls, op_def, g_output):
""" parse ops from string """ """ parse ops from string """
g_ops, g_inputs, defaults = CreateGradientDefsCC(op_def.SerializeToString(), g_output) g_ops, g_inputs, defaults = CreateGradientDefsCC(op_def.SerializeToString(), g_output)
if sys.version_info >= (3, 0):
g_inputs = [g_input.decode('ascii') for g_input in g_inputs]
for idx, g_op in enumerate(g_ops): for idx, g_op in enumerate(g_ops):
new_def = pb.OperatorDef() new_def = pb.OperatorDef()
new_def.ParseFromString(g_op) new_def.ParseFromString(g_op)
......
Dragon/python/core/mpi.py
...@@ -4,8 +4,11 @@ ...@@ -4,8 +4,11 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from dragon import MPIInitCC, MPIRankCC, MPISizeCC, MPICreateGroupCC, MPIFinalizeCC
import numpy as np import numpy as np
from six.moves import range as xrange
from dragon import MPIInitCC, MPIRankCC, MPISizeCC, \
MPICreateGroupCC, MPIFinalizeCC
_is_init = False _is_init = False
_snapshot_ranks = [] _snapshot_ranks = []
......
Dragon/python/core/tensor.py
...@@ -4,13 +4,15 @@ ...@@ -4,13 +4,15 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
import numpy as np
from collections import OrderedDict
import dragon.config as config import dragon.config as config
import dragon.core.workspace as ws
import dragon.protos.dragon_pb2 as pb import dragon.protos.dragon_pb2 as pb
import workspace as ws import numpy as np
from scope import GetOperatorName, GetTensorName from collections import OrderedDict
from dragon.utils import MakeOperatorDef from dragon.core.utils import MakeOperatorDef
from dragon.core.scope import GetOperatorName, GetTensorName
from six.moves import range as xrange
class Tensor(object): class Tensor(object):
REGISTERED_FILLERS = {'Constant', 'Normal', 'TruncatedNormal', REGISTERED_FILLERS = {'Constant', 'Normal', 'TruncatedNormal',
...@@ -39,7 +41,7 @@ class Tensor(object): ...@@ -39,7 +41,7 @@ class Tensor(object):
@name.setter @name.setter
def name(self, value): def name(self, value):
from scope import TENSOR_SCOPE from .scope import TENSOR_SCOPE
if value is None: self._name = TENSOR_SCOPE + GetTensorName() if value is None: self._name = TENSOR_SCOPE + GetTensorName()
else: self._name = TENSOR_SCOPE + value else: self._name = TENSOR_SCOPE + value
...@@ -237,15 +239,15 @@ class Tensor(object): ...@@ -237,15 +239,15 @@ class Tensor(object):
# 1. collect inputs # 1. collect inputs
if not isinstance(inputs, list): inputs = [inputs] if not isinstance(inputs, list): inputs = [inputs]
for input in inputs: for input in inputs:
for op_idx, expr in input.expressions.iteritems(): for op_idx, expr in input.expressions.items():
if not expressions.has_key(op_idx): if not op_idx in expressions:
expressions[op_idx] = expr expressions[op_idx] = expr
if extra_inputs is not None: if extra_inputs is not None:
if not isinstance(extra_inputs, list): extra_inputs = [extra_inputs] if not isinstance(extra_inputs, list): extra_inputs = [extra_inputs]
for input in extra_inputs: for input in extra_inputs:
for op_idx, expr in input.expressions.iteritems(): for op_idx, expr in input.expressions.items():
if not expressions.has_key(op_idx): if not op_idx in expressions:
expressions[op_idx] = expr expressions[op_idx] = expr
# 2. generate outputs # 2. generate outputs
...@@ -286,7 +288,7 @@ class Tensor(object): ...@@ -286,7 +288,7 @@ class Tensor(object):
output.extra_targets.add(input.name) output.extra_targets.add(input.name)
# 5. utils # 5. utils
if kwargs.has_key('static_shape'): if 'static_shape' in kwargs:
outputs[0].tf_shape = kwargs['static_shape'] outputs[0].tf_shape = kwargs['static_shape']
if nout > 1: if nout > 1:
...@@ -302,26 +304,26 @@ class Tensor(object): ...@@ -302,26 +304,26 @@ class Tensor(object):
filler.type = type.lower() filler.type = type.lower()
if filler.type == 'constant': if filler.type == 'constant':
filler.value = kwargs['value'] if kwargs.has_key('value') else 0 filler.value = kwargs['value'] if 'value' in kwargs else 0
elif filler.type == 'normal' or filler.type == 'gaussian': elif filler.type == 'normal' or filler.type == 'gaussian':
filler.mean = kwargs['mean'] if kwargs.has_key('mean') else 0 filler.mean = kwargs['mean'] if 'mean' in kwargs else 0
filler.std = kwargs['std'] if kwargs.has_key('std') else 1 filler.std = kwargs['std'] if 'std' in kwargs else 1
filler.type = 'normal' filler.type = 'normal'
elif filler.type == 'uniform': elif filler.type == 'uniform':
filler.low = kwargs['low'] if kwargs.has_key('low') else 0 filler.low = kwargs['low'] if 'low' in kwargs else 0
filler.high = kwargs['high'] if kwargs.has_key('high') else 1 filler.high = kwargs['high'] if 'high' in kwargs else 1
filler.type = 'uniform' filler.type = 'uniform'
elif filler.type == 'truncated_normal' or filler.type == 'truncatednormal': elif filler.type == 'truncated_normal' or filler.type == 'truncatednormal':
filler.mean = kwargs['mean'] if kwargs.has_key('mean') else 0 filler.mean = kwargs['mean'] if 'mean' in kwargs else 0
filler.std = kwargs['std'] if kwargs.has_key('std') else 1 filler.std = kwargs['std'] if 'std' in kwargs else 1
filler.low = filler.mean - 2.0 * filler.std filler.low = filler.mean - 2.0 * filler.std
filler.high = filler.mean + 2.0 * filler.std filler.high = filler.mean + 2.0 * filler.std
filler.type = 'truncated_normal' filler.type = 'truncated_normal'
elif filler.type == 'parameterized_truncated_normal': elif filler.type == 'parameterized_truncated_normal':
filler.mean = kwargs['mean'] if kwargs.has_key('mean') else 0 filler.mean = kwargs['mean'] if 'mean' in kwargs else 0
filler.std = kwargs['std'] if kwargs.has_key('std') else 1 filler.std = kwargs['std'] if 'std' in kwargs else 1
filler.low = kwargs['low'] if kwargs.has_key('low') else -2.0 filler.low = kwargs['low'] if 'low' in kwargs else -2.0
filler.high = kwargs['high'] if kwargs.has_key('high') else 2.0 filler.high = kwargs['high'] if 'high' in kwargs else 2.0
ws.CreateFiller(filler) ws.CreateFiller(filler)
return self return self
...@@ -331,7 +333,7 @@ class Tensor(object): ...@@ -331,7 +333,7 @@ class Tensor(object):
buffer0 = '-------------------Expressions-------------------\n' buffer0 = '-------------------Expressions-------------------\n'
buffer1 = ''; buffer2 = 'Inputs: [' buffer1 = ''; buffer2 = 'Inputs: ['
for k,v in self.expressions.iteritems(): for k,v in self.expressions.items():
buffer1 = buffer1 + '>>> ' + str(k).zfill(3) + '. (' buffer1 = buffer1 + '>>> ' + str(k).zfill(3) + '. ('
for input in v.input: for input in v.input:
if input not in outputs: if input not in outputs:
......
Dragon/python/core/utils.py 0 → 100644
# --------------------------------------------------------
# Dragon
# Copyright(c) 2017 SeetaTech
# Written by Ting Pan
# --------------------------------------------------------
import sys
from google.protobuf.message import Message
from dragon.protos import dragon_pb2 as pb
import numpy as np
if sys.version_info >= (3,0):
def MakeArgument(key, value):
argument = pb.Argument()
argument.name = key
if type(value) is float: argument.f = value
elif type(value) is int: argument.i = value
elif type(value) is np.int64: argument.i64 = int(value)
elif type(value) is str: argument.s = value
elif type(value) is bool: argument.b = value
elif isinstance(value, Message): argument.s = value.SerializeToString()
elif all(type(v) is float for v in value): argument.floats.extend(value)
elif all(type(v) is int for v in value): argument.ints.extend(value)
elif all(type(v) is str for v in value): argument.strings.extend(value)
elif all(isinstance(v, Message) for v in value):
argument.strings.extend([v.SerializeToString() for v in value])
else:
raise ValueError('unknown argument type: key={} value={} value type={}' \
.format(key, value, type(value)))
return argument
else:
def MakeArgument(key, value):
argument = pb.Argument()
argument.name = key
if type(value) is float: argument.f = value
elif type(value) is int: argument.i = value
elif type(value) is np.int64: argument.i64 = int(value)
elif type(value) is str: argument.s = value
elif type(value) is unicode: argument.s = value
elif type(value) is bool: argument.b = value
elif isinstance(value, Message): argument.s = value.SerializeToString()
elif all(type(v) is float for v in value): argument.floats.extend(value)
elif all(type(v) is int for v in value): argument.ints.extend(value)
elif all(type(v) is str for v in value): argument.strings.extend(value)
elif all(type(v) is unicode or type(v) is str for v in value):
argument.strings.extend(value)
elif all(isinstance(v, Message) for v in value):
argument.strings.extend([v.SerializeToString() for v in value])
else:
raise ValueError('unknown argument type: key={} value={} value type={}' \
.format(key, value, type(value)))
return argument
def MakeOperatorDef(op_type, inputs, outputs, name='',
device_option=None, arg=None, engine=None, **kwargs):
operator = pb.OperatorDef()
operator.type = op_type
operator.name = name
operator.input.extend([str(tensor) for tensor in inputs])
operator.output.extend([str(tensor) for tensor in outputs])
if device_option is not None:
operator.device_option.CopyFrom(device_option)
if engine is not None:
operator.engine = engine
if 'random_seed' in kwargs:
operator.device_option.random_seed = kwargs['random_seed']
del kwargs['random_seed']
if arg is not None:
operator.arg.extend(arg)
for k,v in kwargs.items():
if v is None: continue
operator.arg.add().CopyFrom(MakeArgument(k,v))
return operator
def MakeDeviceOption(device_type, gpu_id, rng_seed = None):
""" return a DeviceOption """
option = pb.DeviceOption()
option.device_type = device_type
option.gpu_id = gpu_id
if rng_seed is not None: option.random_seed = rng_seed
return option
# fix the python stdout
class Unbuffered(object):
def __init__(self, stream):
self.stream = stream
def write(self, data):
self.stream.write(data)
self.stream.flush()
def __getattr__(self, attr):
return getattr(self.stream, attr)
# clear the stdout buffer for mpi(c++ & python)
import sys
sys.stdout = Unbuffered(sys.stdout)
\ No newline at end of file
Dragon/python/core/workspace.py
...@@ -4,15 +4,19 @@ ...@@ -4,15 +4,19 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
import cPickle try:
import os import cPickle
except:
import pickle as cPickle
import dragon.core.utils as utils
import dragon.core.mpi as mpi
import dragon.protos.dragon_pb2 as pb
import numpy as np import numpy as np
from google.protobuf.message import Message import os
from dragon import * from dragon import *
import dragon.protos.dragon_pb2 as pb from dragon.config import logger
import dragon.core.mpi as mpi from google.protobuf.message import Message
import dragon.utils as utils from six.moves import range as xrange
CURRENT_GRAPH_IDX = 0 CURRENT_GRAPH_IDX = 0
...@@ -32,7 +36,7 @@ def CreateGraph(graph_def): ...@@ -32,7 +36,7 @@ def CreateGraph(graph_def):
def WriteOptimizedGraph(graph_def): def WriteOptimizedGraph(graph_def):
with open(graph_def.name + '.txt', 'w') as f: with open(graph_def.name + '.txt', 'w') as f:
f.write(str(graph_def)) f.write(str(graph_def))
print 'write serialized graph to: {}'.format(graph_def.name + '.txt') logger.info('write serialized graph to: {}'.format(graph_def.name + '.txt'))
def HasTensor(tensor): def HasTensor(tensor):
...@@ -113,19 +117,19 @@ def RunGraph(graph_name, inputs=(), outputs=[], stage=None, return_outputs=True) ...@@ -113,19 +117,19 @@ def RunGraph(graph_name, inputs=(), outputs=[], stage=None, return_outputs=True)
def PrintRawGraphDef(graph_def): def PrintRawGraphDef(graph_def):
print graph_def logger.info(graph_def)
def PrintOptimizedGraph(graph_def): def PrintOptimizedGraph(graph_def):
graph_name = graph_def.name graph_name = graph_def.name
graph_tensor = 'GraphDef_' + graph_name graph_tensor = 'GraphDef_' + graph_name
if not HasTensorCC(graph_tensor): if not HasTensorCC(graph_tensor):
print 'graph: {} does not exist, ignore printing....'.format(graph_name) logger.info('graph: {} does not exist, ignore printing....'.format(graph_name))
return return
graph_def = pb.GraphDef() graph_def = pb.GraphDef()
graph_def.ParseFromString(FetchTensor(graph_tensor)) graph_def.ParseFromString(FetchTensor(graph_tensor))
print graph_def logger.info(graph_def)
def Snapshot(tensors, filename, prefix='', suffix='.bin', format=0): def Snapshot(tensors, filename, prefix='', suffix='.bin', format=0):
...@@ -144,8 +148,8 @@ def Snapshot(tensors, filename, prefix='', suffix='.bin', format=0): ...@@ -144,8 +148,8 @@ def Snapshot(tensors, filename, prefix='', suffix='.bin', format=0):
content[tensor.name] = FetchTensor(tensor) content[tensor.name] = FetchTensor(tensor)
with open(filepath, 'wb') as f: with open(filepath, 'wb') as f:
cPickle.dump(content, f, cPickle.HIGHEST_PROTOCOL) cPickle.dump(content, f, cPickle.HIGHEST_PROTOCOL)
print 'Snapshot Model@: ' + filepath logger.info('Snapshot Model@: ' + filepath)
print 'Model Format: cPickle' logger.info('Model Format: cPickle')
elif format is 1: elif format is 1:
# caffe-store # caffe-store
...@@ -162,13 +166,13 @@ def Restore(filename, format=0): ...@@ -162,13 +166,13 @@ def Restore(filename, format=0):
assert os.path.exists(filename), 'model of path({}) does not exist.'.format(filename) assert os.path.exists(filename), 'model of path({}) does not exist.'.format(filename)
if format is 0: if format is 0:
content = cPickle.load(open(filename, 'rb')) content = cPickle.load(open(filename, 'rb'))
print 'Restore From Model@: ' + filename logger.info('Restore From Model@: ' + filename)
print 'Model Format: cPickle' logger.info('Model Format: cPickle')
for key, ndarray in content.iteritems(): for key, ndarray in content.items():
if not HasTensor(key): if not HasTensor(key):
print '[Warning]: Tensor({}) of model does not exist in any Graphs, skip.'.format(key) logger.info('[Warning]: Tensor({}) of model does not exist in any Graphs, skip.'.format(key))
else: else:
print '[Info]: Tensor({}) restored.'.format(key) logger.info('[Info]: Tensor({}) restored.'.format(key))
FeedTensor(key, ndarray) FeedTensor(key, ndarray)
elif format is 1: elif format is 1:
......
Dragon/python/examples/custom_ops/data_process.py
...@@ -9,6 +9,7 @@ import dragon.core.workspace as ws ...@@ -9,6 +9,7 @@ import dragon.core.workspace as ws
import dragon.ops as ops import dragon.ops as ops
import dragon.vm.theano as theano import dragon.vm.theano as theano
from multiprocessing import Process, Queue from multiprocessing import Process, Queue
from dragon.config import logger
""" How to custom a RunOp in Dragon """ """ How to custom a RunOp in Dragon """
...@@ -31,7 +32,7 @@ class Fetcher(Process): ...@@ -31,7 +32,7 @@ class Fetcher(Process):
self.daemon = True self.daemon = True
def cleanup(): def cleanup():
print 'Terminating Fetcher......' logger.info('Terminating Fetcher......')
self.terminate() self.terminate()
self.join() self.join()
...@@ -103,4 +104,4 @@ if __name__ == '__main__': ...@@ -103,4 +104,4 @@ if __name__ == '__main__':
foo() foo()
# fetch # fetch
print 'y \n-------------- \n', y.get_value(), '\n' logger.info('y \n-------------- \n', y.get_value(), '\n')
\ No newline at end of file \ No newline at end of file
Dragon/python/examples/custom_ops/vec_mult.py
...@@ -10,6 +10,7 @@ import dragon.ops as ops ...@@ -10,6 +10,7 @@ import dragon.ops as ops
from dragon.core.tensor import Tensor from dragon.core.tensor import Tensor
import dragon.vm.theano.tensor as T import dragon.vm.theano.tensor as T
import dragon.vm.theano as theano import dragon.vm.theano as theano
from dragon.config import logger
""" How to custom a TemplateOp in Dragon """ """ How to custom a TemplateOp in Dragon """
...@@ -97,7 +98,7 @@ if __name__ == '__main__': ...@@ -97,7 +98,7 @@ if __name__ == '__main__':
foo() foo()
# fetch # fetch
print 'y \n-------------- \n', y.get_value(), '\n' logger.info('y \n-------------- \n', y.get_value(), '\n')
print 'dx1 \n-------------- \n', dx1.get_value(), '\n' logger.info('dx1 \n-------------- \n', dx1.get_value(), '\n')
print 'dx2 \n-------------- \n', dx2.get_value(), '\n' logger.info('dx2 \n-------------- \n', dx2.get_value(), '\n')
Dragon/python/operators/common.py
...@@ -4,9 +4,12 @@ ...@@ -4,9 +4,12 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
import numpy as np
from six.moves import range as xrange
from dragon.core.tensor import Tensor, GetTensorName from dragon.core.tensor import Tensor, GetTensorName
import dragon.core.workspace as ws import dragon.core.workspace as ws
import numpy as np
def At(inputs, indices=[], axis=0, acc_gradient=False, **kwargs): def At(inputs, indices=[], axis=0, acc_gradient=False, **kwargs):
......
Dragon/python/operators/data.py
...@@ -8,29 +8,6 @@ import numpy as np ...@@ -8,29 +8,6 @@ import numpy as np
from dragon.core.tensor import Tensor from dragon.core.tensor import Tensor
from dragon.operators.utils import Run from dragon.operators.utils import Run
def Imagenet(**kwargs):
"""
:param kwargs: a dict of imagenet data param
:param --> mean_value: a list of mean values for channles [B-G-R]
:param --> source: a str of the images root directory
:param --> imageset: a str of text file contains image name / label
:param --> prefetch: a int of the prefetching size
:param --> batch_size: a int of the batch size
:param --> force_gray a bool of whether to use only 1 channel
:param --> shuffle a bool of whether to use shuffle
:param --> scale a float of the coeff to scale
:return: 2 Tensors of data and label
"""
args = locals(); kwargs = args['kwargs']
del args['kwargs']; kwargs = dict(args, **kwargs)
kwargs['module'] = 'dragon.vm.caffe.io.data_layer'
kwargs['op'] = 'DataLayer'
return Run([], param_str=str(kwargs), nout=2, **kwargs)
def LMDBData(**kwargs): def LMDBData(**kwargs):
""" """
:param kwargs: a dict of imagenet data param :param kwargs: a dict of imagenet data param
......
Dragon/python/operators/initializer.py
...@@ -27,7 +27,7 @@ def Fill(shape, value=1.0, **kwargs): ...@@ -27,7 +27,7 @@ def Fill(shape, value=1.0, **kwargs):
del kwargs['shape'] del kwargs['shape']
output = Tensor.CreateOperator([], nout=1, op_type='Fill', **kwargs) output = Tensor.CreateOperator([], nout=1, op_type='Fill', **kwargs)
output.shape = kwargs['static_shape'] if kwargs.has_key('static_shape') else None output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None
return output return output
...@@ -52,7 +52,7 @@ def RandomalUniform(shape, low=-1.0, high=1.0, **kwargs): ...@@ -52,7 +52,7 @@ def RandomalUniform(shape, low=-1.0, high=1.0, **kwargs):
del kwargs['shape'] del kwargs['shape']
output = Tensor.CreateOperator([], nout=1, op_type='RandomUniform', **kwargs) output = Tensor.CreateOperator([], nout=1, op_type='RandomUniform', **kwargs)
output.shape = kwargs['static_shape'] if kwargs.has_key('static_shape') else None output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None
return output return output
...@@ -77,7 +77,7 @@ def RandomalNormal(shape, mean=0.0, std=1.0, **kwargs): ...@@ -77,7 +77,7 @@ def RandomalNormal(shape, mean=0.0, std=1.0, **kwargs):
del kwargs['shape'] del kwargs['shape']
output = Tensor.CreateOperator([], nout=1, op_type='RandomNormal', **kwargs) output = Tensor.CreateOperator([], nout=1, op_type='RandomNormal', **kwargs)
output.shape = kwargs['static_shape'] if kwargs.has_key('static_shape') else None output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None
return output return output
...@@ -102,7 +102,7 @@ def TruncatedNormal(shape, mean=0.0, std=1.0, **kwargs): ...@@ -102,7 +102,7 @@ def TruncatedNormal(shape, mean=0.0, std=1.0, **kwargs):
del kwargs['shape'] del kwargs['shape']
output = Tensor.CreateOperator([], nout=1, op_type='TruncatedNormal', **kwargs) output = Tensor.CreateOperator([], nout=1, op_type='TruncatedNormal', **kwargs)
output.shape = kwargs['static_shape'] if kwargs.has_key('static_shape') else None output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None
return output return output
...@@ -125,7 +125,7 @@ def GlorotUniform(shape, scale=3.0, mode='fan_in', **kwargs): ...@@ -125,7 +125,7 @@ def GlorotUniform(shape, scale=3.0, mode='fan_in', **kwargs):
del kwargs['shape'] del kwargs['shape']
output = Tensor.CreateOperator([], nout=1, op_type='GlorotUniform', **kwargs) output = Tensor.CreateOperator([], nout=1, op_type='GlorotUniform', **kwargs)
output.shape = kwargs['static_shape'] if kwargs.has_key('static_shape') else None output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None
return output return output
...@@ -149,7 +149,7 @@ def GlorotNormal(shape, scale=2.0, mode='fan_in', **kwargs): ...@@ -149,7 +149,7 @@ def GlorotNormal(shape, scale=2.0, mode='fan_in', **kwargs):
del kwargs['shape'] del kwargs['shape']
output = Tensor.CreateOperator([], nout=1, op_type='GlorotNormal', **kwargs) output = Tensor.CreateOperator([], nout=1, op_type='GlorotNormal', **kwargs)
output.shape = kwargs['static_shape'] if kwargs.has_key('static_shape') else None output.shape = kwargs['static_shape'] if 'static_shape' in kwargs else None
return output return output
......
Dragon/python/operators/mpi.py
...@@ -4,9 +4,12 @@ ...@@ -4,9 +4,12 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from six.moves import range as xrange
from dragon.core.tensor import Tensor from dragon.core.tensor import Tensor
import dragon.core.mpi as mpi import dragon.core.mpi as mpi
def MPIBroadcast(inputs, root, mpi_rank=None, **kwargs): def MPIBroadcast(inputs, root, mpi_rank=None, **kwargs):
""" """
:param inputs: a Tensor which to broadcast :param inputs: a Tensor which to broadcast
...@@ -52,7 +55,7 @@ def MPIGather(inputs, root, mpi_rank=None, **kwargs): ...@@ -52,7 +55,7 @@ def MPIGather(inputs, root, mpi_rank=None, **kwargs):
if not isinstance(kwargs['mpi_rank'], list): if not isinstance(kwargs['mpi_rank'], list):
kwargs['mpi_rank'] = [kwargs['mpi_rank']] kwargs['mpi_rank'] = [kwargs['mpi_rank']]
if kwargs.has_key('nout'): if 'nout' in kwargs:
if kwargs['nout'] != len(kwargs['mpi_rank']): if kwargs['nout'] != len(kwargs['mpi_rank']):
raise RuntimeError('specfied nout is {}, but provide {} mpi nodes' raise RuntimeError('specfied nout is {}, but provide {} mpi nodes'
.format(kwargs['nout'], len(kwargs['mpi_rank']))) .format(kwargs['nout'], len(kwargs['mpi_rank'])))
......
Dragon/python/operators/vision.py
...@@ -5,8 +5,11 @@ ...@@ -5,8 +5,11 @@
# -------------------------------------------------------- # --------------------------------------------------------
import math import math
from six.moves import range as xrange
from dragon.core.tensor import Tensor from dragon.core.tensor import Tensor
def Conv2D(inputs, num_output, kernel_size, def Conv2D(inputs, num_output, kernel_size,
stride=1, pad=0, dilation=1, group=1, **kwargs): stride=1, pad=0, dilation=1, group=1, **kwargs):
""" """
......
Dragon/python/ops.py
...@@ -4,21 +4,20 @@ ...@@ -4,21 +4,20 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
import operators.initializer as init from .operators import initializer as init
import operators.vision as vision from .operators import vision as vision
import operators.loss as loss from .operators import loss as loss
import operators.data as data from .operators import data as data
import operators.activation as act from .operators import activation as act
import operators.arithmetic as math from .operators import arithmetic as math
import operators.utils as utils from .operators import utils as utils
import operators.cast as cast from .operators import cast as cast
import operators.mpi as mpi from .operators import mpi as mpi
import operators.common as common from .operators import common as common
import operators.norm as norm from .operators import norm as norm
import operators.recurrent as recurrent from .operators import recurrent as recurrent
# data # data
Imagenet = data.Imagenet
LMDBData = data.LMDBData LMDBData = data.LMDBData
MemoryData = data.MemoryData MemoryData = data.MemoryData
......
Dragon/python/protos/dragon.proto
syntax = "proto2";
message TensorProto { message TensorProto {
repeated int32 dims = 1; repeated int32 dims = 1;
enum DataType { enum DataType {
......
Dragon/python/tools/board/app.py
...@@ -4,6 +4,7 @@ from multiprocessing import Process ...@@ -4,6 +4,7 @@ from multiprocessing import Process
try: try:
from flask import Flask, render_template, make_response, jsonify, request from flask import Flask, render_template, make_response, jsonify, request
except ImportError as e: pass except ImportError as e: pass
from six.moves import range as xrange
class DragonBoard(Process): class DragonBoard(Process):
def __init__(self, log_dir='', port=5000, max_display=1000): def __init__(self, log_dir='', port=5000, max_display=1000):
...@@ -20,7 +21,8 @@ class DragonBoard(Process): ...@@ -20,7 +21,8 @@ class DragonBoard(Process):
'port': port, 'port': port,
'max_display': max_display} 'max_display': max_display}
def cleanup(): def cleanup():
print 'Terminating DragonBoard......' from dragon.config import logger
logger.info('Terminating DragonBoard......')
self.terminate() self.terminate()
self.join() self.join()
import atexit import atexit
......
Dragon/python/tools/db.py
...@@ -6,6 +6,12 @@ ...@@ -6,6 +6,12 @@
import lmdb import lmdb
import os import os
import sys
def wrapper_str(raw_str):
if sys.version_info >= (3, 0):
return raw_str.encode()
return raw_str
class LMDB(object): class LMDB(object):
def __init__(self, max_commit=10000): def __init__(self, max_commit=10000):
...@@ -14,6 +20,7 @@ class LMDB(object): ...@@ -14,6 +20,7 @@ class LMDB(object):
self._total_size = 0 self._total_size = 0
self._buffer = [] self._buffer = []
def open(self, database_path, mode='r'): def open(self, database_path, mode='r'):
if mode == 'r': if mode == 'r':
assert os.path.exists(database_path), 'database path is not exist' assert os.path.exists(database_path), 'database path is not exist'
...@@ -25,13 +32,14 @@ class LMDB(object): ...@@ -25,13 +32,14 @@ class LMDB(object):
self.txn = self.env.begin(write=(mode == 'w')) self.txn = self.env.begin(write=(mode == 'w'))
self.cursor = self.txn.cursor() self.cursor = self.txn.cursor()
def _try_put(self): def _try_put(self):
for pair in self._buffer: for pair in self._buffer:
key, value = pair key, value = pair
try: self.txn.put(key, value) try: self.txn.put(key, value)
except lmdb.MapFullError as e: except lmdb.MapFullError as e:
new_size = self.env.info()['map_size'] * 2 new_size = self.env.info()['map_size'] * 2
print 'doubling LMDB map size to %d MB' % (new_size >> 20) print('doubling LMDB map size to %d MB' % (new_size >> 20))
self.txn.abort() self.txn.abort()
self.env.set_mapsize(new_size) self.env.set_mapsize(new_size)
self.txn = self.env.begin(write=True) self.txn = self.env.begin(write=True)
...@@ -39,22 +47,27 @@ class LMDB(object): ...@@ -39,22 +47,27 @@ class LMDB(object):
self._cur_put = 0 self._cur_put = 0
self._buffer = [] self._buffer = []
def put(self, key, value): def put(self, key, value):
self._buffer.append((key, value)) self._buffer.append((wrapper_str(key), wrapper_str(value)))
self._cur_put += 1 self._cur_put += 1
if (self._cur_put >= self._max_commit): self._try_put() if (self._cur_put >= self._max_commit): self._try_put()
def commit(self): def commit(self):
self._try_put() self._try_put()
self.txn.commit() self.txn.commit()
self.txn = self.env.begin(write=True) self.txn = self.env.begin(write=True)
def set(self, key): def set(self, key):
self.cursor.set_key(key) self.cursor.set_key(wrapper_str(key))
def get(self, key): def get(self, key):
cursor = self.txn.cursor() cursor = self.txn.cursor()
return cursor.get(key) return cursor.get(wrapper_str(key))
def next(self): def next(self):
if not self.cursor.next(): if not self.cursor.next():
...@@ -62,11 +75,14 @@ class LMDB(object): ...@@ -62,11 +75,14 @@ class LMDB(object):
if self.key() == 'size' or self.key() == 'zfill': if self.key() == 'size' or self.key() == 'zfill':
self.next() self.next()
def key(self): def key(self):
return self.cursor.key() return self.cursor.key()
def value(self): def value(self):
return self.cursor.value() return self.cursor.value()
def close(self): def close(self):
self.env.close() self.env.close()
\ No newline at end of file
Dragon/python/updaters.py
...@@ -6,8 +6,9 @@ ...@@ -6,8 +6,9 @@
import numpy as np import numpy as np
import pprint import pprint
import core.workspace as ws from dragon.config import logger
from core.tensor import Tensor import dragon.core.workspace as ws
from dragon.core.tensor import Tensor
class Updater(object): class Updater(object):
def __init__(self, def __init__(self,
...@@ -47,10 +48,10 @@ class Updater(object): ...@@ -47,10 +48,10 @@ class Updater(object):
ws.FeedTensor(self._prefix + 'base_lr', np.array([lr], dtype=np.float32)) ws.FeedTensor(self._prefix + 'base_lr', np.array([lr], dtype=np.float32))
def echo(self): def echo(self):
print '---------------------------------------------------------' logger.info('---------------------------------------------------------')
print 'Optimizer: {}, Using config:'.format(self._type.split('Update')[0]) logger.info('Optimizer: {}, Using config:'.format(self._type.split('Update')[0]))
pprint.pprint(self._hyper_params) pprint.pprint(self._hyper_params)
print '---------------------------------------------------------' logger.info('---------------------------------------------------------')
class SGDUpdater(Updater): class SGDUpdater(Updater):
......
Dragon/python/vm/caffe/io/__init__.py
...@@ -6,4 +6,4 @@ ...@@ -6,4 +6,4 @@
def GetProperty(kwargs, name, default): def GetProperty(kwargs, name, default):
return kwargs[name] \ return kwargs[name] \
if kwargs.has_key(name) else default if name in kwargs else default
\ No newline at end of file \ No newline at end of file
Dragon/python/vm/caffe/io/blob_fetcher.py
...@@ -6,8 +6,11 @@ ...@@ -6,8 +6,11 @@
import numpy as np import numpy as np
from multiprocessing import Process from multiprocessing import Process
from six.moves import range as xrange
from __init__ import GetProperty from dragon.config import logger
from .__init__ import GetProperty
class BlobFetcher(Process): class BlobFetcher(Process):
def __init__(self, **kwargs): def __init__(self, **kwargs):
...@@ -20,7 +23,7 @@ class BlobFetcher(Process): ...@@ -20,7 +23,7 @@ class BlobFetcher(Process):
self.daemon = True self.daemon = True
def cleanup(): def cleanup():
print 'Terminating BlobFetcher......' logger.info('Terminating BlobFetcher......')
self.terminate() self.terminate()
self.join() self.join()
import atexit import atexit
......
Dragon/python/vm/caffe/io/data_layer.py
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
import dragon.vm.caffe as caffe import dragon.vm.caffe as caffe
import dragon.core.workspace as ws import dragon.core.workspace as ws
from minibatch import DataBatch from .minibatch import DataBatch
class DataLayer(caffe.Layer): class DataLayer(caffe.Layer):
def setup(self, bottom, top): def setup(self, bottom, top):
......
Dragon/python/vm/caffe/io/data_reader.py
...@@ -9,9 +9,11 @@ import numpy.random as npr ...@@ -9,9 +9,11 @@ import numpy.random as npr
from multiprocessing import Process from multiprocessing import Process
import dragon.config as config import dragon.config as config
from dragon.config import logger
from dragon.tools.db import LMDB from dragon.tools.db import LMDB
from __init__ import GetProperty from .__init__ import GetProperty
class DataReader(Process): class DataReader(Process):
def __init__(self, **kwargs): def __init__(self, **kwargs):
...@@ -33,7 +35,7 @@ class DataReader(Process): ...@@ -33,7 +35,7 @@ class DataReader(Process):
self.daemon = True self.daemon = True
def cleanup(): def cleanup():
print 'Terminating DataReader......' logger.info('Terminating DataReader......')
self.terminate() self.terminate()
self.join() self.join()
import atexit import atexit
...@@ -46,8 +48,8 @@ class DataReader(Process): ...@@ -46,8 +48,8 @@ class DataReader(Process):
if self._use_shuffle: if self._use_shuffle:
self._cur_chunk_idx = 0 self._cur_chunk_idx = 0
self._perm = npr.permutation(self._num_shuffle_parts) self._perm = npr.permutation(self._num_shuffle_parts)
self._start_idx = self._part_idx * self._num_shuffle_parts + self._perm[self._cur_chunk_idx] self._start_idx = int(self._part_idx * self._num_shuffle_parts + self._perm[self._cur_chunk_idx])
self._start_idx = self._start_idx * self._chunk_size self._start_idx = int(self._start_idx * self._chunk_size)
if self._start_idx >= self._db_size: self.next_chunk() if self._start_idx >= self._db_size: self.next_chunk()
self._end_idx = self._start_idx + self._chunk_size self._end_idx = self._start_idx + self._chunk_size
self._end_idx = min(self._db_size, self._end_idx) self._end_idx = min(self._db_size, self._end_idx)
...@@ -91,7 +93,7 @@ class DataReader(Process): ...@@ -91,7 +93,7 @@ class DataReader(Process):
self._db.open(self._source) self._db.open(self._source)
self._db_size = int(self._db.get('size')) self._db_size = int(self._db.get('size'))
self._db_zfill = int(self._db.get('zfill')) self._db_zfill = int(self._db.get('zfill'))
self._epoch_size = self._db_size / self._num_parts + 1 self._epoch_size = int(self._db_size / self._num_parts + 1)
# search a optimal chunk size by chunks # search a optimal chunk size by chunks
if self._chunk_size == -1: if self._chunk_size == -1:
max_chunk_size = self._db._total_size / ((self._num_chunks * (1 << 20))) max_chunk_size = self._db._total_size / ((self._num_chunks * (1 << 20)))
...@@ -100,7 +102,7 @@ class DataReader(Process): ...@@ -100,7 +102,7 @@ class DataReader(Process):
self._chunk_size = min_chunk_size self._chunk_size = min_chunk_size
self._num_shuffle_parts = int(math.ceil(self._db._total_size * 1.1 / self._num_shuffle_parts = int(math.ceil(self._db._total_size * 1.1 /
(self._num_parts * self._chunk_size << 20))) (self._num_parts * self._chunk_size << 20)))
self._chunk_size = self._db_size / self._num_shuffle_parts / self._num_parts + 1 self._chunk_size = int(self._db_size / self._num_shuffle_parts / self._num_parts + 1)
# init env # init env
self.reset() self.reset()
......
Dragon/python/vm/caffe/io/data_transformer.py
...@@ -9,9 +9,10 @@ import numpy.random as npr ...@@ -9,9 +9,10 @@ import numpy.random as npr
from multiprocessing import Process from multiprocessing import Process
import dragon.config as config import dragon.config as config
from dragon.config import logger
import dragon.vm.caffe.proto.caffe_pb2 as pb import dragon.vm.caffe.proto.caffe_pb2 as pb
from __init__ import GetProperty from .__init__ import GetProperty
try: try:
import cv2 import cv2
...@@ -47,7 +48,7 @@ class DataTransformer(Process): ...@@ -47,7 +48,7 @@ class DataTransformer(Process):
self.daemon = True self.daemon = True
def cleanup(): def cleanup():
print 'Terminating DataTransformer......' logger.info('Terminating DataTransformer......')
self.terminate() self.terminate()
self.join() self.join()
import atexit import atexit
......
Dragon/python/vm/caffe/io/minibatch.py
...@@ -7,13 +7,16 @@ ...@@ -7,13 +7,16 @@
import time import time
import pprint import pprint
from multiprocessing import Queue from multiprocessing import Queue
from six.moves import range as xrange
import dragon.core.mpi as mpi import dragon.core.mpi as mpi
from dragon.config import logger
from data_reader import DataReader from .data_reader import DataReader
from data_transformer import DataTransformer from .data_transformer import DataTransformer
from blob_fetcher import BlobFetcher from .blob_fetcher import BlobFetcher
from __init__ import GetProperty
from .__init__ import GetProperty
class DataBatch(object): class DataBatch(object):
def __init__(self, **kwargs): def __init__(self, **kwargs):
...@@ -106,11 +109,11 @@ class DataBatch(object): ...@@ -106,11 +109,11 @@ class DataBatch(object):
return self.Q_level_3.get() return self.Q_level_3.get()
def echo(self): def echo(self):
print '---------------------------------------------------------' logger.info('---------------------------------------------------------')
print 'BatchReader, Using config:' logger.info('BatchReader, Using config:')
params = {'prefetching': self._prefetch, params = {'prefetching': self._prefetch,
'num_readers': self._num_readers, 'num_readers': self._num_readers,
'num_transformers': self._num_transformers, 'num_transformers': self._num_transformers,
'num_fetchers': self._num_fetchers} 'num_fetchers': self._num_fetchers}
pprint.pprint(params) pprint.pprint(params)
print '---------------------------------------------------------' logger.info('---------------------------------------------------------')
Dragon/python/vm/caffe/layers/__init__.py
...@@ -4,18 +4,18 @@ ...@@ -4,18 +4,18 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from data import DataLayer, MemoryDataLayer from .data import DataLayer, MemoryDataLayer
from vision import ConvolutionLayer, DeconvolutionLayer, PoolingLayer, \ from .vision import ConvolutionLayer, DeconvolutionLayer, PoolingLayer, \
LRNLayer, ROIPoolingLayer, ROIAlignLayer, NNResizeLayer LRNLayer, ROIPoolingLayer, ROIAlignLayer, NNResizeLayer
from neuron import ReLULayer, DropoutLayer, TanhLayer, PowerLayer from .neuron import ReLULayer, DropoutLayer, TanhLayer, PowerLayer
from loss import SoftmaxWithLossLayer, SigmoidCrossEntropyLossLayer, \ from .loss import SoftmaxWithLossLayer, SigmoidCrossEntropyLossLayer, \
L2LossLayer, SmoothL1LossLayer L2LossLayer, SmoothL1LossLayer
from mpi import MPIBroadcastLayer, MPIGatherLayer from .mpi import MPIBroadcastLayer, MPIGatherLayer
from common import InnerProductLayer, AccuracyLayer, BatchNormLayer, \ from .common import InnerProductLayer, AccuracyLayer, BatchNormLayer, \
BatchRenormLayer, BNLayer, ConcatLayer, \ BatchRenormLayer, BNLayer, ConcatLayer, \
CropLayer, PythonLayer, AddLayer, \ CropLayer, PythonLayer, AddLayer, \
ReshapeLayer, EltwiseLayer, ScaleLayer, \ ReshapeLayer, EltwiseLayer, ScaleLayer, \
......
Dragon/python/vm/caffe/layers/common.py
...@@ -5,9 +5,10 @@ ...@@ -5,9 +5,10 @@
# -------------------------------------------------------- # --------------------------------------------------------
from dragon.core.tensor import Tensor from dragon.core.tensor import Tensor
from layer import Layer
import dragon.ops as ops import dragon.ops as ops
from .layer import Layer
class InnerProductLayer(Layer): class InnerProductLayer(Layer):
def __init__(self, LayerParameter): def __init__(self, LayerParameter):
super(InnerProductLayer, self).__init__(LayerParameter) super(InnerProductLayer, self).__init__(LayerParameter)
......
Dragon/python/vm/caffe/layers/data.py
...@@ -4,9 +4,10 @@ ...@@ -4,9 +4,10 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from layer import Layer
import dragon.ops as ops import dragon.ops as ops
from .layer import Layer
class DataLayer(Layer): class DataLayer(Layer):
def __init__(self, LayerParameter): def __init__(self, LayerParameter):
super(DataLayer, self).__init__(LayerParameter) super(DataLayer, self).__init__(LayerParameter)
......
Dragon/python/vm/caffe/layers/loss.py
...@@ -4,9 +4,10 @@ ...@@ -4,9 +4,10 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from layer import Layer
import dragon.ops as ops import dragon.ops as ops
from .layer import Layer
class SoftmaxWithLossLayer(Layer): class SoftmaxWithLossLayer(Layer):
def __init__(self, LayerParameter): def __init__(self, LayerParameter):
super(SoftmaxWithLossLayer, self).__init__(LayerParameter) super(SoftmaxWithLossLayer, self).__init__(LayerParameter)
......
Dragon/python/vm/caffe/layers/mpi.py
...@@ -4,9 +4,10 @@ ...@@ -4,9 +4,10 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from layer import Layer
from dragon.ops import MPIBroadcast, MPIGather from dragon.ops import MPIBroadcast, MPIGather
from .layer import Layer
class MPIBroadcastLayer(Layer): class MPIBroadcastLayer(Layer):
def __init__(self, LayerParameter): def __init__(self, LayerParameter):
super(MPIBroadcastLayer, self).__init__(LayerParameter) super(MPIBroadcastLayer, self).__init__(LayerParameter)
......
Dragon/python/vm/caffe/layers/neuron.py
...@@ -4,9 +4,10 @@ ...@@ -4,9 +4,10 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from layer import Layer
import dragon.ops as ops import dragon.ops as ops
from .layer import Layer
class ReLULayer(Layer): class ReLULayer(Layer):
def __init__(self, LayerParameter): def __init__(self, LayerParameter):
super(ReLULayer, self).__init__(LayerParameter) super(ReLULayer, self).__init__(LayerParameter)
......
Dragon/python/vm/caffe/layers/vision.py
...@@ -4,11 +4,11 @@ ...@@ -4,11 +4,11 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
import dragon
from dragon.core.tensor import Tensor from dragon.core.tensor import Tensor
from layer import Layer
import dragon.ops as ops import dragon.ops as ops
from .layer import Layer
class ConvolutionLayer(Layer): class ConvolutionLayer(Layer):
def __init__(self, LayerParameter): def __init__(self, LayerParameter):
super(ConvolutionLayer, self).__init__(LayerParameter) super(ConvolutionLayer, self).__init__(LayerParameter)
......
Dragon/python/vm/caffe/model_libs.py
...@@ -5,11 +5,13 @@ ...@@ -5,11 +5,13 @@
# -------------------------------------------------------- # --------------------------------------------------------
import os import os
from six.moves import range as xrange
from dragon.vm.caffe import layers as L from dragon.vm.caffe import layers as L
from dragon.vm.caffe import params as P from dragon.vm.caffe import params as P
from dragon.vm.caffe.proto import caffe_pb2 from dragon.vm.caffe.proto import caffe_pb2
def check_if_exist(path): def check_if_exist(path):
return os.path.exists(path) return os.path.exists(path)
......
Dragon/python/vm/caffe/net.py
...@@ -6,12 +6,14 @@ ...@@ -6,12 +6,14 @@
from collections import OrderedDict from collections import OrderedDict
from google.protobuf.text_format import Parse from google.protobuf.text_format import Parse
import dragon.core.workspace as ws import dragon.core.workspace as ws
from dragon.core.tensor import Tensor from dragon.core.tensor import Tensor
import dragon.vm.theano as theano import dragon.vm.theano as theano
import dragon.vm.theano.tensor as T import dragon.vm.theano.tensor as T
import proto.caffe_pb2 as pb
import layers as Layer from .proto import caffe_pb2 as pb
from . import layers
class Blob(object): class Blob(object):
def __init__(self, tuple): def __init__(self, tuple):
...@@ -36,7 +38,7 @@ class Net(object): ...@@ -36,7 +38,7 @@ class Net(object):
if len(self._net.input) > 0: if len(self._net.input) > 0:
for input in self._net.input: for input in self._net.input:
if not self._blobs.has_key(input): if not input in self._blobs:
# create new tensors # create new tensors
self._blobs[input] = {'data':Tensor(input).Variable(), self._blobs[input] = {'data':Tensor(input).Variable(),
'diff': Tensor(input + '_grad')} 'diff': Tensor(input + '_grad')}
...@@ -44,7 +46,7 @@ class Net(object): ...@@ -44,7 +46,7 @@ class Net(object):
for layer in self._net.layer: for layer in self._net.layer:
if not self.FilterNet(layer): continue if not self.FilterNet(layer): continue
self._layers.append(getattr(Layer, layer.type + 'Layer')(layer)) self._layers.append(getattr(layers, layer.type + 'Layer')(layer))
self.Setup() self.Setup()
...@@ -134,8 +136,8 @@ class Net(object): ...@@ -134,8 +136,8 @@ class Net(object):
raise TypeError('only type of Net can be shared.') raise TypeError('only type of Net can be shared.')
other_params = other_net.params other_params = other_net.params
for name, blobs in self.params.iteritems(): for name, blobs in self.params.items():
if other_params.has_key(name): if name in other_params:
for idx, blob in enumerate(blobs): for idx, blob in enumerate(blobs):
self._swap_blobs[blob.data] = other_params[name][idx].data self._swap_blobs[blob.data] = other_params[name][idx].data
...@@ -151,7 +153,7 @@ class Net(object): ...@@ -151,7 +153,7 @@ class Net(object):
ret[output] = ws.FetchTensor(net.blobs[output].data) ret[output] = ws.FetchTensor(net.blobs[output].data)
return ret return ret
if kwargs: if kwargs:
for name, blob in kwargs.iteritems(): for name, blob in kwargs.items():
ws.FeedTensor(self._inputs_to_tensors[name], blob) ws.FeedTensor(self._inputs_to_tensors[name], blob)
self.function(return_outputs=False, stage='forward') self.function(return_outputs=False, stage='forward')
...@@ -167,7 +169,7 @@ class Net(object): ...@@ -167,7 +169,7 @@ class Net(object):
if not hasattr(self, '_function'): func = self.function if not hasattr(self, '_function'): func = self.function
tensors = [] tensors = []
for layer in self._net.layer: for layer in self._net.layer:
if self.params.has_key(layer.name): if layer.name in self.params:
for param in self.params[layer.name]: for param in self.params[layer.name]:
tensors.append(param.data) tensors.append(param.data)
...@@ -182,7 +184,7 @@ class Net(object): ...@@ -182,7 +184,7 @@ class Net(object):
def blobs(self): def blobs(self):
""" simply follow the pycaffe style """ """ simply follow the pycaffe style """
return OrderedDict([(name,Blob((blob['data'], blob['diff']))) return OrderedDict([(name,Blob((blob['data'], blob['diff'])))
for name, blob in self._blobs.iteritems()]) for name, blob in self._blobs.items()])
@property @property
def params(self): def params(self):
""" simply follow the pycaffe style """ """ simply follow the pycaffe style """
...@@ -221,6 +223,6 @@ class Net(object): ...@@ -221,6 +223,6 @@ class Net(object):
class PartialNet(Net): class PartialNet(Net):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self._blobs = {} self._blobs = {}
for input, tensor in kwargs.iteritems(): for input, tensor in kwargs.items():
self._blobs[input] = {'data': tensor, 'diff': None} self._blobs[input] = {'data': tensor, 'diff': None}
super(PartialNet, self).__init__(*args) super(PartialNet, self).__init__(*args)
Dragon/python/vm/caffe/net_spec.py
...@@ -5,7 +5,6 @@ ...@@ -5,7 +5,6 @@
# -------------------------------------------------------- # --------------------------------------------------------
from collections import OrderedDict, Counter from collections import OrderedDict, Counter
from .proto import caffe_pb2 from .proto import caffe_pb2
import six import six
......
Dragon/python/vm/caffe/solver.py
...@@ -5,17 +5,20 @@ ...@@ -5,17 +5,20 @@
# -------------------------------------------------------- # --------------------------------------------------------
import time import time
from six.moves import range as xrange
import dragon.core.mpi as mpi import dragon.core.mpi as mpi
import dragon.updaters as updaters import dragon.updaters as updaters
import dragon.tools.summary_writer as sw import dragon.tools.summary_writer as sw
import proto.caffe_pb2 as pb from dragon.vm.caffe.proto import caffe_pb2 as pb
from dragon.core.workspace import FetchTensor, Snapshot from dragon.core.workspace import FetchTensor, Snapshot
from dragon.config import logger
from dragon.vm.caffe.common import root_solver from dragon.vm.caffe.common import root_solver
from dragon.vm.caffe.net import Net from dragon.vm.caffe.net import Net
from dragon.vm.theano import function from dragon.vm.theano import function
from google.protobuf.text_format import Parse from google.protobuf.text_format import Parse
class Solver(object): class Solver(object):
def __init__(self, prototxt): def __init__(self, prototxt):
self._param = pb.SolverParameter() self._param = pb.SolverParameter()
...@@ -102,13 +105,13 @@ class Solver(object): ...@@ -102,13 +105,13 @@ class Solver(object):
if root_solver() and self._param.display: if root_solver() and self._param.display:
if self._iter % self._param.display == 0: if self._iter % self._param.display == 0:
base_lr = self._updater.lr base_lr = self._updater.lr
print 'Iteration %d, lr = %s, loss = %f, time = %.2fs' % \ logger.info('Iteration %d, lr = %s, loss = %f, time = %.2fs' % \
(self._iter, str(base_lr), smoothed_loss, time.time() - tic) (self._iter, str(base_lr), smoothed_loss, time.time() - tic))
tic = time.time() tic = time.time()
for idx, net_output in enumerate(self._net._net_outputs): for idx, net_output in enumerate(self._net._net_outputs):
vals = FetchTensor(self._net.blobs[net_output].data) vals = FetchTensor(self._net.blobs[net_output].data)
for val in vals: for val in vals:
print ' Train net output #{}({}): {}'.format(idx, net_output, val) logger.info(' Train net output #{}({}): {}'.format(idx, net_output, val))
self.scalar_writer.add_summary((net_output, val), self._iter) self.scalar_writer.add_summary((net_output, val), self._iter)
self._iter = self._iter + 1 self._iter = self._iter + 1
...@@ -137,9 +140,9 @@ class Solver(object): ...@@ -137,9 +140,9 @@ class Solver(object):
for idx, val in enumerate(vals): for idx, val in enumerate(vals):
test_score[i] += val; i = i + 1 test_score[i] += val; i = i + 1
if not root_solver(): return if not root_solver(): return
print 'Iteration {}, Test net #{}'.format(self._iter, test_idx) logger.info('Iteration {}, Test net #{}'.format(self._iter, test_idx))
for idx, score in enumerate(test_score): for idx, score in enumerate(test_score):
print ' Test net output #%d(%s): %.4f' % (idx, output_id[idx], score / test_iter) logger.info(' Test net output #%d(%s): %.4f' % (idx, output_id[idx], score / test_iter))
self.scalar_writer.add_summary((output_id[idx], score / test_iter), self._iter) self.scalar_writer.add_summary((output_id[idx], score / test_iter), self._iter)
...@@ -164,8 +167,8 @@ class Solver(object): ...@@ -164,8 +167,8 @@ class Solver(object):
if self._current_step < len(self._param.stepvalue) \ if self._current_step < len(self._param.stepvalue) \
and self._iter >= self._param.stepvalue[self._current_step]: and self._iter >= self._param.stepvalue[self._current_step]:
self._current_step = self._current_step + 1 self._current_step = self._current_step + 1
print 'MultiStep Status: Iteration {}, step = {}' \ logger.info('MultiStep Status: Iteration {}, step = {}' \
.format(self._iter, self._current_step) .format(self._iter, self._current_step))
new_lr = self._param.base_lr * \ new_lr = self._param.base_lr * \
pow(self._param.gamma, self._current_step) pow(self._param.gamma, self._current_step)
self._updater.lr = new_lr self._updater.lr = new_lr
...@@ -178,8 +181,8 @@ class Solver(object): ...@@ -178,8 +181,8 @@ class Solver(object):
else: else:
if self._current_step + 1 < len(stage_iters): if self._current_step + 1 < len(stage_iters):
self._current_step = self._current_step + 1 self._current_step = self._current_step + 1
print 'MultiFixed Status: Iteration {}, stage = {}' \ logger.info('MultiFixed Status: Iteration {}, stage = {}' \
.format(self._iter, self._current_step) .format(self._iter, self._current_step))
self._updater.lr = stage_lrs[self._current_step] self._updater.lr = stage_lrs[self._current_step]
if policy == 'inv': if policy == 'inv':
...@@ -223,7 +226,7 @@ class SGDSolver(Solver): ...@@ -223,7 +226,7 @@ class SGDSolver(Solver):
self._updater = updaters.SGDUpdater(**self._update_param) self._updater = updaters.SGDUpdater(**self._update_param)
# generates update targets # generates update targets
for layer, blobs in self._net.params.iteritems(): self._lr_blobs.extend(blobs) for layer, blobs in self._net.params.items(): self._lr_blobs.extend(blobs)
for idx, blob in enumerate(self._lr_blobs): for idx, blob in enumerate(self._lr_blobs):
if self._net._lr_mults[idx] > 0: if self._net._lr_mults[idx] > 0:
if blob.diff is None: continue if blob.diff is None: continue
...@@ -248,7 +251,7 @@ class NesterovSolver(Solver): ...@@ -248,7 +251,7 @@ class NesterovSolver(Solver):
self._updater = updaters.NesterovUpdater(**self._update_param) self._updater = updaters.NesterovUpdater(**self._update_param)
# generates update targets # generates update targets
for layer, blobs in self._net.params.iteritems(): self._lr_blobs.extend(blobs) for layer, blobs in self._net.params.items(): self._lr_blobs.extend(blobs)
for idx, blob in enumerate(self._lr_blobs): for idx, blob in enumerate(self._lr_blobs):
if self._net._lr_mults[idx] > 0: if self._net._lr_mults[idx] > 0:
if blob.diff is None: continue if blob.diff is None: continue
...@@ -273,7 +276,7 @@ class RMSPropSolver(Solver): ...@@ -273,7 +276,7 @@ class RMSPropSolver(Solver):
self._updater = updaters.RMSPropUpdater(**self._update_param) self._updater = updaters.RMSPropUpdater(**self._update_param)
# generates update targets # generates update targets
for layer, blobs in self._net.params.iteritems(): self._lr_blobs.extend(blobs) for layer, blobs in self._net.params.items(): self._lr_blobs.extend(blobs)
for idx, blob in enumerate(self._lr_blobs): for idx, blob in enumerate(self._lr_blobs):
if self._net._lr_mults[idx] > 0: if self._net._lr_mults[idx] > 0:
if blob.diff is None: continue if blob.diff is None: continue
...@@ -297,7 +300,7 @@ class AdamSolver(Solver): ...@@ -297,7 +300,7 @@ class AdamSolver(Solver):
self._updater = updaters.AdamUpdater(**self._update_param) self._updater = updaters.AdamUpdater(**self._update_param)
# generates update targets # generates update targets
for layer, blobs in self._net.params.iteritems(): self._lr_blobs.extend(blobs) for layer, blobs in self._net.params.items(): self._lr_blobs.extend(blobs)
for idx, blob in enumerate(self._lr_blobs): for idx, blob in enumerate(self._lr_blobs):
if self._net._lr_mults[idx] > 0: if self._net._lr_mults[idx] > 0:
if blob.diff is None: continue if blob.diff is None: continue
......
Dragon/python/vm/tensorflow/__init__.py
...@@ -6,13 +6,14 @@ ...@@ -6,13 +6,14 @@
from dragon.core.scope import TensorScope as variable_scope from dragon.core.scope import TensorScope as variable_scope
from dragon.core.scope import TensorScope as name_scope from dragon.core.scope import TensorScope as name_scope
from core.session import *
from core.variables import * from .core.session import *
from core.collection import * from .core.variables import *
from core.device import * from .core.collection import *
import contrib from .core.device import *
import ops.nn as nn from . import contrib
from ops import * from .ops import nn
from training import train from .ops import *
from utils.gradients import * from .training import train
from .utils.gradients import *
Dragon/python/vm/tensorflow/contrib/__init__.py
...@@ -4,4 +4,4 @@ ...@@ -4,4 +4,4 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from layers import * from .layers import *
\ No newline at end of file \ No newline at end of file
Dragon/python/vm/tensorflow/contrib/layers/__init__.py
...@@ -4,4 +4,4 @@ ...@@ -4,4 +4,4 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from layers import * from .layers import *
\ No newline at end of file \ No newline at end of file
Dragon/python/vm/tensorflow/core/dtypes.py 0 → 100644
# --------------------------------------------------------
# TensorFlow for Dragon
# Copyright(c) 2017 SeetaTech
# Written by Ting Pan
# --------------------------------------------------------
import numpy as np
__all__ = ['int32', 'int64', 'float32', 'bool']
int32 = np.int32
int64 = np.int64
float32 = np.float32
bool = np.bool
\ No newline at end of file
Dragon/python/vm/tensorflow/core/variables.py
...@@ -20,7 +20,7 @@ TRAINABLE_VARIABLES = {} ...@@ -20,7 +20,7 @@ TRAINABLE_VARIABLES = {}
def initialize_all_variables(): def initialize_all_variables():
outputs = [] outputs = []
for tensor, initializer in VARIABLES.iteritems(): for tensor, initializer in VARIABLES.items():
outputs.append(initializer) outputs.append(initializer)
return outputs return outputs
......
Dragon/python/vm/tensorflow/ops/__init__.py
...@@ -4,10 +4,9 @@ ...@@ -4,10 +4,9 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from dtypes import float32, bool from .array_ops import *
from array_ops import * from .constant_op import *
from init_ops import * from .control_flow_ops import *
from random_ops import * from .init_ops import *
from constant_op import * from .math_ops import *
from math_ops import * from .random_ops import *
from control_flow_ops import * \ No newline at end of file
\ No newline at end of file
Dragon/python/vm/tensorflow/ops/array_ops.py
...@@ -15,8 +15,8 @@ __all__ = [ ...@@ -15,8 +15,8 @@ __all__ = [
'reshape' 'reshape'
] ]
import dtypes
import dragon.ops as ops import dragon.ops as ops
from ..core import dtypes
def expand_dims(input, axis=None, name=None, dim=None): def expand_dims(input, axis=None, name=None, dim=None):
......
Dragon/python/vm/tensorflow/ops/constant_op.py
...@@ -7,9 +7,10 @@ ...@@ -7,9 +7,10 @@
__all__ = ['constant'] __all__ = ['constant']
import numpy as np import numpy as np
import dragon.core.workspace as ws import dragon.core.workspace as ws
from dragon.core.tensor import Tensor from dragon.core.tensor import Tensor
import dtypes from ..core import dtypes
def constant(value, dtype=None, shape=None, name=None): def constant(value, dtype=None, shape=None, name=None):
......
Dragon/python/vm/tensorflow/ops/init_ops.py
...@@ -16,8 +16,8 @@ __all__ = [ ...@@ -16,8 +16,8 @@ __all__ = [
'glorot_normal_initializer', 'glorot_normal_initializer',
] ]
import dtypes
import dragon.ops as ops import dragon.ops as ops
from ..core import dtypes
class Initializer(object): class Initializer(object):
......
Dragon/python/vm/tensorflow/ops/math_ops.py
...@@ -25,6 +25,8 @@ __all__ = [ ...@@ -25,6 +25,8 @@ __all__ = [
'add_n' 'add_n'
] ]
from six.moves import range as xrange
import dragon.ops as ops import dragon.ops as ops
......
Dragon/python/vm/tensorflow/ops/nn.py
...@@ -4,4 +4,4 @@ ...@@ -4,4 +4,4 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from nn_ops import * from .nn_ops import *
Dragon/python/vm/tensorflow/ops/random_ops.py
...@@ -11,7 +11,8 @@ __all__ = [ ...@@ -11,7 +11,8 @@ __all__ = [
] ]
import dragon.ops as ops import dragon.ops as ops
import dtypes from ..core import dtypes
def random_normal(shape, def random_normal(shape,
mean=0.0, mean=0.0,
......
Dragon/python/vm/tensorflow/training/train.py
...@@ -4,5 +4,5 @@ ...@@ -4,5 +4,5 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from optimizer import * from .optimizer import *
from saver import * from .saver import *
\ No newline at end of file \ No newline at end of file
Dragon/python/vm/tensorflow/utils/check.py
...@@ -4,11 +4,14 @@ ...@@ -4,11 +4,14 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from six.moves import range as xrange
from dragon.vm.tensorflow.core.variables import placeholder from dragon.vm.tensorflow.core.variables import placeholder
def feed_check(feed_dict): def feed_check(feed_dict):
if feed_dict is not None: if feed_dict is not None:
for key, value in feed_dict.iteritems(): for key, value in feed_dict.items():
if type(key) != placeholder: if type(key) != placeholder:
raise TypeError('only a placeholder can be feeded.') raise TypeError('only a placeholder can be feeded.')
if key.shape is not None: if key.shape is not None:
......
Dragon/python/vm/theano/__init__.py
...@@ -4,9 +4,11 @@ ...@@ -4,9 +4,11 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
from core.swap import shared import numpy as np
from core.function import function
from core.scan import scan
import config
from .core.swap import shared
from .core.function import function
from .core.scan import scan
floatX = np.float32
Dragon/python/vm/theano/core/function.py
...@@ -5,14 +5,18 @@ ...@@ -5,14 +5,18 @@
# -------------------------------------------------------- # --------------------------------------------------------
import copy import copy
from collections import OrderedDict
import numpy as np import numpy as np
import dragon.core.workspace as ws import sys
import dragon.core.mpi as mpi import dragon.core.mpi as mpi
import dragon.core.workspace as ws
import dragon.protos.dragon_pb2 as pb import dragon.protos.dragon_pb2 as pb
from dragon.core.tensor import Tensor from dragon.core.utils import MakeArgument
from dragon.core.gradient_maker import GraphGradientMaker from dragon.core.gradient_maker import GraphGradientMaker
from dragon.core.scope import GetOperatorName, GetTensorName from dragon.core.scope import GetOperatorName, GetTensorName
from dragon.utils import MakeArgument from dragon.core.tensor import Tensor
def GraphDef_Grad(graph_def, targets): def GraphDef_Grad(graph_def, targets):
""" generate all graident targets for CC Graph """ """ generate all graident targets for CC Graph """
...@@ -48,7 +52,7 @@ def GraphDef_Update(graph_def, updater): ...@@ -48,7 +52,7 @@ def GraphDef_Update(graph_def, updater):
extra_kwargs['domain'] = updater._prefix extra_kwargs['domain'] = updater._prefix
# wrap hyper-parameters as Tensor for CC # wrap hyper-parameters as Tensor for CC
for k,v in updater._hyper_params.iteritems(): for k,v in updater._hyper_params.items():
ws.FeedTensor(updater._prefix + k, np.array([v], dtype=np.float32)) ws.FeedTensor(updater._prefix + k, np.array([v], dtype=np.float32))
# check data parallel if necessary # check data parallel if necessary
...@@ -69,7 +73,7 @@ def GraphDef_Update(graph_def, updater): ...@@ -69,7 +73,7 @@ def GraphDef_Update(graph_def, updater):
_, u_target.name = GetOperatorName() _, u_target.name = GetOperatorName()
for tensor in tensors: for tensor in tensors:
u_target.tensor.append(tensor) u_target.tensor.append(tensor)
for k,v in kwargs.iteritems(): for k,v in kwargs.items():
u_target.arg.add().CopyFrom(MakeArgument(k, v)) u_target.arg.add().CopyFrom(MakeArgument(k, v))
graph_def.u_target.extend([u_target]) graph_def.u_target.extend([u_target])
...@@ -109,12 +113,16 @@ def function(inputs=[], outputs=[], swaps=None, updater=None): ...@@ -109,12 +113,16 @@ def function(inputs=[], outputs=[], swaps=None, updater=None):
existing_grads = False existing_grads = False
for output in outputs: for output in outputs:
graph_def.target.extend([output.name]) graph_def.target.extend([output.name])
all_exprs = dict(all_exprs, **output.expressions) if sys.version_info >= (3, 0):
all_exprs = OrderedDict(all_exprs, **output.expressions)
else:
all_exprs = dict(all_exprs, **output.expressions)
all_extra_targets = all_extra_targets.union(output.extra_targets) all_extra_targets = all_extra_targets.union(output.extra_targets)
if len(output.grad_wrts) > 0: existing_grads = True if len(output.grad_wrts) > 0: existing_grads = True
for extra_target in all_extra_targets: graph_def.target.extend([extra_target]) for extra_target in all_extra_targets: graph_def.target.extend([extra_target])
# we should sort out the topology of these operators before using # we should sort out the topology of these operators before using
all_exprs = sorted(all_exprs.iteritems(), key=lambda d:d[0]) all_exprs = sorted(all_exprs.items(), key=lambda d:d[0])
forward_ops = copy.deepcopy([v for k,v in all_exprs]) forward_ops = copy.deepcopy([v for k,v in all_exprs])
# handle swap # handle swap
...@@ -122,16 +130,19 @@ def function(inputs=[], outputs=[], swaps=None, updater=None): ...@@ -122,16 +130,19 @@ def function(inputs=[], outputs=[], swaps=None, updater=None):
name_dict = {} name_dict = {}
external_input_exprs = {} external_input_exprs = {}
for old_tenosr, new_tensor in swaps.iteritems(): for old_tenosr, new_tensor in swaps.items():
if isinstance(new_tensor, Tensor): if isinstance(new_tensor, Tensor):
name_dict[old_tenosr.name] = new_tensor._name name_dict[old_tenosr.name] = new_tensor._name
external_input_exprs = dict(external_input_exprs, **new_tensor.expressions) if sys.version_info >= (3, 0):
external_input_exprs = OrderedDict(external_input_exprs, **new_tensor.expressions)
else:
external_input_exprs = dict(external_input_exprs, **new_tensor.expressions)
elif isinstance(new_tensor, np.ndarray): ws.FeedTensor(new_tensor, GetTensorName()) elif isinstance(new_tensor, np.ndarray): ws.FeedTensor(new_tensor, GetTensorName())
external_input_ops = [v for k,v in external_input_exprs.iteritems()] external_input_ops = [v for k,v in external_input_exprs.items()]
for op in forward_ops: for op in forward_ops:
op.input.extend([name_dict[input] if name_dict.has_key(input) op.input.extend([name_dict[input] if input in name_dict
else input for input in op.input]) else input for input in op.input])
del op.input[:len(op.input)/2] del op.input[:int(len(op.input)/2)]
forward_ops = external_input_ops + forward_ops forward_ops = external_input_ops + forward_ops
......
Dragon/python/vm/theano/core/scan.py
...@@ -41,7 +41,7 @@ def scan(fn, sequences, outputs_info, n_steps=None, axis=0): ...@@ -41,7 +41,7 @@ def scan(fn, sequences, outputs_info, n_steps=None, axis=0):
for output in outputs: for output in outputs:
graph_def.target.extend([output._name]) graph_def.target.extend([output._name])
all_exprs = dict(all_exprs, **output.expressions) all_exprs = dict(all_exprs, **output.expressions)
all_exprs = sorted(all_exprs.iteritems(), key=lambda d:d[0]) all_exprs = sorted(all_exprs.items(), key=lambda d:d[0])
forward_ops = copy.deepcopy([v for k,v in all_exprs]) forward_ops = copy.deepcopy([v for k,v in all_exprs])
graph_def.op.extend(forward_ops) graph_def.op.extend(forward_ops)
......
Dragon/python/vm/theano/tensor/__init__.py
...@@ -5,8 +5,9 @@ ...@@ -5,8 +5,9 @@
# -------------------------------------------------------- # --------------------------------------------------------
from dragon.core.tensor import Tensor from dragon.core.tensor import Tensor
import nnet
import ops from . import nnet
from . import ops
def matrix(name=None): def matrix(name=None):
if name is None: return Tensor().Variable() if name is None: return Tensor().Variable()
......
Dragon/python/vm/theano/tensor/nnet.py
...@@ -4,6 +4,4 @@ ...@@ -4,6 +4,4 @@
# Written by Ting Pan # Written by Ting Pan
# -------------------------------------------------------- # --------------------------------------------------------
import ops from .ops import sigmoid
\ No newline at end of file
sigmoid = ops.sigmoid
\ No newline at end of file
Dragon/src/operators/utils/proposal_op.cu
...@@ -397,4 +397,4 @@ void ProposalOp<Context>::RunWithType() { ...@@ -397,4 +397,4 @@ void ProposalOp<Context>::RunWithType() {
template void ProposalOp<CUDAContext>::RunWithType<float>(); template void ProposalOp<CUDAContext>::RunWithType<float>();
} }
\ No newline at end of file
Dragon/src/protos/dragon.proto
syntax = "proto2";
message TensorProto { message TensorProto {
repeated int32 dims = 1; repeated int32 dims = 1;
enum DataType { enum DataType {
......
README.md
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
### Compile Requirements for C++ ### Compile Requirements for C++
0. Google Protocol Buffer 0. Google Protocol Buffer
1. Python (2.7, 64bit) &nbsp; | &nbsp; Anaconda (2.7, 64bit) 1. Python (2 or 3, 64bit) &nbsp; | &nbsp; Anaconda (2 or 3, 64bit)
2. CUDA [Optional] 2. CUDA [Optional]
3. CUDNN [Optional] 3. CUDNN [Optional]
4. OpenMPI [Optional] 4. OpenMPI [Optional]
...@@ -24,16 +24,17 @@ ...@@ -24,16 +24,17 @@
3. (Optional) Download 3rdparty.zip and unzip to Dragon/3rdparty (Out of source code dir) 3. (Optional) Download 3rdparty.zip and unzip to Dragon/3rdparty (Out of source code dir)
[*Win64*](https://pan.baidu.com/s/1pLmGOLt) (OpenBLAS / Protobuf for VS2013 / CUDNN v6 / Microsoft MPI) [*Win64*](https://pan.baidu.com/s/1pLmGOLt) (OpenBLAS / Protobuf2.6 for VS2013 / CUDNN v6 / Microsoft MPI)
[*Linux64*](https://pan.baidu.com/s/1qXPEOWG) (OpenMPI) [*Linux64*](https://pan.baidu.com/s/1qXPEOWG) (OpenMPI)
4. Configure Dragon/CMakeLists.txt 4. Configure Dragon/CMakeLists.txt
- Select optional libraries [CUDA / CUDNN / BLAS / SSE / MPI / MPI_CUDA_AWARE / CUDA_FP16] - Select optional libraries [PYTHON3 / CUDA / CUDNN / BLAS / SSE / MPI / MPI_CUDA_AWARE / CUDA_FP16]
- Set 3rdparty path (recommend to keep defualt) - Set 3rdparty path (recommend to keep defualt)
- Set python & numpy root path - Set Python include path & Numpy root path
- Set cuda compiling architectures if necessary - Set CUDA compiling architectures if necessary
- GCC version(4.8+, 5.0-) should add ``-std=c++11`` to ``CUDA_NVCC_FLAGS``, if ``nullptr`` is not found. - GCC version(4.8+, 5.0-) should add ``-std=c++11`` to ``CUDA_NVCC_FLAGS``, if ``nullptr`` is not found
- We generate *.h and *.cc files under the ``Dragon/src/protos`` with protobuf2.6, run protoc by yourself if higher are required
5. Environment Variables 5. Environment Variables
### Linux(Only for OpenMPI): ### Linux(Only for OpenMPI):
......
examples/cifar10/gen_lmdb.py
...@@ -11,6 +11,7 @@ import sys ...@@ -11,6 +11,7 @@ import sys
import time import time
import shutil import shutil
import tarfile import tarfile
from six.moves import range as xrange
import cv2 import cv2
...@@ -23,6 +24,11 @@ def untar(tar_file): ...@@ -23,6 +24,11 @@ def untar(tar_file):
t = tarfile.open(tar_file) t = tarfile.open(tar_file)
t.extractall(path='data') t.extractall(path='data')
def wrapper_str(raw_str):
if sys.version_info >= (3, 0):
return raw_str.encode()
return raw_str
def extract_images(): def extract_images():
prefix = 'data/cifar-10-batches-py' prefix = 'data/cifar-10-batches-py'
extract_path = 'data/extract' extract_path = 'data/extract'
...@@ -48,10 +54,9 @@ def extract_images(): ...@@ -48,10 +54,9 @@ def extract_images():
import cPickle import cPickle
with open(batch, 'rb') as f: with open(batch, 'rb') as f:
dict = cPickle.load(f) dict = cPickle.load(f)
for item_idx in xrange(len(dict[wrapper_str('labels')])):
for item_idx in xrange(len(dict['labels'])): im = dict[wrapper_str('data')][item_idx].reshape((3, 32, 32))
im = dict['data'][item_idx].reshape((3, 32, 32)) label = dict[wrapper_str('labels')][item_idx]
label = dict['labels'][item_idx]
im = im.transpose((1, 2, 0)) im = im.transpose((1, 2, 0))
im = im[:, :, ::-1] im = im[:, :, ::-1]
filename = str(total_idx).zfill(ZFILL) + '.jpg' filename = str(total_idx).zfill(ZFILL) + '.jpg'
...@@ -79,7 +84,7 @@ def make_db(image_path, label_path, database_path): ...@@ -79,7 +84,7 @@ def make_db(image_path, label_path, database_path):
if os.path.isdir(database_path) is True: if os.path.isdir(database_path) is True:
raise ValueError('the database path is already exist.') raise ValueError('the database path is already exist.')
print 'start time: ', time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()) print('start time: ', time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()))
db = LMDB(max_commit=10000) db = LMDB(max_commit=10000)
db.open(database_path, mode='w') db.open(database_path, mode='w')
...@@ -97,8 +102,8 @@ def make_db(image_path, label_path, database_path): ...@@ -97,8 +102,8 @@ def make_db(image_path, label_path, database_path):
count += 1 count += 1
if count % 10000 == 0: if count % 10000 == 0:
now_time = time.time() now_time = time.time()
print '{0} / {1} in {2:.2f} sec'.format( print('{0} / {1} in {2:.2f} sec'.format(
count, total_line, now_time - start_time) count, total_line, now_time - start_time))
db.commit() db.commit()
record = record.split() record = record.split()
...@@ -116,26 +121,26 @@ def make_db(image_path, label_path, database_path): ...@@ -116,26 +121,26 @@ def make_db(image_path, label_path, database_path):
db.put(zfill_flag.format(count - 1), datum.SerializeToString()) db.put(zfill_flag.format(count - 1), datum.SerializeToString())
now_time = time.time() now_time = time.time()
print '{0} / {1} in {2:.2f} sec'.format(count, total_line, now_time - start_time) print('{0} / {1} in {2:.2f} sec'.format(count, total_line, now_time - start_time))
db.put('size', str(count)) db.put('size', wrapper_str(str(count)))
db.put('zfill', str(ZFILL)) db.put('zfill', wrapper_str(str(ZFILL)))
db.commit() db.commit()
db.close() db.close()
shutil.copy(label_path, database_path + '/image_list.txt') shutil.copy(label_path, database_path + '/image_list.txt')
end_time = time.time() end_time = time.time()
print '{0} images have been stored in the database.'.format(total_line) print('{0} images have been stored in the database.'.format(total_line))
print 'This task finishes within {0:.2f} seconds.'.format( print('This task finishes within {0:.2f} seconds.'.format(
end_time - start_time) end_time - start_time))
print 'The size of database is {0} MB.'.format( print('The size of database is {0} MB.'.format(
float(os.path.getsize(database_path + '/data.mdb') / 1000 / 1000)) float(os.path.getsize(database_path + '/data.mdb') / 1000 / 1000)))
if __name__ == '__main__': if __name__ == '__main__':
untar('data/cifar-10-python.tar.gz') #untar('data/cifar-10-python.tar.gz')
extract_images() #extract_images()
make_db('data/extract/JPEGImages', make_db('data/extract/JPEGImages',
'data/extract/ImageSets/train.txt', 'data/extract/ImageSets/train.txt',
......
examples/cifar10/infer.py
...@@ -14,7 +14,7 @@ classes = ['airplane', 'automobile', 'bird', 'cat', 'deer', ...@@ -14,7 +14,7 @@ classes = ['airplane', 'automobile', 'bird', 'cat', 'deer',
'dog', 'frog', 'horse', 'ship', 'truck'] 'dog', 'frog', 'horse', 'ship', 'truck']
# init # init
caffe.set_mode_gpu() #caffe.set_mode_gpu()
# load net # load net
net = caffe.Net("cifar10_quick_deploy.prototxt", net = caffe.Net("cifar10_quick_deploy.prototxt",
'snapshots/cifar10_quick_iter_5000.caffemodel', caffe.TEST) 'snapshots/cifar10_quick_iter_5000.caffemodel', caffe.TEST)
...@@ -39,9 +39,9 @@ def run(filename): ...@@ -39,9 +39,9 @@ def run(filename):
pred = score.argmax(0) pred = score.argmax(0)
# show # show
print classes[pred] print(classes[pred])
if __name__ == '__main__': if __name__ == '__main__':
run('data/demo/cat.jpg') run('data/demo/cat.jpg')
\ No newline at end of file
examples/cifar10/solve_full.py
...@@ -15,5 +15,5 @@ if __name__ == '__main__': ...@@ -15,5 +15,5 @@ if __name__ == '__main__':
# solve # solve
solver = caffe.SGDSolver('cifar10_full_solver.prototxt') solver = caffe.SGDSolver('cifar10_full_solver.prototxt')
solver.step(5000) solver.step(70000)
solver.snapshot() solver.snapshot()
\ No newline at end of file
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!