diff --git a/.travis.yml b/.travis.yml index 34a6fa5e4..7d73ed260 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,9 @@ install: - pushd . - ./scripts/travis/install-caffe.sh $(pwd)/deps/caffe + # Torch + - ./scripts/travis/install-torch.sh $(pwd)/deps/torch + # DIGITS - sudo apt-get install graphviz # conda (fast) @@ -29,6 +32,7 @@ install: - pip install -r requirements_test.txt before_script: - export CAFFE_HOME=$(pwd)/deps/caffe + - source $(pwd)/deps/torch/install/bin/torch-activate script: ./digits-test -v --with-coverage --cover-package=digits,tools,scripts after_success: diff --git a/README.md b/README.md index 7c6b1fad9..cf73a50dd 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ For greater performance, you can also install cuDNN. At least one deep learning framework backend is required. -* Caffe (NVIDIA's fork) - [installation instructions](docs/InstallCaffe.md) +* [Mandatory] Caffe (NVIDIA's fork) - [installation instructions](docs/InstallCaffe.md) +* [Optional] Torch7 - [installation instructions](docs/InstallTorch.md) ## Install DIGITS diff --git a/digits/config/current_config.py b/digits/config/current_config.py index 8dee71611..0dab17a4f 100644 --- a/digits/config/current_config.py +++ b/digits/config/current_config.py @@ -7,6 +7,7 @@ from server_name import ServerNameOption from secret_key import SecretKeyOption from caffe_option import CaffeOption +from torch_option import TorchOption option_list = None @@ -24,6 +25,7 @@ def reset(): ServerNameOption(), SecretKeyOption(), CaffeOption(), + TorchOption(), ] reset() diff --git a/digits/config/torch_option.py b/digits/config/torch_option.py new file mode 100644 index 000000000..22cb456f1 --- /dev/null +++ b/digits/config/torch_option.py @@ -0,0 +1,110 @@ +# Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. + +import os +import re +import sys +import imp +import platform +import subprocess + +from digits import device_query +import config_option +import prompt + +class TorchOption(config_option.FrameworkOption): + @staticmethod + def config_file_key(): + return 'torch_root' + + @classmethod + def prompt_title(cls): + return 'Torch' + + @classmethod + def prompt_message(cls): + return 'Where is torch installed?' + + def optional(self): + return True + + def suggestions(self): + suggestions = [] + if 'TORCH_ROOT' in os.environ: + d = os.environ['TORCH_ROOT'] + try: + suggestions.append(prompt.Suggestion( + self.validate(d), 'R', + desc='TORCH_ROOT', default=True)) + except config_option.BadValue as e: + print 'TORCH_ROOT "%s" is invalid:' % d + print '\t%s' % e + if 'TORCH_HOME' in os.environ: + d = os.environ['TORCH_HOME'] + try: + default = True + if len(suggestions) > 0: + default = False + suggestions.append(prompt.Suggestion( + self.validate(d), 'H', + desc='TORCH_HOME', default=default)) + except config_option.BadValue as e: + print 'TORCH_HOME "%s" is invalid:' % d + print '\t%s' % e + suggestions.append(prompt.Suggestion('', 'P', + desc='PATH/TORCHPATH', default=True)) + return suggestions + + @staticmethod + def is_path(): + return True + + @classmethod + def validate(cls, value): + if not value: + return value + + if value == '': + # Find the executable + executable = cls.find_executable('th') + if not executable: + raise config_option.BadValue('torch binary not found in PATH') + #cls.validate_version(executable) + return value + else: + # Find the executable + value = os.path.abspath(value) + if not os.path.isdir(value): + raise config_option.BadValue('"%s" is not a directory' % value) + expected_path = os.path.join(value, 'bin', 'th') + if not os.path.exists(expected_path): + raise config_option.BadValue('torch binary not found at "%s"' % value) + #cls.validate_version(expected_path) + return value + + @staticmethod + def find_executable(program): + """ + Finds an executable by searching through PATH + Returns the path to the executable or None + """ + for path in os.environ['PATH'].split(os.pathsep): + path = path.strip('"') + executable = os.path.join(path, program) + if os.path.isfile(executable) and os.access(executable, os.X_OK): + return executable + return None + + @classmethod + def validate_version(cls, executable): + """ + Utility for checking the caffe version from within validate() + Throws BadValue + + Arguments: + executable -- path to a caffe executable + """ + # Currently DIGITS don't have any restrictions on Torch version, so no need to implement this. + pass + + def apply(self): + pass diff --git a/digits/dataset/tasks/analyze_db.py b/digits/dataset/tasks/analyze_db.py index 44986c765..ef1a90290 100644 --- a/digits/dataset/tasks/analyze_db.py +++ b/digits/dataset/tasks/analyze_db.py @@ -74,7 +74,7 @@ def offer_resources(self, resources): return None @override - def task_arguments(self, resources): + def task_arguments(self, resources, env): args = [sys.executable, os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(digits.__file__))), 'tools', 'analyze_db.py'), diff --git a/digits/dataset/tasks/create_db.py b/digits/dataset/tasks/create_db.py index 06366d337..7a4e33da8 100644 --- a/digits/dataset/tasks/create_db.py +++ b/digits/dataset/tasks/create_db.py @@ -137,7 +137,7 @@ def offer_resources(self, resources): return None @override - def task_arguments(self, resources): + def task_arguments(self, resources, env): args = [sys.executable, os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(digits.__file__))), 'tools', 'create_db.py'), diff --git a/digits/dataset/tasks/parse_folder.py b/digits/dataset/tasks/parse_folder.py index 5aaa19403..f11b7d609 100644 --- a/digits/dataset/tasks/parse_folder.py +++ b/digits/dataset/tasks/parse_folder.py @@ -114,7 +114,7 @@ def offer_resources(self, resources): return None @override - def task_arguments(self, resources): + def task_arguments(self, resources, env): args = [sys.executable, os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(digits.__file__))), 'tools', 'parse_folder.py'), diff --git a/digits/frameworks/__init__.py b/digits/frameworks/__init__.py index bf907a401..0aae9a0e7 100644 --- a/digits/frameworks/__init__.py +++ b/digits/frameworks/__init__.py @@ -2,12 +2,16 @@ from framework import Framework from caffe_framework import CaffeFramework +from torch_framework import TorchFramework from digits.config import config_value # # create framework instances # +# torch is optional +torch = TorchFramework() if config_value('torch_root') else None + # caffe is mandatory caffe = CaffeFramework() @@ -21,6 +25,8 @@ def get_frameworks(): there may be more than one instance per framework class """ frameworks = [caffe] + if torch: + frameworks.append(torch) return frameworks def get_framework_by_id(framework_id): diff --git a/digits/frameworks/caffe_framework.py b/digits/frameworks/caffe_framework.py index 3a5b3697a..6edcf58b2 100644 --- a/digits/frameworks/caffe_framework.py +++ b/digits/frameworks/caffe_framework.py @@ -98,6 +98,7 @@ def get_network_from_previous(self, previous_network): ip_layers[-1].name = '%s_retrain' % ip_layers[-1].name return network + @override def get_network_visualization(self, desc): """ return visualization of network diff --git a/digits/frameworks/errors.py b/digits/frameworks/errors.py index ff0c6fd4f..edbd47d2f 100644 --- a/digits/frameworks/errors.py +++ b/digits/frameworks/errors.py @@ -17,6 +17,17 @@ def __init__(self, message): def __str__(self): return repr(self.message) +@subclass +class NetworkVisualizationError(Error): + """ + Errors that occur when validating a network + """ + def __init__(self, message): + self.message = message + + def __str__(self): + return repr(self.message) + @subclass class InferenceError(Error): """ diff --git a/digits/frameworks/torch_framework.py b/digits/frameworks/torch_framework.py new file mode 100644 index 000000000..cace99860 --- /dev/null +++ b/digits/frameworks/torch_framework.py @@ -0,0 +1,162 @@ +# Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. + +import os +import digits +import re +import subprocess +import time +import tempfile +import flask + +from framework import Framework +from digits import utils +from digits.config import config_value +from digits.model.tasks import TorchTrainTask +from digits.utils import subclass, override +from errors import Error, NetworkVisualizationError, BadNetworkError + +@subclass +class TorchFramework(Framework): + + """ + Defines required methods to interact with the Torch framework + """ + + # short descriptive name + NAME = 'Torch (experimental)' + + # identifier of framework class + CLASS = 'torch' + + # whether this framework can shuffle data during training + CAN_SHUFFLE_DATA = True + + def __init__(self): + super(TorchFramework, self).__init__() + # id must be unique + self.framework_id = self.CLASS + + @override + def create_train_task(self, **kwargs): + """ + create train task + """ + return TorchTrainTask(framework_id = self.framework_id, **kwargs) + + @override + def get_standard_network_desc(self, network): + """ + return description of standard network + """ + networks_dir = os.path.join(os.path.dirname(digits.__file__), 'standard-networks', self.CLASS) + + # Torch's GoogLeNet and AlexNet models are placed in sub folder + if (network == "alexnet" or network == "googlenet"): + networks_dir = os.path.join(networks_dir, 'ImageNet-Training') + + for filename in os.listdir(networks_dir): + path = os.path.join(networks_dir, filename) + if os.path.isfile(path): + match = None + match = re.match(r'%s.lua' % network, filename) + if match: + with open(path) as infile: + return infile.read() + # return None if not found + return None + + @override + def get_network_from_desc(self, network_desc): + """ + return network object from a string representation + """ + # return the same string + return network_desc + + @override + def get_network_from_previous(self, previous_network): + """ + return new instance of network from previous network + """ + # return the same string + return previous_network + + @override + def validate_network(self, data): + """ + validate a network + """ + return True + + @override + def get_network_visualization(self, desc): + """ + return visualization of network + """ + # save network description to temporary file + _, temp_network_path = tempfile.mkstemp(suffix='.lua') + with open(temp_network_path, "w") as outfile: + outfile.write(desc) + + try: # do this in a try..finally clause to make sure we delete the temp file + # build command line + if config_value('torch_root') == '': + torch_bin = 'th' + else: + torch_bin = os.path.join(config_value('torch_root'), 'bin', 'th') + + args = [torch_bin, + os.path.join(os.path.dirname(os.path.dirname(digits.__file__)),'tools','torch','main.lua'), + '--network=%s' % os.path.splitext(os.path.basename(temp_network_path))[0], + '--networkDirectory=%s' % os.path.dirname(temp_network_path), + '--visualizeModel=yes' + ] + + # execute command + p = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + close_fds=True, + ) + + regex = re.compile('\x1b\[[0-9;]*m', re.UNICODE) #TODO: need to include regular expression for MAC color codes + + # the network description will be accumulated from the command output + # when collecting_net_definition==True + collecting_net_definition = False + desc = [] + unrecognized_output = [] + while p.poll() is None: + for line in utils.nonblocking_readlines(p.stdout): + if line is not None: + # Remove whitespace and color codes. color codes are appended to begining and end of line by torch binary i.e., 'th'. Check the below link for more information + # https://groups.google.com/forum/#!searchin/torch7/color$20codes/torch7/8O_0lSgSzuA/Ih6wYg9fgcwJ + line = regex.sub('', line) + timestamp, level, message = TorchTrainTask.preprocess_output_torch(line.strip()) + if message: + if message.startswith('Network definition'): + collecting_net_definition = not collecting_net_definition + else: + if collecting_net_definition: + desc.append(line) + elif len(line): + unrecognized_output.append(line) + else: + time.sleep(0.05) + + if not len(desc): + # we did not find a network description + raise NetworkVisualizationError(''.join(unrecognized_output)) + else: + output = flask.Markup('
')
+                for line in desc:
+                    output += flask.Markup.escape(line)
+                output += flask.Markup('
') + return output + finally: + os.remove(temp_network_path) + + + + + diff --git a/digits/model/images/classification/test_views.py b/digits/model/images/classification/test_views.py index 0682508b3..9c7d20387 100644 --- a/digits/model/images/classification/test_views.py +++ b/digits/model/images/classification/test_views.py @@ -64,6 +64,26 @@ class BaseViewsTest(digits.test_views.BaseViewsTest): } """ + TORCH_NETWORK = \ +""" +require 'nn' +local model = nn.Sequential() +model:add(nn.View(-1):setNumInputDims(3)) -- 10*10*3 -> 300 +model:add(nn.Linear(300, 3)) +model:add(nn.LogSoftMax()) +return function(params) + return { + model = model + } +end +""" + + @classmethod + def setUpClass(cls): + super(BaseViewsTest, cls).setUpClass() + if cls.FRAMEWORK=='torch' and not config_value('torch_root'): + raise unittest.SkipTest('Torch not found') + @classmethod def model_exists(cls, job_id): return cls.job_exists(job_id, 'models') @@ -89,7 +109,7 @@ def delete_model(cls, job_id): @classmethod def network(cls): - return cls.CAFFE_NETWORK + return cls.TORCH_NETWORK if cls.FRAMEWORK=='torch' else cls.CAFFE_NETWORK class BaseViewsTestWithDataset(BaseViewsTest, digits.dataset.images.classification.test_views.BaseViewsTestWithDataset): @@ -99,6 +119,10 @@ class BaseViewsTestWithDataset(BaseViewsTest, # Inherited classes may want to override these attributes CROP_SIZE = None + TRAIN_EPOCHS = 1 + SHUFFLE = False + LR_POLICY = None + LEARNING_RATE = None @classmethod def setUpClass(cls): @@ -113,7 +137,7 @@ def tearDownClass(cls): super(BaseViewsTestWithDataset, cls).tearDownClass() @classmethod - def create_model(cls, **kwargs): + def create_model(cls, network=None, **kwargs): """ Create a model Returns the job_id @@ -122,17 +146,24 @@ def create_model(cls, **kwargs): Keyword arguments: **kwargs -- data to be sent with POST request """ + if network is None: + network = cls.network() data = { 'model_name': 'test_model', 'dataset': cls.dataset_id, 'method': 'custom', - 'custom_network': cls.network(), + 'custom_network': network, 'batch_size': 10, - 'train_epochs': 1, - 'framework' : cls.FRAMEWORK + 'train_epochs': cls.TRAIN_EPOCHS, + 'framework' : cls.FRAMEWORK, + 'shuffle': 'true' if cls.SHUFFLE else 'false' } if cls.CROP_SIZE is not None: data['crop_size'] = cls.CROP_SIZE + if cls.LR_POLICY is not None: + data['lr_policy'] = cls.LR_POLICY + if cls.LEARNING_RATE is not None: + data['learning_rate'] = cls.LEARNING_RATE data.update(kwargs) request_json = data.pop('json', False) @@ -197,6 +228,11 @@ def test_visualize_network(self): image = s.select('img') assert image is not None, "didn't return an image" + def test_customize(self): + rv = self.app.post('/models/customize?network=lenet&framework='+self.FRAMEWORK) + s = BeautifulSoup(rv.data) + body = s.select('body') + assert rv.status_code == 200, 'POST failed with %s\n\n%s' % (rv.status_code, body) class BaseTestCreation(BaseViewsTestWithDataset): """ @@ -270,6 +306,8 @@ def test_select_gpus(self): gpu_list = config_value('gpu_list').split(',') for i in xrange(len(gpu_list)): for combination in itertools.combinations(gpu_list, i+1): + if self.FRAMEWORK=='torch' and len(combination)>1: + raise unittest.SkipTest('Torch not tested with multi-GPU') yield self.check_select_gpus, combination def check_select_gpus(self, gpu_list): @@ -316,6 +354,37 @@ def test_retrain_twice(self): job3_id = self.create_model(**options_3) assert self.model_wait_completion(job3_id) == 'Done', 'third job failed' + def test_bad_network_definition(self): + if self.FRAMEWORK == 'caffe': + bogus_net = """ + layer { + name: "hidden" + type: 'BogusCode' + bottom: "data" + top: "output" + } + layer { + name: "loss" + type: "SoftmaxWithLoss" + bottom: "output" + bottom: "label" + top: "loss" + } + """ + elif self.FRAMEWORK == 'torch': + bogus_net = """ + local model = BogusCode(0) + return function(params) + return { + model = model + } + end + """ + job_id = self.create_model(json=True, network=bogus_net) + assert self.model_wait_completion(job_id) == 'Error', 'job should have failed' + job_info = self.job_info_html(job_id=job_id, job_type='models') + assert 'BogusCode' in job_info + class BaseTestCreated(BaseViewsTestWithModel): """ @@ -633,6 +702,22 @@ class TestCaffeCreatedCropInForm(BaseTestCreatedCropInForm): class TestCaffeCreatedCropInNetwork(BaseTestCreatedCropInNetwork): FRAMEWORK = 'caffe' +class TestTorchViews(BaseTestViews): + FRAMEWORK = 'torch' + +class TestTorchCreation(BaseTestCreation): + FRAMEWORK = 'torch' + +class TestTorchCreated(BaseTestCreated): + FRAMEWORK = 'torch' + TRAIN_EPOCHS = 10 + +class TestTorchCreatedHdf5(TestTorchCreated): + BACKEND = 'hdf5' + +class TestTorchDatasetModelInteractions(BaseTestDatasetModelInteractions): + FRAMEWORK = 'torch' + class TestCaffeLeNet(TestCaffeCreated): IMAGE_WIDTH = 28 IMAGE_HEIGHT = 28 @@ -643,3 +728,22 @@ class TestCaffeLeNet(TestCaffeCreated): 'standard-networks', 'caffe', 'lenet.prototxt') ).read() +class TestTorchLeNet(TestTorchCreated): + IMAGE_WIDTH = 28 + IMAGE_HEIGHT = 28 + IMAGE_CHANNELS = 1 + TRAIN_EPOCHS = 20 + # need more aggressive learning rate + # on such a small dataset + LR_POLICY = 'fixed' + LEARNING_RATE = 0.1 + + TORCH_NETWORK=open( + os.path.join( + os.path.dirname(digits.__file__), + 'standard-networks', 'torch', 'lenet.lua') + ).read() + + +class TestTorchHdf5LeNet(TestTorchLeNet): + BACKEND = 'hdf5' diff --git a/digits/model/images/generic/test_views.py b/digits/model/images/generic/test_views.py index 4f960a06e..998ee51bb 100644 --- a/digits/model/images/generic/test_views.py +++ b/digits/model/images/generic/test_views.py @@ -64,6 +64,18 @@ class BaseViewsTest(digits.test_views.BaseViewsTest): bottom: "label" top: "loss" } +""" + + TORCH_NETWORK = \ +""" +require 'nn' +require 'cunn' +local model = nn.Sequential() +model:add(nn.View(-1):setNumInputDims(3)) -- 10*10*3 -> 300 +model:add(nn.Linear(300, 3)) +model:add(nn.LogSoftMax()) +model:cuda() +return model """ @classmethod @@ -91,7 +103,7 @@ def delete_model(cls, job_id): @classmethod def network(cls): - return cls.CAFFE_NETWORK + return cls.TORCH_NETWORK if cls.FRAMEWORK=='torch' else cls.CAFFE_NETWORK class BaseViewsTestWithDataset(BaseViewsTest, diff --git a/digits/model/tasks/__init__.py b/digits/model/tasks/__init__.py index c8f259e13..40290e453 100644 --- a/digits/model/tasks/__init__.py +++ b/digits/model/tasks/__init__.py @@ -2,4 +2,4 @@ from train import TrainTask from caffe_train import CaffeTrainTask - +from torch_train import TorchTrainTask diff --git a/digits/model/tasks/caffe_train.py b/digits/model/tasks/caffe_train.py index a2f9cb167..b06437efe 100644 --- a/digits/model/tasks/caffe_train.py +++ b/digits/model/tasks/caffe_train.py @@ -690,7 +690,7 @@ def iteration_to_epoch(self, it): return float(it * self.train_epochs) / self.solver.max_iter @override - def task_arguments(self, resources): + def task_arguments(self, resources, env): args = [config_value('caffe_root')['executable'], 'train', '--solver=%s' % self.path(self.solver_file), @@ -1020,7 +1020,7 @@ def get_layer_visualizations(self, net, layers='all'): for bottom in layer.bottom: if bottom in net.blobs and bottom not in added_activations: data = net.blobs[bottom].data[0] - vis = self.get_layer_vis_square(data, + vis = utils.image.get_layer_vis_square(data, allow_heatmap=bool(bottom != 'data')) mean, std, hist = self.get_layer_statistics(data) visualizations.append( @@ -1040,7 +1040,7 @@ def get_layer_visualizations(self, net, layers='all'): if layer.name in net.params: data = net.params[layer.name][0].data if layer.type not in ['InnerProduct']: - vis = self.get_layer_vis_square(data) + vis = utils.image.get_layer_vis_square(data) else: vis = None mean, std, hist = self.get_layer_statistics(data) @@ -1073,7 +1073,7 @@ def get_layer_visualizations(self, net, layers='all'): # don't normalize softmax layers if layer.type == 'Softmax': normalize = False - vis = self.get_layer_vis_square(data, + vis = utils.image.get_layer_vis_square(data, normalize = normalize, allow_heatmap = bool(top != 'data')) mean, std, hist = self.get_layer_statistics(data) @@ -1096,89 +1096,6 @@ def get_layer_visualizations(self, net, layers='all'): return visualizations - def get_layer_vis_square(self, data, - allow_heatmap = True, - normalize = True, - min_img_dim = 100, - max_width = 1200, - ): - """ - Returns a vis_square for the given layer data - - Arguments: - data -- a np.ndarray - - Keyword arguments: - allow_heatmap -- if True, convert single channel images to heatmaps - normalize -- whether to normalize the data when visualizing - max_width -- maximum width for the vis_square - """ - if data.ndim == 1: - # interpret as 1x1 grayscale images - # (N, 1, 1) - data = data[:, np.newaxis, np.newaxis] - elif data.ndim == 2: - # interpret as 1x1 grayscale images - # (N, 1, 1) - data = data.reshape((data.shape[0]*data.shape[1], 1, 1)) - elif data.ndim == 3: - if data.shape[0] == 3: - # interpret as a color image - # (1, H, W,3) - data = data[[2,1,0],...] # BGR to RGB (see issue #59) - data = data.transpose(1,2,0) - data = data[np.newaxis,...] - else: - # interpret as grayscale images - # (N, H, W) - pass - elif data.ndim == 4: - if data.shape[0] == 3: - # interpret as HxW color images - # (N, H, W, 3) - data = data.transpose(1,2,3,0) - data = data[:,:,:,[2,1,0]] # BGR to RGB (see issue #59) - elif data.shape[1] == 3: - # interpret as HxW color images - # (N, H, W, 3) - data = data.transpose(0,2,3,1) - data = data[:,:,:,[2,1,0]] # BGR to RGB (see issue #59) - else: - # interpret as HxW grayscale images - # (N, H, W) - data = data.reshape((data.shape[0]*data.shape[1], data.shape[2], data.shape[3])) - else: - raise RuntimeError('unrecognized data shape: %s' % (data.shape,)) - - # chop off data so that it will fit within max_width - padsize = 0 - width = data.shape[2] - if width > max_width: - data = data[:1,:max_width,:max_width] - else: - if width > 1: - padsize = 1 - width += 1 - n = max(max_width/width,1) - n *= n - data = data[:n] - - if not allow_heatmap and data.ndim == 3: - data = data[...,np.newaxis] - - vis = utils.image.vis_square(data, - padsize = padsize, - normalize = normalize, - ) - - # find minimum dimension and upscale if necessary - _min = sorted(vis.shape[:2])[0] - if _min < min_img_dim: - # upscale image - ratio = min_img_dim/float(_min) - vis = utils.image.upscale(vis, ratio) - return vis - def get_layer_statistics(self, data): """ Returns statistics for the given layer data: diff --git a/digits/model/tasks/torch_train.py b/digits/model/tasks/torch_train.py new file mode 100644 index 000000000..4adae8a32 --- /dev/null +++ b/digits/model/tasks/torch_train.py @@ -0,0 +1,882 @@ +# Copyright (c) 2014-2015, NVIDIA CORPORATION. All rights reserved. + +import os +import re +import caffe +import time +import math +import subprocess +import sys +import operator +import shutil + +import numpy as np + +import h5py + +import tempfile +import PIL.Image +import digits +from train import TrainTask +from digits.config import config_value +from digits.status import Status +from digits import utils, dataset +from digits.utils import subclass, override, constants, errors +from digits.dataset import ImageClassificationDatasetJob +#from digits.frameworks.errors import InferenceError + +# NOTE: Increment this everytime the pickled object changes +PICKLE_VERSION = 1 + +# Constants +TORCH_MODEL_FILE = 'model.lua' +TORCH_SNAPSHOT_PREFIX = 'snapshot' +TORCH_USE_MEAN_PIXEL = True + +@subclass +class TorchTrainTask(TrainTask): + """ + Trains a torch model + """ + + TORCH_LOG = 'torch_output.log' + + def __init__(self, **kwargs): + """ + Arguments: + network -- a NetParameter defining the network + """ + super(TorchTrainTask, self).__init__(**kwargs) + + # save network description to file + with open(os.path.join(self.job_dir, TORCH_MODEL_FILE), "w") as outfile: + outfile.write(self.network) + + self.pickver_task_torch_train = PICKLE_VERSION + + self.current_epoch = 0 + + self.loaded_snapshot_file = None + self.loaded_snapshot_epoch = None + self.image_mean = None + self.classifier = None + self.solver = None + + self.model_file = TORCH_MODEL_FILE + self.train_file = constants.TRAIN_DB + self.val_file = constants.VAL_DB + self.snapshot_prefix = TORCH_SNAPSHOT_PREFIX + self.log_file = self.TORCH_LOG + self.trained_on_cpu = None + + def __getstate__(self): + state = super(TorchTrainTask, self).__getstate__() + + # Don't pickle these things + if 'labels' in state: + del state['labels'] + if 'image_mean' in state: + del state['image_mean'] + if 'classifier' in state: + del state['classifier'] + if 'torch_log' in state: + del state['torch_log'] + + return state + + def __setstate__(self, state): + super(TorchTrainTask, self).__setstate__(state) + + # Make changes to self + self.loaded_snapshot_file = None + self.loaded_snapshot_epoch = None + + # These things don't get pickled + self.image_mean = None + self.classifier = None + + ### Task overrides + + @override + def name(self): + return 'Train Torch Model' + + @override + def before_run(self): + super(TorchTrainTask, self).before_run() + + if not isinstance(self.dataset, dataset.ImageClassificationDatasetJob): + raise NotImplementedError() + + self.torch_log = open(self.path(self.TORCH_LOG), 'a') + self.saving_snapshot = False + self.receiving_train_output = False + self.receiving_val_output = False + self.last_train_update = None + self.displaying_network = False + self.temp_unrecognized_output = [] + return True + + @override + def task_arguments(self, resources, env): + if config_value('torch_root') == '': + torch_bin = 'th' + else: + torch_bin = os.path.join(config_value('torch_root'), 'bin', 'th') + + if self.batch_size is None: + self.batch_size = constants.DEFAULT_BATCH_SIZE + + dataset_backend = self.dataset.train_db_task().backend + assert dataset_backend=='lmdb' or dataset_backend=='hdf5' + + args = [torch_bin, + os.path.join(os.path.dirname(os.path.dirname(digits.__file__)),'tools','torch','main.lua'), + '--network=%s' % self.model_file.split(".")[0], + '--epoch=%d' % int(self.train_epochs), + '--train=%s' % self.dataset.path(constants.TRAIN_DB), + '--networkDirectory=%s' % self.job_dir, + '--save=%s' % self.job_dir, + '--snapshotPrefix=%s' % self.snapshot_prefix, + '--snapshotInterval=%s' % self.snapshot_interval, + '--useMeanPixel=yes', + '--mean=%s' % self.dataset.path(constants.MEAN_FILE_IMAGE), + '--labels=%s' % self.dataset.path(self.dataset.labels_file), + '--batchSize=%d' % self.batch_size, + '--learningRate=%s' % self.learning_rate, + '--policy=%s' % str(self.lr_policy['policy']), + '--dbbackend=%s' % dataset_backend + ] + + #learning rate policy input parameters + if self.lr_policy['policy'] == 'fixed': + pass + elif self.lr_policy['policy'] == 'step': + args.append('--gamma=%s' % self.lr_policy['gamma']) + args.append('--stepvalues=%s' % self.lr_policy['stepsize']) + elif self.lr_policy['policy'] == 'multistep': + args.append('--stepvalues=%s' % self.lr_policy['stepvalue']) + args.append('--gamma=%s' % self.lr_policy['gamma']) + elif self.lr_policy['policy'] == 'exp': + args.append('--gamma=%s' % self.lr_policy['gamma']) + elif self.lr_policy['policy'] == 'inv': + args.append('--gamma=%s' % self.lr_policy['gamma']) + args.append('--power=%s' % self.lr_policy['power']) + elif self.lr_policy['policy'] == 'poly': + args.append('--power=%s' % self.lr_policy['power']) + elif self.lr_policy['policy'] == 'sigmoid': + args.append('--stepvalues=%s' % self.lr_policy['stepsize']) + args.append('--gamma=%s' % self.lr_policy['gamma']) + + if self.shuffle: + args.append('--shuffle=yes') + + if self.crop_size: + args.append('--crop=yes') + args.append('--croplen=%d' % self.crop_size) + + if self.use_mean: + args.append('--subtractMean=yes') + else: + args.append('--subtractMean=no') + + if self.random_seed is not None: + args.append('--seed=%s' % self.random_seed) + + if self.solver_type == 'NESTEROV': + args.append('--optimization=nag') + + if self.solver_type == 'ADAGRAD': + args.append('--optimization=adagrad') + + if self.solver_type == 'SGD': + args.append('--optimization=sgd') + + if os.path.exists(self.dataset.path(constants.VAL_DB)) and self.val_interval > 0: + args.append('--validation=%s' % self.dataset.path(constants.VAL_DB)) + args.append('--interval=%s' % self.val_interval) + + if 'gpus' in resources: + identifiers = [] + for identifier, value in resources['gpus']: + identifiers.append(int(identifier)) + if len(identifiers) == 1: + # only one device must be visible to the th process + # to prevent Torch from loading libraries on all GPUs + env['CUDA_VISIBLE_DEVICES'] = str(identifiers[0]) + args.append('--devid=1') + elif len(identifiers) > 1: + raise NotImplementedError("Multi-GPU with Torch not supported yet") + else: + # switch to CPU mode + args.append('--type=float') + self.trained_on_cpu = True + + if self.pretrained_model: + args.append('--weights=%s' % self.path(self.pretrained_model)) + + return args + + @override + def process_output(self, line): + from digits.webapp import socketio + regex = re.compile('\x1b\[[0-9;]*m', re.UNICODE) #TODO: need to include regular expression for MAC color codes + line=regex.sub('', line).strip() + self.torch_log.write('%s\n' % line) + self.torch_log.flush() + + # parse torch output + timestamp, level, message = self.preprocess_output_torch(line) + + # return false when unrecognized output is encountered + if not level: + # network display in progress + if self.displaying_network: + self.temp_unrecognized_output.append(line) + return True + return False + + if not message: + return True + + # network display ends + if self.displaying_network: + if message.startswith('Network definition ends'): + self.temp_unrecognized_output = [] + self.displaying_network = False + return True + + float_exp = '([-]?inf|[-+]?[0-9]*\.?[0-9]+(e[-+]?[0-9]+)?)' + + # loss and learning rate updates + match = re.match(r'Training \(epoch (\d+\.?\d*)\): \w*loss\w* = %s, lr = %s' % (float_exp, float_exp), message) + if match: + index = float(match.group(1)) + l = match.group(2) + assert l.lower() != '-inf', 'Network reported -inf for training loss. Try changing your learning rate.' #TODO: messages needs to be corrected + assert l.lower() != 'inf', 'Network reported inf for training loss. Try decreasing your learning rate.' + l = float(l) + lr = match.group(4) + assert lr.lower() != '-inf', 'Network reported -inf for learning rate. Try changing your learning rate.' + assert lr.lower() != 'inf', 'Network reported inf for learning rate. Try decreasing your learning rate.' + lr = float(lr) + # epoch updates + self.send_progress_update(index) + + self.save_train_output('loss', 'SoftmaxWithLoss', l) + self.save_train_output('learning_rate', 'LearningRate', lr) + self.logger.debug(message) + + return True + + # testing loss and accuracy updates + match = re.match(r'Validation \(epoch (\d+\.?\d*)\): \w*loss\w* = %s, accuracy = %s' % (float_exp,float_exp), message, flags=re.IGNORECASE) + if match: + index = float(match.group(1)) + l = match.group(2) + a = match.group(4) + if l.lower() != 'inf' and l.lower() != '-inf' and a.lower() != 'inf' and a.lower() != '-inf': + l = float(l) + a = float(a) + self.logger.debug('Network accuracy #%s: %s' % (index, a)) + # epoch updates + self.send_progress_update(index) + + self.save_val_output('accuracy', 'Accuracy', a) + self.save_val_output('loss', 'SoftmaxWithLoss', l) + + return True + + # snapshot saved + if self.saving_snapshot: + if not message.startswith('Snapshot saved'): + self.logger.warning('Torch output format seems to have changed. Expected "Snapshot saved..." after "Snapshotting to..."') + else: + self.logger.info('Snapshot saved.') # to print file name here, you can use "message" + self.detect_snapshots() + self.send_snapshot_update() + self.saving_snapshot = False + return True + + # snapshot starting + match = re.match(r'Snapshotting to (.*)\s*$', message) + if match: + self.saving_snapshot = True + return True + + # network display starting + if message.startswith('Network definition:'): + self.displaying_network = True + return True + + if level in ['error', 'critical']: + self.logger.error('%s: %s' % (self.name(), message)) + self.exception = message + return True + + # skip remaining info and warn messages + return True + + @staticmethod + def preprocess_output_torch(line): + """ + Takes line of output and parses it according to caffe's output format + Returns (timestamp, level, message) or (None, None, None) + """ + # NOTE: This must change when the logging format changes + # LMMDD HH:MM:SS.MICROS pid file:lineno] message + match = re.match(r'(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})\s\[(\w+)\s*]\s+(\S.*)$', line) + if match: + timestamp = time.mktime(time.strptime(match.group(1), '%Y-%m-%d %H:%M:%S')) + level = match.group(2) + message = match.group(3) + if level == 'INFO': + level = 'info' + elif level == 'WARNING': + level = 'warning' + elif level == 'ERROR': + level = 'error' + elif level == 'FAIL': #FAIL + level = 'critical' + return (timestamp, level, message) + else: + #self.logger.warning('Unrecognized task output "%s"' % line) + return (None, None, None) + + def send_snapshot_update(self): + """ + Sends socketio message about the snapshot list + """ + # TODO: move to TrainTask + from digits.webapp import socketio + + socketio.emit('task update', + { + 'task': self.html_id(), + 'update': 'snapshots', + 'data': self.snapshot_list(), + }, + namespace='/jobs', + room=self.job_id, + ) + + ### TrainTask overrides + @override + def after_run(self): + if self.temp_unrecognized_output: + if self.traceback: + self.traceback = self.traceback + ('\n'.join(self.temp_unrecognized_output)) + else: + self.traceback = '\n'.join(self.temp_unrecognized_output) + self.temp_unrecognized_output = [] + self.torch_log.close() + + @override + def after_runtime_error(self): + if os.path.exists(self.path(self.TORCH_LOG)): + output = subprocess.check_output(['tail', '-n40', self.path(self.TORCH_LOG)]) + lines = [] + for line in output.split('\n'): + # parse torch header + timestamp, level, message = self.preprocess_output_torch(line) + + if message: + lines.append(message) + # return the last 20 lines + traceback = '\n\nLast output:\n' + '\n'.join(lines[len(lines)-20:]) if len(lines)>0 else '' + if self.traceback: + self.traceback = self.traceback + traceback + else: + self.traceback = traceback + + @override + def detect_snapshots(self): + self.snapshots = [] + + snapshot_dir = os.path.join(self.job_dir, os.path.dirname(self.snapshot_prefix)) + snapshots = [] + solverstates = [] + + for filename in os.listdir(snapshot_dir): + # find models + match = re.match(r'%s_(\d+)\.?(\d*)_Weights\.t7' % os.path.basename(self.snapshot_prefix), filename) + if match: + epoch = 0 + if match.group(2) == '': + epoch = int(match.group(1)) + else: + epoch = float(match.group(1) + '.' + match.group(2)) + snapshots.append( ( + os.path.join(snapshot_dir, filename), + epoch + ) + ) + + self.snapshots = sorted(snapshots, key=lambda tup: tup[1]) + + return len(self.snapshots) > 0 + + @override + def est_next_snapshot(self): + # TODO: Currently this function is not in use. Probably in future we may have to implement this + return None + + @override + def can_view_weights(self): + return False + + @override + def can_infer_one(self): + if isinstance(self.dataset, ImageClassificationDatasetJob): + return True + return False + + @override + def infer_one(self, data, snapshot_epoch=None, layers=None): + if isinstance(self.dataset, ImageClassificationDatasetJob): + return self.classify_one(data, + snapshot_epoch=snapshot_epoch, + layers=layers, + ) + raise NotImplementedError() + + def classify_one(self, image, snapshot_epoch=None, layers=None): + """ + Classify an image + Returns (predictions, visualizations) + predictions -- an array of [ (label, confidence), ...] for each label, sorted by confidence + visualizations -- an array of (layer_name, activations, weights) for the specified layers + Returns (None, None) if something goes wrong + + Arguments: + image -- a np.array + + Keyword arguments: + snapshot_epoch -- which snapshot to use + layers -- which layer activation[s] and weight[s] to visualize + """ + _, temp_image_path = tempfile.mkstemp(suffix='.jpeg') + image = PIL.Image.fromarray(image) + try: + image.save(temp_image_path, format='jpeg') + except KeyError: + error_message = 'Unable to save file to "%s"' % temp_image_path + self.logger.error(error_message) + raise digits.frameworks.errors.InferenceError(error_message) + + if config_value('torch_root') == '': + torch_bin = 'th' + else: + torch_bin = os.path.join(config_value('torch_root'), 'bin', 'th') + + args = [torch_bin, + os.path.join(os.path.dirname(os.path.dirname(digits.__file__)),'tools','torch','test.lua'), + '--image=%s' % temp_image_path, + '--network=%s' % self.model_file.split(".")[0], + '--networkDirectory=%s' % self.job_dir, + '--load=%s' % self.job_dir, + '--snapshotPrefix=%s' % self.snapshot_prefix, + '--mean=%s' % self.dataset.path(constants.MEAN_FILE_IMAGE), + '--labels=%s' % self.dataset.path(self.dataset.labels_file) + ] + if snapshot_epoch: + args.append('--epoch=%d' % int(snapshot_epoch)) + if TORCH_USE_MEAN_PIXEL: + args.append('--useMeanPixel=yes') + if self.trained_on_cpu: + args.append('--type=float') + + # input image has been resized to network input dimensions by caller + args.append('--crop=no') + + if self.use_mean: + args.append('--subtractMean=yes') + else: + args.append('--subtractMean=no') + + if layers=='all': + args.append('--visualization=yes') + args.append('--save=%s' % self.job_dir) + + # Convert them all to strings + args = [str(x) for x in args] + + regex = re.compile('\x1b\[[0-9;]*m', re.UNICODE) #TODO: need to include regular expression for MAC color codes + self.logger.info('%s classify one task started.' % self.get_framework_id()) + + unrecognized_output = [] + predictions = [] + self.visualization_file = None + + p = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=self.job_dir, + close_fds=True, + ) + + try: + while p.poll() is None: + for line in utils.nonblocking_readlines(p.stdout): + if self.aborted.is_set(): + p.terminate() + raise digits.frameworks.errors.InferenceError('%s classify one task got aborted. error code - %d' % (self.get_framework_id(), p.returncode())) + + if line is not None: + # Remove color codes and whitespace + line=regex.sub('', line).strip() + if line: + if not self.process_test_output(line, predictions, 'one'): + self.logger.warning('%s classify one task unrecognized input: %s' % (self.get_framework_id(), line.strip())) + unrecognized_output.append(line) + else: + time.sleep(0.05) + + except Exception as e: + if p.poll() is None: + p.terminate() + error_message = '' + if type(e) == digits.frameworks.errors.InferenceError: + error_message = e.__str__() + else: + error_message = '%s classify one task failed with error code %d \n %s' % (self.get_framework_id(), p.returncode(), str(e)) + self.logger.error(error_message) + if unrecognized_output: + unrecognized_output = '\n'.join(unrecognized_output) + error_message = error_message + unrecognized_output + raise digits.frameworks.errors.InferenceError(error_message) + + finally: + self.after_test_run(temp_image_path) + + if p.returncode != 0: + error_message = '%s classify one task failed with error code %d' % (self.get_framework_id(), p.returncode) + self.logger.error(error_message) + if unrecognized_output: + unrecognized_output = '\n'.join(unrecognized_output) + error_message = error_message + unrecognized_output + raise digits.frameworks.errors.InferenceError(error_message) + else: + self.logger.info('%s classify one task completed.' % self.get_framework_id()) + + + visualizations = [] + + if layers=='all' and self.visualization_file: + vis_db = h5py.File(self.visualization_file, 'r') + # the HDF5 database is organized as follows: + # + # |- layers + # |- 1 + # | |- name + # | |- activations + # | |- weights + # |- 2 + for layer_id,layer in vis_db['layers'].items(): + layer_desc = layer['name'][...].tostring() + if 'Sequential' in layer_desc or 'Parallel' in layer_desc: + # ignore containers + continue + idx = int(layer_id) + # activations + data = np.array(layer['activations'][...]) + # skip batch dimension + if len(data.shape)>1 and data.shape[0]==1: + data = data[0] + vis = utils.image.get_layer_vis_square(data) + mean, std, hist = self.get_layer_statistics(data) + visualizations.append( + { + 'id': idx, + 'name': layer_desc, + 'vis_type': 'Activations', + 'image_html': utils.image.embed_image_html(vis), + 'data_stats': { + 'shape': data.shape, + 'mean': mean, + 'stddev': std, + 'histogram': hist, + } + } + ) + # weights + if 'weights' in layer: + data = np.array(layer['weights'][...]) + if 'Linear' not in layer_desc: + vis = utils.image.get_layer_vis_square(data) + else: + # Linear (inner product) layers have too many weights + # to display + vis = None + mean, std, hist = self.get_layer_statistics(data) + parameter_count = reduce(operator.mul, data.shape, 1) + if 'bias' in layer: + bias = np.array(layer['bias'][...]) + parameter_count += reduce(operator.mul, bias.shape, 1) + visualizations.append( + { + 'id': idx, + 'name': layer_desc, + 'vis_type': 'Weights', + 'image_html': utils.image.embed_image_html(vis), + 'param_count': parameter_count, + 'data_stats': { + 'shape': data.shape, + 'mean': mean, + 'stddev': std, + 'histogram': hist, + } + } + ) + # sort by layer ID + visualizations = sorted(visualizations,key=lambda x:x['id']) + return (predictions,visualizations) + + def get_layer_statistics(self, data): + """ + Returns statistics for the given layer data: + (mean, standard deviation, histogram) + histogram -- [y, x, ticks] + + Arguments: + data -- a np.ndarray + """ + # XXX These calculations can be super slow + mean = np.mean(data) + std = np.std(data) + y, x = np.histogram(data, bins=20) + y = list(y) + ticks = x[[0,len(x)/2,-1]] + x = [(x[i]+x[i+1])/2.0 for i in xrange(len(x)-1)] + ticks = list(ticks) + return (mean, std, [y, x, ticks]) + + + def after_test_run(self, temp_image_path): + try: + os.remove(temp_image_path) + except OSError: + pass + + def process_test_output(self, line, predictions, test_category): + #from digits.webapp import socketio + + # parse torch output + timestamp, level, message = self.preprocess_output_torch(line) + + # return false when unrecognized output is encountered + if not (level or message): + return False + + if not message: + return True + + float_exp = '([-]?inf|[-+]?[0-9]*\.?[0-9]+(e[-+]?[0-9]+)?)' + + # format of output while testing single image + match = re.match(r'For image \d+, predicted class \d+: \d+ \((.*?)\) %s' % (float_exp), message) + if match: + label = match.group(1) + confidence = match.group(2) + assert confidence.lower() != 'nan', 'Network reported "nan" for confidence value. Please check image and network' + confidence = float(confidence) + predictions.append((label, confidence)) + return True + + # format of output while testing multiple images + match = re.match(r'Predictions for image \d+: (.*)', message) + if match: + values = match.group(1).strip().split(" ") + predictions.append(map(float, values)) + return True + + # path to visualization file + match = re.match(r'Saving visualization to (.*)', message) + if match: + self.visualization_file = match.group(1).strip() + return True + + # displaying info and warn messages as we aren't maintaining seperate log file for model testing + if level == 'info': + self.logger.debug('%s classify %s task : %s' % (self.get_framework_id(), test_category, message)) + return True + if level == 'warning': + self.logger.warning('%s classify %s task : %s' % (self.get_framework_id(), test_category, message)) + return True + + if level in ['error', 'critical']: + raise digits.frameworks.errors.InferenceError('%s classify %s task failed with error message - %s' % (self.get_framework_id(), test_category, message)) + + return True # control never reach this line. It can be removed. + + @override + def can_infer_many(self): + if isinstance(self.dataset, ImageClassificationDatasetJob): + return True + raise NotImplementedError() + + @override + def infer_many(self, data, snapshot_epoch=None): + if isinstance(self.dataset, ImageClassificationDatasetJob): + return self.classify_many(data, snapshot_epoch=snapshot_epoch) + raise NotImplementedError() + + def classify_many(self, images, snapshot_epoch=None): + """ + Returns (labels, results): + labels -- an array of strings + results -- a 2D np array: + [ + [image0_label0_confidence, image0_label1_confidence, ...], + [image1_label0_confidence, image1_label1_confidence, ...], + ... + ] + + Arguments: + images -- a list of np.arrays + + Keyword arguments: + snapshot_epoch -- which snapshot to use + """ + + # create a temporary folder to store images and a temporary file + # to store a list of paths to the images + temp_dir_path = tempfile.mkdtemp() + try: # this try...finally clause is used to clean up the temp directory in any case + _, temp_imgfile_path = tempfile.mkstemp(dir=temp_dir_path, suffix='.txt') + temp_imgfile = open(temp_imgfile_path, "w") + for image in images: + _, temp_image_path = tempfile.mkstemp(dir=temp_dir_path, suffix='.jpeg') + image = PIL.Image.fromarray(image) + try: + image.save(temp_image_path, format='jpeg') + except KeyError: + error_message = 'Unable to save file to "%s"' % temp_image_path + self.logger.error(error_message) + raise digits.frameworks.errors.InferenceError(error_message) + temp_imgfile.write("%s\n" % temp_image_path) + temp_imgfile.close() + + labels = self.get_labels() #TODO: probably we no need to return this, as we can directly access from the calling function + + if config_value('torch_root') == '': + torch_bin = 'th' + else: + torch_bin = os.path.join(config_value('torch_root'), 'bin', 'th') + + args = [torch_bin, + os.path.join(os.path.dirname(os.path.dirname(digits.__file__)),'tools','torch','test.lua'), + '--testMany=yes', + '--allPredictions=yes', #all predictions are grabbed and formatted as required by DIGITS + '--image=%s' % str(temp_imgfile_path), + '--resizeMode=%s' % str(self.dataset.resize_mode), # Here, we are using original images, so they will be resized in Torch code. This logic needs to be changed to eliminate the rework of resizing. Need to find a way to send python images array to Lua script efficiently + '--network=%s' % self.model_file.split(".")[0], + '--networkDirectory=%s' % self.job_dir, + '--load=%s' % self.job_dir, + '--snapshotPrefix=%s' % self.snapshot_prefix, + '--mean=%s' % self.dataset.path(constants.MEAN_FILE_IMAGE), + '--pythonPrefix=%s' % sys.executable, + '--labels=%s' % self.dataset.path(self.dataset.labels_file) + ] + if snapshot_epoch: + args.append('--epoch=%d' % int(snapshot_epoch)) + if TORCH_USE_MEAN_PIXEL: + args.append('--useMeanPixel=yes') + if self.trained_on_cpu: + args.append('--type=float') + + # input images have been resized to network input dimensions by caller + args.append('--crop=no') + + if self.use_mean: + args.append('--subtractMean=yes') + else: + args.append('--subtractMean=no') + + #print args + + # Convert them all to strings + args = [str(x) for x in args] + + regex = re.compile('\x1b\[[0-9;]*m', re.UNICODE) #TODO: need to include regular expression for MAC color codes + self.logger.info('%s classify many task started.' % self.name()) + + unrecognized_output = [] + predictions = [] + p = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=self.job_dir, + close_fds=True, + ) + + try: + while p.poll() is None: + for line in utils.nonblocking_readlines(p.stdout): + if self.aborted.is_set(): + p.terminate() + raise digits.frameworks.errors.InferenceError('%s classify many task got aborted. error code - %d' % (self.get_framework_id(), p.returncode())) + + if line is not None: + # Remove whitespace and color codes. color codes are appended to begining and end of line by torch binary i.e., 'th'. Check the below link for more information + # https://groups.google.com/forum/#!searchin/torch7/color$20codes/torch7/8O_0lSgSzuA/Ih6wYg9fgcwJ + line=regex.sub('', line).strip() + if line: + if not self.process_test_output(line, predictions, 'many'): + self.logger.warning('%s classify many task unrecognized input: %s' % (self.get_framework_id(), line.strip())) + unrecognized_output.append(line) + else: + time.sleep(0.05) + except Exception as e: + if p.poll() is None: + p.terminate() + error_message = '' + if type(e) == digits.frameworks.errors.InferenceError: + error_message = e.__str__() + else: + error_message = '%s classify many task failed with error code %d \n %s' % (self.get_framework_id(), p.returncode(), str(e)) + self.logger.error(error_message) + if unrecognized_output: + unrecognized_output = '\n'.join(unrecognized_output) + error_message = error_message + unrecognized_output + raise digits.frameworks.errors.InferenceError(error_message) + + if p.returncode != 0: + error_message = '%s classify many task failed with error code %d' % (self.get_framework_id(), p.returncode) + self.logger.error(error_message) + if unrecognized_output: + unrecognized_output = '\n'.join(unrecognized_output) + error_message = error_message + unrecognized_output + raise digits.frameworks.errors.InferenceError(error_message) + else: + self.logger.info('%s classify many task completed.' % self.get_framework_id()) + finally: + shutil.rmtree(temp_dir_path) + + return (labels,np.array(predictions)) + + def has_model(self): + """ + Returns True if there is a model that can be used + """ + return len(self.snapshots) != 0 + + @override + def get_model_files(self): + """ + return paths to model files + """ + return { + "Network": self.model_file + } + + @override + def get_network_desc(self): + """ + return text description of network + """ + with open (os.path.join(self.job_dir,TORCH_MODEL_FILE), "r") as infile: + desc = infile.read() + return desc + + diff --git a/digits/standard-networks/torch/ImageNet-Training/alexnet.lua b/digits/standard-networks/torch/ImageNet-Training/alexnet.lua new file mode 100644 index 000000000..13035ba2c --- /dev/null +++ b/digits/standard-networks/torch/ImageNet-Training/alexnet.lua @@ -0,0 +1,81 @@ +-- source: https://github.com/soumith/imagenet-multiGPU.torch/blob/master/models/alexnet_cudnn.lua + +require 'nn' +if pcall(function() require('cudnn') end) then + print('Using CuDNN backend') + backend = cudnn + convLayer = cudnn.SpatialConvolution + convLayerName = 'cudnn.SpatialConvolution' +else + print('Failed to load cudnn backend (is libcudnn.so in your library path?)') + if pcall(function() require('cunn') end) then + print('Falling back to legacy cunn backend') + else + print('Failed to load cunn backend (is CUDA installed?)') + print('Falling back to legacy nn backend') + end + backend = nn -- works with cunn or nn + convLayer = nn.SpatialConvolutionMM + convLayerName = 'nn.SpatialConvolutionMM' +end + +function createModel(nGPU) + assert(nGPU == 1 or nGPU == 2, '1-GPU or 2-GPU supported for AlexNet') + local features + if nGPU == 1 then + features = nn.Concat(2) + else + features = nn.ModelParallel(2) + end + + local fb1 = nn.Sequential() -- branch 1 + fb1:add(convLayer(3,48,11,11,4,4,2,2)) -- 224 -> 55 + fb1:add(backend.ReLU(true)) + fb1:add(backend.SpatialMaxPooling(3,3,2,2)) -- 55 -> 27 + fb1:add(convLayer(48,128,5,5,1,1,2,2)) -- 27 -> 27 + fb1:add(backend.ReLU(true)) + fb1:add(backend.SpatialMaxPooling(3,3,2,2)) -- 27 -> 13 + fb1:add(convLayer(128,192,3,3,1,1,1,1)) -- 13 -> 13 + fb1:add(backend.ReLU(true)) + fb1:add(convLayer(192,192,3,3,1,1,1,1)) -- 13 -> 13 + fb1:add(backend.ReLU(true)) + fb1:add(convLayer(192,128,3,3,1,1,1,1)) -- 13 -> 13 + fb1:add(backend.ReLU(true)) + fb1:add(backend.SpatialMaxPooling(3,3,2,2)) -- 13 -> 6 + + local fb2 = fb1:clone() -- branch 2 + for k,v in ipairs(fb2:findModules(convLayerName)) do + v:reset() -- reset branch 2's weights + end + + features:add(fb1) + features:add(fb2) + + -- 1.3. Create Classifier (fully connected layers) + local classifier = nn.Sequential() + classifier:add(nn.View(256*6*6)) + classifier:add(nn.Dropout(0.5)) + classifier:add(nn.Linear(256*6*6, 4096)) + classifier:add(nn.Threshold(0, 1e-6)) + classifier:add(nn.Dropout(0.5)) + classifier:add(nn.Linear(4096, 4096)) + classifier:add(nn.Threshold(0, 1e-6)) + classifier:add(nn.Linear(4096, 1000)) + classifier:add(nn.LogSoftMax()) + + -- 1.4. Combine 1.1 and 1.3 to produce final model + local model = nn.Sequential():add(features):add(classifier) + + return model +end + +-- return function that returns network definition +return function(params) + assert(params.ngpus<=1, 'Model supports only one GPU') + return { + model = createModel(1), + croplen = 224 + } +end + + diff --git a/digits/standard-networks/torch/ImageNet-Training/googlenet.lua b/digits/standard-networks/torch/ImageNet-Training/googlenet.lua new file mode 100644 index 000000000..cc957fe4a --- /dev/null +++ b/digits/standard-networks/torch/ImageNet-Training/googlenet.lua @@ -0,0 +1,127 @@ +-- source: https://github.com/soumith/imagenet-multiGPU.torch/blob/master/models/alexnet_cudnn.lua + +require 'nn' +if pcall(function() require('cudnn') end) then + print('Using CuDNN backend') + backend = cudnn + convLayer = cudnn.SpatialConvolution +else + print('Failed to load cudnn backend (is libcudnn.so in your library path?)') + if pcall(function() require('cunn') end) then + print('Falling back to legacy cunn backend') + else + print('Failed to load cunn backend (is CUDA installed?)') + print('Falling back to legacy nn backend') + end + backend = nn -- works with cunn or nn + convLayer = nn.SpatialConvolutionMM +end + +local function inception(input_size, config) + local concat = nn.Concat(2) + if config[1][1] ~= 0 then + local conv1 = nn.Sequential() + conv1:add(convLayer(input_size, config[1][1],1,1,1,1)):add(backend.ReLU(true)) + concat:add(conv1) + end + + local conv3 = nn.Sequential() + conv3:add(convLayer( input_size, config[2][1],1,1,1,1)):add(backend.ReLU(true)) + conv3:add(convLayer(config[2][1], config[2][2],3,3,1,1,1,1)):add(backend.ReLU(true)) + concat:add(conv3) + + local conv3xx = nn.Sequential() + conv3xx:add(convLayer( input_size, config[3][1],1,1,1,1)):add(backend.ReLU(true)) + conv3xx:add(convLayer(config[3][1], config[3][2],3,3,1,1,1,1)):add(backend.ReLU(true)) + conv3xx:add(convLayer(config[3][2], config[3][2],3,3,1,1,1,1)):add(backend.ReLU(true)) + concat:add(conv3xx) + + local pool = nn.Sequential() + pool:add(nn.SpatialZeroPadding(1,1,1,1)) -- remove after getting cudnn R2 into fbcode + if config[4][1] == 'max' then + pool:add(backend.SpatialMaxPooling(3,3,1,1):ceil()) + elseif config[4][1] == 'avg' then + local l = backend.SpatialAveragePooling(3,3,1,1) + if backend == cudnn then l = l:ceil() end + pool:add(l) + else + error('Unknown pooling') + end + if config[4][2] ~= 0 then + pool:add(convLayer(input_size, config[4][2],1,1,1,1)):add(backend.ReLU(true)) + end + concat:add(pool) + + return concat +end + +function createModel(nGPU) + -- batch normalization added on top of convolutional layers in feature branch + -- in order to help the network learn faster + local features = nn.Sequential() + features:add(nn.MulConstant(0.02)) + features:add(convLayer(3,64,7,7,2,2,3,3)):add(nn.SpatialBatchNormalization(64,1e-3)):add(backend.ReLU(true)) + features:add(backend.SpatialMaxPooling(3,3,2,2):ceil()) + features:add(convLayer(64,64,1,1)):add(nn.SpatialBatchNormalization(64,1e-3)):add(backend.ReLU(true)) + features:add(convLayer(64,192,3,3,1,1,1,1)):add(nn.SpatialBatchNormalization(192,1e-3)):add(backend.ReLU(true)) + features:add(backend.SpatialMaxPooling(3,3,2,2):ceil()) + features:add(inception( 192, {{ 64},{ 64, 64},{ 64, 96},{'avg', 32}})) -- 3(a) + features:add(inception( 256, {{ 64},{ 64, 96},{ 64, 96},{'avg', 64}})) -- 3(b) + features:add(inception( 320, {{ 0},{128,160},{ 64, 96},{'max', 0}})) -- 3(c) + features:add(convLayer(576,576,2,2,2,2)):add(nn.SpatialBatchNormalization(576,1e-3)) + features:add(inception( 576, {{224},{ 64, 96},{ 96,128},{'avg',128}})) -- 4(a) + features:add(inception( 576, {{192},{ 96,128},{ 96,128},{'avg',128}})) -- 4(b) + features:add(inception( 576, {{160},{128,160},{128,160},{'avg', 96}})) -- 4(c) + features:add(inception( 576, {{ 96},{128,192},{160,192},{'avg', 96}})) -- 4(d) + + local main_branch = nn.Sequential() + main_branch:add(inception( 576, {{ 0},{128,192},{192,256},{'max', 0}})) -- 4(e) + main_branch:add(convLayer(1024,1024,2,2,2,2)):add(nn.SpatialBatchNormalization(1024,1e-3)) + main_branch:add(inception(1024, {{352},{192,320},{160,224},{'avg',128}})) -- 5(a) + main_branch:add(inception(1024, {{352},{192,320},{192,224},{'max',128}})) -- 5(b) + main_branch:add(backend.SpatialAveragePooling(7,7,1,1)) + main_branch:add(nn.View(1024):setNumInputDims(3)) + main_branch:add(nn.Linear(1024,1000)) + main_branch:add(nn.LogSoftMax()) + + -- add auxillary classifier here (thanks to Christian Szegedy for the details) + local aux_classifier = nn.Sequential() + local l = backend.SpatialAveragePooling(5,5,3,3) + if backend == cudnn then l = l:ceil() end + aux_classifier:add(l) + aux_classifier:add(convLayer(576,128,1,1,1,1)):add(nn.SpatialBatchNormalization(128,1e-3)) + aux_classifier:add(nn.View(128*4*4):setNumInputDims(3)) + aux_classifier:add(nn.Linear(128*4*4,768)) + aux_classifier:add(nn.ReLU()) + aux_classifier:add(nn.Linear(768,1000)) + aux_classifier:add(nn.LogSoftMax()) + + local splitter = nn.Concat(2) + splitter:add(main_branch):add(aux_classifier) + --local model = nn.Sequential():add(features):add(splitter) + local model = nn.Sequential():add(features):add(main_branch) + + if nGPU > 1 then + assert(nGPU <= cutorch.getDeviceCount(), 'number of GPUs less than nGPU specified') + require 'fbcunn' + local model_single = model + model = nn.DataParallel(1) + for i=1,nGPU do + cutorch.withDevice(i, function() + model:add(model_single:clone()) + end) + end + end + + return model +end + +-- return function that returns network definition +return function(params) + assert(params.ngpus<=1, 'Model supports only one GPU') + return { + model = createModel(1), + croplen = 224 + } +end + diff --git a/digits/standard-networks/torch/lenet.lua b/digits/standard-networks/torch/lenet.lua new file mode 100644 index 000000000..126bd0ce7 --- /dev/null +++ b/digits/standard-networks/torch/lenet.lua @@ -0,0 +1,26 @@ +require 'nn' + +-- -- This is a LeNet model. For more information: http://yann.lecun.com/exdb/lenet/ + +local lenet = nn.Sequential() +lenet:add(nn.MulConstant(0.00390625)) +lenet:add(nn.SpatialConvolution(1,20,5,5,1,1,0)) -- 1*28*28 -> 20*24*24 +lenet:add(nn.SpatialMaxPooling(2, 2, 2, 2)) -- 20*24*24 -> 20*12*12 +lenet:add(nn.SpatialConvolution(20,50,5,5,1,1,0)) -- 20*12*12 -> 50*8*8 +lenet:add(nn.SpatialMaxPooling(2,2,2,2)) -- 50*8*8 -> 50*4*4 +lenet:add(nn.View(-1):setNumInputDims(3)) -- 50*4*4 -> 800 +lenet:add(nn.Linear(800,500)) -- 800 -> 500 +lenet:add(nn.ReLU()) +lenet:add(nn.Linear(500, 10)) -- 500 -> 10 +lenet:add(nn.LogSoftMax()) + +-- return function that returns network definition +return function(params) + assert(params.ngpus<=1, 'Model supports only CPU or single-GPU') + return { + model = lenet, + loss = nn.ClassNLLCriterion() + } +end + + diff --git a/digits/task.py b/digits/task.py index fffe5a5cc..c07da361d 100644 --- a/digits/task.py +++ b/digits/task.py @@ -76,6 +76,12 @@ def name(self): """ raise NotImplementedError + def get_framework_id(self): + """ + Returns a string + """ + raise NotImplementedError('Please implement me') + def html_id(self): """ Returns a string @@ -151,13 +157,14 @@ def offer_resources(self, resources): """ raise NotImplementedError - def task_arguments(self, resources): + def task_arguments(self, resources, env): """ Returns args used by subprocess.Popen to execute the task Returns False if the args cannot be set properly Arguments: resources -- the resources assigned by the scheduler for this task + environ -- os.environ instance to run process in """ raise NotImplementedError @@ -177,7 +184,8 @@ def run(self, resources): """ self.before_run() - args = self.task_arguments(resources) + env = os.environ.copy() + args = self.task_arguments(resources, env ) if not args: self.logger.error('Could not create the arguments for Popen') self.status = Status.ERROR @@ -195,6 +203,7 @@ def run(self, resources): stderr=subprocess.STDOUT, cwd=self.job_dir, close_fds=False if platform.system() == 'Windows' else True, + env=env, ) try: diff --git a/digits/templates/models/images/classification/classify_one.html b/digits/templates/models/images/classification/classify_one.html index 228362b59..36a6403b4 100644 --- a/digits/templates/models/images/classification/classify_one.html +++ b/digits/templates/models/images/classification/classify_one.html @@ -89,7 +89,7 @@

Predictions

{% for vis in visualizations %} -

"{{vis['name']}}"

+

{{vis['name']}}

{{vis['vis_type']}} {% if 'layer_type' in vis %} diff --git a/digits/templates/models/images/classification/new.html b/digits/templates/models/images/classification/new.html index 1c341dd3d..d4809f776 100644 --- a/digits/templates/models/images/classification/new.html +++ b/digits/templates/models/images/classification/new.html @@ -353,8 +353,30 @@

Solver Options

frameworks['{{ fw.get_id() }}'] = framework; {% endfor %} +function setFramework(fwid) +{ + $('#framework').val(fwid); + if (frameworks[fwid].can_shuffle) + $("#shuffle-data").show(); + else + $("#shuffle-data").hide(); + if (fwid == 'torch') + $("#torch-warning").show(); + else + $("#torch-warning").hide(); + $('#stdnetRole a[href="'+"#"+fwid+"_standard"+'"]').tab('show'); + $('#customFramework a[href="'+"#"+fwid+"_custom"+'"]').tab('show'); +} +