diff --git a/onnxruntime/test/server/integration_tests/function_tests.py b/onnxruntime/test/server/integration_tests/function_tests.py index aabeb8bfc1ceb..9baa095569184 100644 --- a/onnxruntime/test/server/integration_tests/function_tests.py +++ b/onnxruntime/test/server/integration_tests/function_tests.py @@ -26,19 +26,21 @@ class HttpJsonPayloadTests(unittest.TestCase): @classmethod def setUpClass(cls): - cmd = [cls.server_app_path, '--http_port', str(cls.server_port), '--model_path', os.path.join(cls.model_path, 'mnist.onnx'), '--log_level', cls.log_level] - print('Launching server app: [{0}]'.format(' '.join(cmd))) + onnx_model = os.path.join(cls.model_path, 'mnist.onnx') + test_util.prepare_mnist_model(onnx_model) + cmd = [cls.server_app_path, '--http_port', str(cls.server_port), '--model_path', onnx_model, '--log_level', cls.log_level] + test_util.test_log('Launching server app: [{0}]'.format(' '.join(cmd))) cls.server_app_proc = subprocess.Popen(cmd) - print('Server app PID: {0}'.format(cls.server_app_proc.pid)) - print('Sleep {0} second(s) to wait for server initialization'.format(cls.wait_server_ready_in_seconds)) + test_util.test_log('Server app PID: {0}'.format(cls.server_app_proc.pid)) + test_util.test_log('Sleep {0} second(s) to wait for server initialization'.format(cls.wait_server_ready_in_seconds)) time.sleep(cls.wait_server_ready_in_seconds) @classmethod def tearDownClass(cls): - print('Shutdown server app') + test_util.test_log('Shutdown server app') cls.server_app_proc.kill() - print('PID {0} has been killed: {1}'.format(cls.server_app_proc.pid, test_util.is_process_killed(cls.server_app_proc.pid))) + test_util.test_log('PID {0} has been killed: {1}'.format(cls.server_app_proc.pid, test_util.is_process_killed(cls.server_app_proc.pid))) def test_mnist_happy_path(self): @@ -59,7 +61,7 @@ def test_mnist_happy_path(self): } url = self.url_pattern.format(self.server_ip, self.server_port, 'default_model', 12345) - print(url) + test_util.test_log(url) r = requests.post(url, headers=request_headers, data=request_payload) self.assertEqual(r.status_code, 200) self.assertEqual(r.headers.get('Content-Type'), 'application/json') @@ -89,7 +91,7 @@ def test_mnist_happy_path(self): def test_mnist_invalid_url(self): url = self.url_pattern.format(self.server_ip, self.server_port, 'default_model', -1) - print(url) + test_util.test_log(url) request_headers = { 'Content-Type': 'application/json', @@ -105,7 +107,7 @@ def test_mnist_invalid_url(self): def test_mnist_invalid_content_type(self): input_data_file = os.path.join(self.test_data_path, 'mnist_test_data_set_0_input.json') url = self.url_pattern.format(self.server_ip, self.server_port, 'default_model', 12345) - print(url) + test_util.test_log(url) request_headers = { 'Content-Type': 'application/abc', @@ -127,7 +129,7 @@ def test_mnist_invalid_content_type(self): def test_mnist_missing_content_type(self): input_data_file = os.path.join(self.test_data_path, 'mnist_test_data_set_0_input.json') url = self.url_pattern.format(self.server_ip, self.server_port, 'default_model', 12345) - print(url) + test_util.test_log(url) request_headers = { 'Accept': 'application/json' @@ -161,7 +163,7 @@ def test_single_model_shortcut(self): } url = "http://{0}:{1}/score".format(self.server_ip, self.server_port) - print(url) + test_util.test_log(url) r = requests.post(url, headers=request_headers, data=request_payload) self.assertEqual(r.status_code, 200) self.assertEqual(r.headers.get('Content-Type'), 'application/json') @@ -202,19 +204,21 @@ class HttpProtobufPayloadTests(unittest.TestCase): @classmethod def setUpClass(cls): - cmd = [cls.server_app_path, '--http_port', str(cls.server_port), '--model_path', os.path.join(cls.model_path, 'mnist.onnx'), '--log_level', cls.log_level] - print('Launching server app: [{0}]'.format(' '.join(cmd))) + onnx_model = os.path.join(cls.model_path, 'mnist.onnx') + test_util.prepare_mnist_model(onnx_model) + cmd = [cls.server_app_path, '--http_port', str(cls.server_port), '--model_path', onnx_model, '--log_level', cls.log_level] + test_util.test_log('Launching server app: [{0}]'.format(' '.join(cmd))) cls.server_app_proc = subprocess.Popen(cmd) - print('Server app PID: {0}'.format(cls.server_app_proc.pid)) - print('Sleep {0} second(s) to wait for server initialization'.format(cls.wait_server_ready_in_seconds)) + test_util.test_log('Server app PID: {0}'.format(cls.server_app_proc.pid)) + test_util.test_log('Sleep {0} second(s) to wait for server initialization'.format(cls.wait_server_ready_in_seconds)) time.sleep(cls.wait_server_ready_in_seconds) @classmethod def tearDownClass(cls): - print('Shutdown server app') + test_util.test_log('Shutdown server app') cls.server_app_proc.kill() - print('PID {0} has been killed: {1}'.format(cls.server_app_proc.pid, test_util.is_process_killed(cls.server_app_proc.pid))) + test_util.test_log('PID {0} has been killed: {1}'.format(cls.server_app_proc.pid, test_util.is_process_killed(cls.server_app_proc.pid))) def test_mnist_happy_path(self): @@ -233,7 +237,7 @@ def test_mnist_happy_path(self): } url = self.url_pattern.format(self.server_ip, self.server_port, 'default_model', 12345) - print(url) + test_util.test_log(url) r = requests.post(url, headers=request_headers, data=request_payload) self.assertEqual(r.status_code, 200) self.assertEqual(r.headers.get('Content-Type'), 'application/x-protobuf') @@ -277,7 +281,7 @@ def test_respect_accept_header(self): } url = self.url_pattern.format(self.server_ip, self.server_port, 'default_model', 12345) - print(url) + test_util.test_log(url) r = requests.post(url, headers=request_headers, data=request_payload) self.assertEqual(r.status_code, 200) self.assertEqual(r.headers.get('Content-Type'), h) @@ -294,7 +298,7 @@ def test_missing_accept_header(self): } url = self.url_pattern.format(self.server_ip, self.server_port, 'default_model', 12345) - print(url) + test_util.test_log(url) r = requests.post(url, headers=request_headers, data=request_payload) self.assertEqual(r.status_code, 200) self.assertEqual(r.headers.get('Content-Type'), 'application/octet-stream') @@ -312,7 +316,7 @@ def test_any_accept_header(self): } url = self.url_pattern.format(self.server_ip, self.server_port, 'default_model', 12345) - print(url) + test_util.test_log(url) r = requests.post(url, headers=request_headers, data=request_payload) self.assertEqual(r.status_code, 200) self.assertEqual(r.headers.get('Content-Type'), 'application/octet-stream') @@ -330,24 +334,26 @@ class HttpEndpointTests(unittest.TestCase): @classmethod def setUpClass(cls): - cmd = [cls.server_app_path, '--http_port', str(cls.server_port), '--model_path', os.path.join(cls.model_path, 'mnist.onnx'), '--log_level', cls.log_level] - print('Launching server app: [{0}]'.format(' '.join(cmd))) + onnx_model = os.path.join(cls.model_path, 'mnist.onnx') + test_util.prepare_mnist_model(onnx_model) + cmd = [cls.server_app_path, '--http_port', str(cls.server_port), '--model_path', onnx_model, '--log_level', cls.log_level] + test_util.test_log('Launching server app: [{0}]'.format(' '.join(cmd))) cls.server_app_proc = subprocess.Popen(cmd) - print('Server app PID: {0}'.format(cls.server_app_proc.pid)) - print('Sleep {0} second(s) to wait for server initialization'.format(cls.wait_server_ready_in_seconds)) + test_util.test_log('Server app PID: {0}'.format(cls.server_app_proc.pid)) + test_util.test_log('Sleep {0} second(s) to wait for server initialization'.format(cls.wait_server_ready_in_seconds)) time.sleep(cls.wait_server_ready_in_seconds) @classmethod def tearDownClass(cls): - print('Shutdown server app') + test_util.test_log('Shutdown server app') cls.server_app_proc.kill() - print('PID {0} has been killed: {1}'.format(cls.server_app_proc.pid, test_util.is_process_killed(cls.server_app_proc.pid))) + test_util.test_log('PID {0} has been killed: {1}'.format(cls.server_app_proc.pid, test_util.is_process_killed(cls.server_app_proc.pid))) def test_health_endpoint(self): url = url = "http://{0}:{1}/".format(self.server_ip, self.server_port) - print(url) + test_util.test_log(url) r = requests.get(url) self.assertEqual(r.status_code, 200) self.assertEqual(r.content.decode('utf-8'), 'Healthy') diff --git a/onnxruntime/test/server/integration_tests/test_util.py b/onnxruntime/test/server/integration_tests/test_util.py index e268a735c8c68..4b142f3e6dd3f 100644 --- a/onnxruntime/test/server/integration_tests/test_util.py +++ b/onnxruntime/test/server/integration_tests/test_util.py @@ -13,6 +13,7 @@ import socket import errno import sys +import urllib.request import predict_pb2 import onnx_ml_pb2 @@ -38,6 +39,14 @@ def is_process_killed(pid): else: return True +def prepare_mnist_model(target_path): + # TODO: This need to be replaced by test data on build machine after merged to upstream master. + if not os.path.isfile(target_path): + test_log('Downloading model from blob storage: https://ortsrvdev.blob.core.windows.net/test-data/mnist.onnx to {0}'.format(target_path)) + urllib.request.urlretrieve('https://ortsrvdev.blob.core.windows.net/test-data/mnist.onnx', target_path) + else: + test_log('Found mnist model at {0}'.format(target_path)) + def decode_base64_string(s, count_and_type): b = base64.b64decode(s)