Merge pull request #321 from HttpRunner/report

1.5.7

httprunner/__about__.py

@@ -1,7 +1,7 @@
 __title__ = 'HttpRunner'
 __description__ = 'One-stop solution for HTTP(S) testing.'
 __url__ = 'https://github.com/HttpRunner/HttpRunner'
-__version__ = '1.5.6'
+__version__ = '1.5.7'
 __author__ = 'debugtalk'
 __author_email__ = 'mail@debugtalk.com'
 __license__ = 'MIT'

httprunner/built_in.py

@@ -13,7 +13,7 @@ import string
 import time
 
 from httprunner.compat import basestring, builtin_str, integer_types, str
-from httprunner.exception import ParamsError
+from httprunner.exceptions import ParamsError
 from requests_toolbelt import MultipartEncoder
 
 

httprunner/client.py

@@ -6,7 +6,7 @@ import time
 import requests
 import urllib3
 from httprunner import logger
-from httprunner.exception import ParamsError
+from httprunner.exceptions import ParamsError
 from requests import Request, Response
 from requests.exceptions import (InvalidSchema, InvalidURL, MissingSchema,
                                  RequestException)
@@ -115,7 +115,7 @@ class HttpSession(requests.Session):
         def log_print(request_response):
             msg = "\n================== {} details ==================\n".format(request_response)
             for key, value in self.meta_data[request_response].items():
-                msg += "{:<16} : {}\n".format(key, value)
+                msg += "{:<16} : {}\n".format(key, repr(value))
             logger.log_debug(msg)
 
         # record original request info

httprunner/compat.py

@@ -8,6 +8,11 @@ This module handles import compatibility issues between Python 2 and
 Python 3.
 """
 
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
 import sys
 
 # -------
@@ -23,15 +28,16 @@ is_py2 = (_ver[0] == 2)
 #: Python 3.x?
 is_py3 = (_ver[0] == 3)
 
-try:
-    import simplejson as json
-except ImportError:
-    import json
 
 # ---------
 # Specifics
 # ---------
 
+try:
+    JSONDecodeError = json.JSONDecodeError
+except AttributeError:
+    JSONDecodeError = ValueError
+
 if is_py2:
     from urllib3.packages.ordered_dict import OrderedDict
 
@@ -42,6 +48,8 @@ if is_py2:
     numeric_types = (int, long, float)
     integer_types = (int, long)
 
+    FileNotFoundError = IOError
+
 elif is_py3:
     from collections import OrderedDict
 
@@ -51,3 +59,5 @@ elif is_py3:
     basestring = (str, bytes)
     numeric_types = (int, float)
     integer_types = (int,)
+
+    FileNotFoundError = FileNotFoundError

httprunner/context.py

@@ -5,7 +5,7 @@ import os
 import re
 import sys
 
-from httprunner import exception, testcase, utils
+from httprunner import exceptions, logger, testcase, utils
 from httprunner.compat import OrderedDict
 
 
@@ -17,6 +17,7 @@ class Context(object):
         self.testset_shared_variables_mapping = OrderedDict()
         self.testcase_variables_mapping = OrderedDict()
         self.testcase_parser = testcase.TestcaseParser()
+        self.evaluated_validators = []
         self.init_context()
 
     def init_context(self, level='testset'):
@@ -201,13 +202,8 @@ class Context(object):
             # format 1/2/3
             check_value = self.eval_content(check_item)
         else:
-            try:
+            # format 4/5
-                # format 4/5
+            check_value = resp_obj.extract_field(check_item)
-                check_value = resp_obj.extract_field(check_item)
-            except exception.ParseResponseError:
-                msg = "failed to extract check item from response!\n"
-                msg += "response content: {}".format(resp_obj.content)
-                raise exception.ParseResponseError(msg)
 
         validator["check_value"] = check_value
 
@@ -227,7 +223,7 @@ class Context(object):
         validate_func = self.testcase_parser.get_bind_function(comparator)
 
         if not validate_func:
-            raise exception.FunctionNotFound("comparator not found: {}".format(comparator))
+            raise exceptions.FunctionNotFound("comparator not found: {}".format(comparator))
 
         check_item = validator_dict["check"]
         check_value = validator_dict["check_value"]
@@ -235,34 +231,56 @@ class Context(object):
 
         if (check_value is None or expect_value is None) \
             and comparator not in ["is", "eq", "equals", "=="]:
-            raise exception.ParamsError("Null value can only be compared with comparator: eq/equals/==")
+            raise exceptions.ParamsError("Null value can only be compared with comparator: eq/equals/==")
+
+        validate_msg = "validate: {} {} {}({})".format(
+            check_item,
+            comparator,
+            expect_value,
+            type(expect_value).__name__
+        )
 
         try:
-            validator_dict["check_result"] = "passed"
+            validator_dict["check_result"] = "pass"
-            validate_func(validator_dict["check_value"], validator_dict["expect"])
+            validate_func(check_value, expect_value)
+            validate_msg += "\t==> pass"
+            logger.log_debug(validate_msg)
         except (AssertionError, TypeError):
-            err_msg = "\n" + "\n".join([
+            validate_msg += "\t==> fail"
-                "\tcheck item name: %s;" % check_item,
+            validate_msg += "\n{}({}) {} {}({})".format(
-                "\tcheck item value: %s (%s);" % (check_value, type(check_value).__name__),
+                check_value,
-                "\tcomparator: %s;" % comparator,
+                type(check_value).__name__,
-                "\texpected value: %s (%s)." % (expect_value, type(expect_value).__name__)
+                comparator,
-            ])
+                expect_value,
-            validator_dict["check_result"] = "failed"
+                type(expect_value).__name__
-            raise exception.ValidationError(err_msg)
+            )
+            logger.log_error(validate_msg)
+            validator_dict["check_result"] = "fail"
+            raise exceptions.ValidationFailure(validate_msg)
 
-    def eval_validators(self, validators, resp_obj):
+    def validate(self, validators, resp_obj):
-        """ evaluate validators with context variable mapping.
+        """ make validations
         """
-        return [
+        if not validators:
-            self.eval_check_item(
+            return
+
+        logger.log_info("start to validate.")
+        self.evaluated_validators = []
+        validate_pass = True
+
+        for validator in validators:
+            # evaluate validators with context variable mapping.
+            evaluated_validator = self.eval_check_item(
                 testcase.parse_validator(validator),
                 resp_obj
             )
-            for validator in validators
-        ]
 
-    def validate(self, validators):
+            try:
-        """ make validations
+                self.do_validation(evaluated_validator)
-        """
+            except exceptions.ValidationFailure:
-        for validator_dict in validators:
+                validate_pass = False
-            self.do_validation(validator_dict)
+
+            self.evaluated_validators.append(evaluated_validator)
+
+        if not validate_pass:
+            raise exceptions.ValidationFailure

httprunner/exceptions.py

@@ -1,16 +1,24 @@
 # encoding: utf-8
 
-import json
+from httprunner.compat import JSONDecodeError, FileNotFoundError
 
-try:
+""" failure type exceptions
-    FileNotFoundError = FileNotFoundError
+    these exceptions will mark test as failure
-except NameError:
+"""
-    FileNotFoundError = IOError
 
-try:
+class MyBaseFailure(BaseException):
-    JSONDecodeError = json.decoder.JSONDecodeError
+    pass
-except AttributeError:
+
-    JSONDecodeError = ValueError
+class ValidationFailure(MyBaseFailure):
+    pass
+
+class ExtractFailure(MyBaseFailure):
+    pass
+
+
+""" error type exceptions
+    these exceptions will mark test as error
+"""
 
 class MyBaseError(BaseException):
     pass
@@ -21,18 +29,12 @@ class FileFormatError(MyBaseError):
 class ParamsError(MyBaseError):
     pass
 
-class ResponseError(MyBaseError):
-    pass
-
-class ParseResponseError(MyBaseError):
-    pass
-
-class ValidationError(MyBaseError):
-    pass
-
 class NotFoundError(MyBaseError):
     pass
 
+class FileNotFound(FileNotFoundError, NotFoundError):
+    pass
+
 class FunctionNotFound(NotFoundError):
     pass
 

httprunner/report.py

@@ -90,7 +90,9 @@ def render_html_report(summary, html_report_name=None, html_report_template=None
     if not os.path.isdir(report_dir_path):
         os.makedirs(report_dir_path)
 
-    for suite_summary in summary["details"]:
+    for index, suite_summary in enumerate(summary["details"]):
+        if not suite_summary.get("name"):
+            suite_summary["name"] = "test suite {}".format(index)
         for record in suite_summary.get("records"):
             meta_data = record['meta_data']
             stringify_data(meta_data, 'request')

httprunner/response.py

@@ -3,7 +3,7 @@
 import json
 import re
 
-from httprunner import exception, logger, testcase, utils
+from httprunner import exceptions, logger, testcase, utils
 from httprunner.compat import OrderedDict, basestring
 from requests.structures import CaseInsensitiveDict
 from requests.models import PreparedRequest
@@ -31,7 +31,7 @@ class ResponseObject(object):
         except AttributeError:
             err_msg = "ResponseObject does not have attribute: {}".format(key)
             logger.log_error(err_msg)
-            raise exception.ParamsError(err_msg)
+            raise exceptions.ParamsError(err_msg)
 
     def _extract_field_with_regex(self, field):
         """ extract field from response content with regex.
@@ -44,11 +44,10 @@ class ResponseObject(object):
         """
         matched = re.search(field, self.text)
         if not matched:
-            err_msg = u"Failed to extract data with regex!\n"
+            err_msg = u"Failed to extract data with regex! => {}\n".format(field)
-            err_msg += u"response content: {}\n".format(self.content)
+            err_msg += u"response body: {}\n".format(self.text)
-            err_msg += u"regex: {}\n".format(field)
             logger.log_error(err_msg)
-            raise exception.ParamsError(err_msg)
+            raise exceptions.ExtractFailure(err_msg)
 
         return matched.group(1)
 
@@ -63,92 +62,120 @@ class ResponseObject(object):
             "headers.content-type"
             "content.person.name.first_name"
         """
+        # string.split(sep=None, maxsplit=-1) -> list of strings
+        # e.g. "content.person.name" => ["content", "person.name"]
         try:
-            # string.split(sep=None, maxsplit=-1) -> list of strings
+            top_query, sub_query = field.split('.', 1)
-            # e.g. "content.person.name" => ["content", "person.name"]
+        except ValueError:
-            try:
+            top_query = field
-                top_query, sub_query = field.split('.', 1)
+            sub_query = None
-            except ValueError:
-                top_query = field
-                sub_query = None
-
-            if top_query == "cookies":
-                cookies = self.cookies
-                try:
-                    return cookies[sub_query]
-                except KeyError:
-                    err_msg = u"Failed to extract attribute from cookies!\n"
-                    err_msg += u"cookies: {}\n".format(cookies)
-                    err_msg += u"attribute: {}".format(sub_query)
-                    logger.log_error(err_msg)
-                    raise exception.ParamsError(err_msg)
-            elif top_query == "elapsed":
-                if sub_query in ["days", "seconds", "microseconds"]:
-                    return getattr(self.elapsed, sub_query)
-                elif sub_query == "total_seconds":
-                    return self.elapsed.total_seconds()
-                else:
-                    err_msg = "{}: {} is not valid timedelta attribute.\n".format(field, sub_query)
-                    err_msg += "elapsed only support attributes: days, seconds, microseconds, total_seconds.\n"
-                    logger.log_error(err_msg)
-                    raise exception.ParamsError(err_msg)
-
-            try:
-                top_query_content = getattr(self, top_query)
-            except AttributeError:
-                err_msg = u"Failed to extract attribute from response object: resp_obj.{}".format(top_query)
-                logger.log_error(err_msg)
-                raise exception.ParamsError(err_msg)
 
+        # status_code
+        if top_query in ["status_code", "encoding", "ok", "reason", "url"]:
             if sub_query:
-                if not isinstance(top_query_content, (dict, CaseInsensitiveDict, list)):
+                # status_code.XX
-                    try:
+                err_msg = u"Failed to extract: {}\n".format(field)
-                        # TODO: remove compatibility for content, text
+                logger.log_error(err_msg)
-                        if isinstance(top_query_content, bytes):
+                raise exceptions.ParamsError(err_msg)
-                            top_query_content = top_query_content.decode("utf-8")
 
-                        if isinstance(top_query_content, PreparedRequest):
+            return getattr(self, top_query)
-                            top_query_content = top_query_content.__dict__
-                        else:
-                            top_query_content = json.loads(top_query_content)
-                    except json.decoder.JSONDecodeError:
-                        err_msg = u"Failed to extract data with delimiter!\n"
-                        err_msg += u"response content: {}\n".format(self.content)
-                        err_msg += u"regex: {}\n".format(field)
-                        logger.log_error(err_msg)
-                        raise exception.ParamsError(err_msg)
 
-                # e.g. key: resp_headers_content_type, sub_query = "content-type"
+        # cookies
-                return utils.query_json(top_query_content, sub_query)
+        elif top_query == "cookies":
+            cookies = self.cookies.get_dict()
+            if not sub_query:
+                # extract cookies
+                return cookies
+
+            try:
+                return cookies[sub_query]
+            except KeyError:
+                err_msg = u"Failed to extract cookie! => {}\n".format(field)
+                err_msg += u"response cookies: {}\n".format(cookies)
+                logger.log_error(err_msg)
+                raise exceptions.ExtractFailure(err_msg)
+
+        # elapsed
+        elif top_query == "elapsed":
+            available_attributes = u"available attributes: days, seconds, microseconds, total_seconds"
+            if not sub_query:
+                err_msg = u"elapsed is datetime.timedelta instance, attribute should also be specified!\n"
+                err_msg += available_attributes
+                logger.log_error(err_msg)
+                raise exceptions.ParamsError(err_msg)
+            elif sub_query in ["days", "seconds", "microseconds"]:
+                return getattr(self.elapsed, sub_query)
+            elif sub_query == "total_seconds":
+                return self.elapsed.total_seconds()
             else:
-                # e.g. key: resp_status_code, resp_content
+                err_msg = "{} is not valid datetime.timedelta attribute.\n".format(sub_query)
-                return top_query_content
+                err_msg += available_attributes
+                logger.log_error(err_msg)
+                raise exceptions.ParamsError(err_msg)
 
-        except AttributeError:
+        # headers
-            err_msg = u"Failed to extract value from response!\n"
+        elif top_query == "headers":
-            err_msg += u"response content: {}\n".format(self.content)
+            headers = self.headers
-            err_msg += u"extract field: {}\n".format(field)
+            if not sub_query:
+                # extract headers
+                return headers
+
+            try:
+                return headers[sub_query]
+            except KeyError:
+                err_msg = u"Failed to extract header! => {}\n".format(field)
+                err_msg += u"response headers: {}\n".format(headers)
+                logger.log_error(err_msg)
+                raise exceptions.ExtractFailure(err_msg)
+
+        # response body
+        elif top_query in ["content", "text", "json"]:
+            try:
+                body = self.json
+            except exceptions.JSONDecodeError:
+                body = self.text
+
+            if not sub_query:
+                # extract response body
+                return body
+
+            if isinstance(body, (dict, list)):
+                # content = {"xxx": 123}, content.xxx
+                return utils.query_json(body, sub_query)
+            elif sub_query.isdigit():
+                # content = "abcdefg", content.3 => d
+                return utils.query_json(body, sub_query)
+            else:
+                # content = "<html>abcdefg</html>", content.xxx
+                err_msg = u"Failed to extract attribute from response body! => {}\n".format(field)
+                err_msg += u"response body: {}\n".format(body)
+                logger.log_error(err_msg)
+                raise exceptions.ExtractFailure(err_msg)
+
+        # others
+        else:
+            err_msg = u"Failed to extract attribute from response! => {}\n".format(field)
+            err_msg += u"available response attributes: status_code, cookies, elapsed, headers, content, text, json, encoding, ok, reason, url."
             logger.log_error(err_msg)
-            raise exception.ParamsError(err_msg)
+            raise exceptions.ParamsError(err_msg)
 
     def extract_field(self, field):
         """ extract value from requests.Response.
         """
-        msg = "extract field: {}".format(field)
+        if not isinstance(field, basestring):
+            err_msg = u"Invalid extractor! => {}\n".format(field)
+            logger.log_error(err_msg)
+            raise exceptions.ParamsError(err_msg)
 
-        try:
+        msg = "extract: {}".format(field)
-            if text_extractor_regexp_compile.match(field):
-                value = self._extract_field_with_regex(field)
-            else:
-                value = self._extract_field_with_delimiter(field)
 
-            msg += "\t=> {}".format(value)
+        if text_extractor_regexp_compile.match(field):
-            logger.log_debug(msg)
+            value = self._extract_field_with_regex(field)
+        else:
+            value = self._extract_field_with_delimiter(field)
 
-        # TODO: unify ParseResponseError type
+        msg += "\t=> {}".format(value)
-        except (exception.ParseResponseError, TypeError):
+        logger.log_debug(msg)
-            logger.log_error("failed to extract field: {}".format(field))
-            raise
 
         return value
 
@@ -171,9 +198,6 @@ class ResponseObject(object):
         extract_binds_order_dict = utils.convert_to_order_dict(extractors)
 
         for key, field in extract_binds_order_dict.items():
-            if not isinstance(field, basestring):
-                raise exception.ParamsError("invalid extractors in testcase!")
-
             extracted_variables_mapping[key] = self.extract_field(field)
 
         return extracted_variables_mapping

httprunner/runner.py

@@ -2,7 +2,7 @@
 
 from unittest.case import SkipTest
 
-from httprunner import exception, logger, response, utils
+from httprunner import exceptions, logger, response, utils
 from httprunner.client import HttpSession
 from httprunner.context import Context
 
@@ -11,7 +11,6 @@ class Runner(object):
 
     def __init__(self, config_dict=None, http_client_session=None):
         self.http_client_session = http_client_session
-        self.evaluated_validators = []
         self.context = Context()
 
         config_dict = config_dict or {}
@@ -155,7 +154,15 @@ class Runner(object):
             method = parsed_request.pop('method')
             group_name = parsed_request.pop("group", None)
         except KeyError:
-            raise exception.ParamsError("URL or METHOD missed!")
+            raise exceptions.ParamsError("URL or METHOD missed!")
+
+        # TODO: move method validation to json schema
+        valid_methods = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]
+        if method.upper() not in valid_methods:
+            err_msg = u"Invalid HTTP method! => {}\n".format(method)
+            err_msg += "Available HTTP methods: {}".format("/".join(valid_methods))
+            logger.log_error(err_msg)
+            raise exceptions.ParamsError(err_msg)
 
         logger.log_info("{method} {url}".format(method=method, url=url))
         logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_request))
@@ -183,22 +190,21 @@ class Runner(object):
         # validate
         validators = testcase_dict.get("validate", []) or testcase_dict.get("validators", [])
         try:
-            self.evaluated_validators = self.context.eval_validators(validators, resp_obj)
+            self.context.validate(validators, resp_obj)
-            self.context.validate(self.evaluated_validators)
+        except (exceptions.ParamsError, \
-        except (exception.ParamsError, exception.ResponseError, \
+                exceptions.ValidationFailure, exceptions.ExtractFailure):
-            exception.ValidationError, exception.ParseResponseError):
             # log request
             err_req_msg = "request: \n"
             err_req_msg += "headers: {}\n".format(parsed_request.pop("headers", {}))
             for k, v in parsed_request.items():
-                err_req_msg += "{}: {}\n".format(k, v)
+                err_req_msg += "{}: {}\n".format(k, repr(v))
             logger.log_error(err_req_msg)
 
             # log response
             err_resp_msg = "response: \n"
             err_resp_msg += "status_code: {}\n".format(resp_obj.status_code)
             err_resp_msg += "headers: {}\n".format(resp_obj.headers)
-            err_resp_msg += "content: {}\n".format(resp_obj.content)
+            err_resp_msg += "body: {}\n".format(repr(resp_obj.text))
             logger.log_error(err_resp_msg)
 
             raise

httprunner/task.py

@@ -4,7 +4,7 @@ import copy
 import sys
 import unittest
 
-from httprunner import exception, logger, runner, testcase, utils
+from httprunner import exceptions, logger, runner, testcase, utils
 from httprunner.compat import is_py3
 from httprunner.report import (HtmlTestResult, get_platform, get_summary,
                                render_html_report)
@@ -25,10 +25,12 @@ class TestCase(unittest.TestCase):
         """
         try:
             self.test_runner.run_test(self.testcase_dict)
+        except exceptions.MyBaseFailure as ex:
+            self.fail(repr(ex))
         finally:
             if hasattr(self.test_runner.http_client_session, "meta_data"):
                 self.meta_data = self.test_runner.http_client_session.meta_data
-                self.meta_data["validators"] = self.test_runner.evaluated_validators
+                self.meta_data["validators"] = self.test_runner.context.evaluated_validators
                 self.test_runner.http_client_session.init_meta_data()
 
 
@@ -106,7 +108,7 @@ class TestSuite(unittest.TestSuite):
                     self.testcase_parser.update_binded_variables(variables)
                     try:
                         testcase_name = self.testcase_parser.eval_content_with_bindings(testcase_dict["name"])
-                    except (AssertionError, exception.ParamsError):
+                    except (AssertionError, exceptions.ParamsError):
                         logger.log_warning("failed to eval testcase name: {}".format(testcase_dict["name"]))
                         testcase_name = testcase_dict["name"]
                     self.test_runner_list.append((test_runner, variables))
@@ -189,7 +191,7 @@ def init_test_suites(path_or_testsets, mapping=None, http_client_session=None):
     mapping = mapping or {}
 
     if not testsets:
-        raise exception.TestcaseNotFound
+        raise exceptions.TestcaseNotFound
 
     if isinstance(testsets, dict):
         testsets = [testsets]
@@ -236,7 +238,7 @@ class HttpRunner(object):
         """
         try:
             test_suite_list = init_test_suites(path_or_testsets, mapping)
-        except exception.TestcaseNotFound:
+        except exceptions.TestcaseNotFound:
             logger.log_error("Testcases not found in {}".format(path_or_testsets))
             sys.exit(1)
 
@@ -301,7 +303,7 @@ class LocustTask(object):
             for test in test_suite:
                 try:
                     test.runTest()
-                except exception.MyBaseError as ex:
+                except exceptions.MyBaseError as ex:
                     from locust.events import request_failure
                     request_failure.fire(
                         request_type=test.testcase_dict.get("request", {}).get("method"),

httprunner/templates/default_report_template.html

@@ -187,69 +187,91 @@
   {% for test_suite_summary in details %}
   {% set suite_index = loop.index %}
   <h3>{{test_suite_summary.name}}</h3>
-    <table id="suite_{{suite_index}}" class="details">
+  <table id="suite_{{suite_index}}" class="details">
-      <tr>
+    <tr>
-        <th>base_url</th>
+      <th>base_url</th>
-        <td colspan="2">{{test_suite_summary.base_url}}</td>
+      <td colspan="2">{{test_suite_summary.base_url}}</td>
-        <th colspan="2" class="detail">
+      <th colspan="2" class="detail">
-          <a class="button" href="#suite_output_{{suite_index}}">parameters & output</a>
+        <a class="button" href="#suite_output_{{suite_index}}">parameters & output</a>
-          <div id="suite_output_{{suite_index}}" class="overlay">
+        <div id="suite_output_{{suite_index}}" class="overlay">
-              <div class="popup">
-                  <h2>Parameters and Output</h2>
-                  <a class="close" href="#suite_{{suite_index}}">&times;</a>
-                  <div class="content">
-                    <div style="overflow: auto">
-                        <table>
-                          <tr>
-                            <th>variables</th>
-                            <th>output</th>
-                          </tr>
-                          {% for in_out in test_suite_summary.output %}
-                          <tr>
-                            <td>{{in_out.in}}</td>
-                            <td>{{in_out.out}}</td>
-                          </tr>
-                          {% endfor %}
-                        </table>
-                    </div>
-                  </div>
-              </div>
-          </div>
-        </td>
-      </tr>
-      <tr>
-        <td>TOTAL: {{test_suite_summary.stat.testsRun}}</td>
-        <td>SUCCESS: {{test_suite_summary.stat.successes}}</td>
-        <td>FAILED: {{test_suite_summary.stat.failures}}</td>
-        <td>ERROR: {{test_suite_summary.stat.errors}}</td>
-        <td>SKIPPED: {{test_suite_summary.stat.skipped}}</td>
-      </tr>
-      <tr>
-        <th>Status</th>
-        <th colspan="2">Name</th>
-        <th>Response Time</th>
-        <th>Detail</th>
-      </tr>
-
-      {% for record in test_suite_summary.records %}
-      {% set record_index = "{}_{}".format(suite_index, loop.index) %}
-      <tr id="record_{{record_index}}">
-        <th class="{{record.status}}" style="width:5em;">{{record.status}}</td>
-        <td colspan="2">{{record.name}}</td>
-        <td style="text-align:center;width:6em;">{{ record.meta_data.response.response_time_ms }} ms</td>
-        <td class="detail">
-
-          <a class="button" href="#popup_log_{{record_index}}">log</a>
-          <div id="popup_log_{{record_index}}" class="overlay">
             <div class="popup">
-              <h2>Request and Response data</h2>
+                <h2>Parameters and Output</h2>
-              <a class="close" href="#record_{{record_index}}">&times;</a>
+                <a class="close" href="#suite_{{suite_index}}">&times;</a>
+                <div class="content">
+                  <div style="overflow: auto">
+                      <table>
+                        <tr>
+                          <th>variables</th>
+                          <th>output</th>
+                        </tr>
+                        {% for in_out in test_suite_summary.output %}
+                        <tr>
+                          <td>{{in_out.in}}</td>
+                          <td>{{in_out.out}}</td>
+                        </tr>
+                        {% endfor %}
+                      </table>
+                  </div>
+                </div>
+            </div>
+        </div>
+      </td>
+    </tr>
+    <tr>
+      <td>TOTAL: {{test_suite_summary.stat.testsRun}}</td>
+      <td>SUCCESS: {{test_suite_summary.stat.successes}}</td>
+      <td>FAILED: {{test_suite_summary.stat.failures}}</td>
+      <td>ERROR: {{test_suite_summary.stat.errors}}</td>
+      <td>SKIPPED: {{test_suite_summary.stat.skipped}}</td>
+    </tr>
+    <tr>
+      <th>Status</th>
+      <th colspan="2">Name</th>
+      <th>Response Time</th>
+      <th>Detail</th>
+    </tr>
 
-              <div class="content">
+    {% for record in test_suite_summary.records %}
-                <h3>Request:</h3>
+    {% set record_index = "{}_{}".format(suite_index, loop.index) %}
-                <div style="overflow: auto">
+    <tr id="record_{{record_index}}">
-                  <table>
+      <th class="{{record.status}}" style="width:5em;">{{record.status}}</td>
-                    {% for key, value in record.meta_data.request.items() %}
+      <td colspan="2">{{record.name}}</td>
+      <td style="text-align:center;width:6em;">{{ record.meta_data.response.response_time_ms }} ms</td>
+      <td class="detail">
+
+        <a class="button" href="#popup_log_{{record_index}}">log</a>
+        <div id="popup_log_{{record_index}}" class="overlay">
+          <div class="popup">
+            <h2>Request and Response data</h2>
+            <a class="close" href="#record_{{record_index}}">&times;</a>
+
+            <div class="content">
+              <h3>Request:</h3>
+              <div style="overflow: auto">
+                <table>
+                  {% for key, value in record.meta_data.request.items() %}
+                    <tr>
+                      <th>{{key}}</th>
+                      <td>
+                        {% if key == "headers" %}
+                          {% for header_key, header_value in record.meta_data.request.headers.items() %}
+                          <div>
+                            <strong>{{ header_key }}</strong>: {{ header_value }}
+                          </div>
+                          {% endfor %}
+                        {% else %}
+                          {{value}}
+                        {% endif %}
+                      </td>
+                    </tr>
+                  {% endfor %}
+                </table>
+              </div>
+
+              <h3>Response:</h3>
+              <div style="overflow: auto">
+                <table>
+                    {% for key, value in record.meta_data.response.items() %}
                       <tr>
                         <th>{{key}}</th>
                         <td>
@@ -259,6 +281,12 @@
                               <strong>{{ header_key }}</strong>: {{ header_value }}
                             </div>
                             {% endfor %}
+                          {% elif key == "content_type" %}
+                            {% if value == "image" %}
+                              <img src="{{ record.meta_data.response.body }}" />
+                            {% else %}
+                              <pre>{{ record.meta_data.response.body }}</pre>
+                            {% endif %}
                           {% else %}
                             {{value}}
                           {% endif %}
@@ -266,100 +294,72 @@
                       </tr>
                     {% endfor %}
                   </table>
-                </div>
-
-                <h3>Response:</h3>
-                <div style="overflow: auto">
-                  <table>
-                      {% for key, value in record.meta_data.response.items() %}
-                        <tr>
-                          <th>{{key}}</th>
-                          <td>
-                            {% if key == "headers" %}
-                              {% for header_key, header_value in record.meta_data.request.headers.items() %}
-                              <div>
-                                <strong>{{ header_key }}</strong>: {{ header_value }}
-                              </div>
-                              {% endfor %}
-                            {% elif key == "content_type" %}
-                              {% if value == "image" %}
-                                <img src="{{ record.meta_data.response.body }}" />
-                              {% else %}
-                                <pre>{{ record.meta_data.response.body }}</pre>
-                              {% endif %}
-                            {% else %}
-                              {{value}}
-                            {% endif %}
-                          </td>
-                        </tr>
-                      {% endfor %}
-                    </table>
-                </div>
-
-                <h3>Validators:</h3>
-                <div style="overflow: auto">
-                    <table>
-                      <tr>
-                        <th>check</th>
-                        <th>comparator</th>
-                        <th>expect value</th>
-                        <th>actual value</th>
-                      </tr>
-                      {% for validator in record.meta_data.validators %}
-                      <tr>
-                        {% if validator.check_result == "passed" %}
-                        <td class="passed">
-                        {% elif validator.check_result == "failed" %}
-                        <td class="failed">
-                        {% elif validator.check_result == "unchecked" %}
-                        <td class="unchecked">
-                        {% endif %}
-                          {{validator.check}}
-                        </td>
-                        <td>{{validator.comparator}}</td>
-                        <td>{{validator.expect}}</td>
-                        <td>{{validator.check_value}}</td>
-                      </tr>
-                      {% endfor %}
-                    </table>
-                </div>
-
-                <h3>Statistics:</h3>
-                <div style="overflow: auto">
-                  <table>
-                    <tr>
-                        <th>content_size(bytes)</th>
-                        <td>{{ record.meta_data.response.content_size }}</td>
-                      </tr>
-                    <tr>
-                      <th>response_time(ms)</th>
-                      <td>{{ record.meta_data.response.response_time_ms }}</td>
-                    </tr>
-                    <tr>
-                      <th>elapsed(ms)</th>
-                      <td>{{ record.meta_data.response.elapsed_ms }}</td>
-                    </tr>
-                  </table>
-                </div>
-
               </div>
+
+              <h3>Validators:</h3>
+              <div style="overflow: auto">
+                  <table>
+                    <tr>
+                      <th>check</th>
+                      <th>comparator</th>
+                      <th>expect value</th>
+                      <th>actual value</th>
+                    </tr>
+                    {% for validator in record.meta_data.validators %}
+                    <tr>
+                      {% if validator.check_result == "pass" %}
+                      <td class="passed">
+                      {% elif validator.check_result == "fail" %}
+                      <td class="failed">
+                      {% elif validator.check_result == "unchecked" %}
+                      <td class="unchecked">
+                      {% endif %}
+                        {{validator.check}}
+                      </td>
+                      <td>{{validator.comparator}}</td>
+                      <td>{{validator.expect}}</td>
+                      <td>{{validator.check_value}}</td>
+                    </tr>
+                    {% endfor %}
+                  </table>
+              </div>
+
+              <h3>Statistics:</h3>
+              <div style="overflow: auto">
+                <table>
+                  <tr>
+                      <th>content_size(bytes)</th>
+                      <td>{{ record.meta_data.response.content_size }}</td>
+                    </tr>
+                  <tr>
+                    <th>response_time(ms)</th>
+                    <td>{{ record.meta_data.response.response_time_ms }}</td>
+                  </tr>
+                  <tr>
+                    <th>elapsed(ms)</th>
+                    <td>{{ record.meta_data.response.elapsed_ms }}</td>
+                  </tr>
+                </table>
+              </div>
+
             </div>
           </div>
+        </div>
 
-          {% if record.attachment %}
+        {% if record.attachment %}
-            <a class="button" href="#popup_attachment_{{record_index}}">traceback</a>
+          <a class="button" href="#popup_attachment_{{record_index}}">traceback</a>
-            <div id="popup_attachment_{{record_index}}" class="overlay">
+          <div id="popup_attachment_{{record_index}}" class="overlay">
-              <div class="popup">
+            <div class="popup">
-                <h2>Traceback Message</h2>
+              <h2>Traceback Message</h2>
-                <a class="close" href="#record_{{record_index}}">&times;</a>
+              <a class="close" href="#record_{{record_index}}">&times;</a>
-                <div class="content"><pre>{{ record.attachment }}</pre></div>
+              <div class="content"><pre>{{ record.attachment }}</pre></div>
-              </div>
             </div>
-          {% endif %}
+          </div>
+        {% endif %}
 
-        </td>
+      </td>
-      </tr>
+    </tr>
-    {% endfor %}
+  {% endfor %}
-    </table>
+  </table>
   {% endfor %}
 </body>

httprunner/testcase.py

@@ -9,8 +9,9 @@ import os
 import random
 import re
 
-from httprunner import exception, logger, utils
+from httprunner import exceptions, logger, utils
-from httprunner.compat import OrderedDict, basestring, numeric_types
+from httprunner.compat import (OrderedDict, basestring, builtin_str,
+                               numeric_types, str)
 from httprunner.utils import FileUtils
 
 variable_regexp = r"\$([\w_]+)"
@@ -77,7 +78,7 @@ def parse_function(content):
     """
     matched = function_regexp_compile.match(content)
     if not matched:
-        raise exception.FunctionNotFound("{} not found!".format(content))
+        raise exceptions.FunctionNotFound("{} not found!".format(content))
 
     function_meta = {
         "func_name": matched.group(1),
@@ -126,7 +127,7 @@ class TestcaseLoader(object):
         for suite_file in FileUtils.load_folder_files(suite_def_folder):
             suite = TestcaseLoader.load_test_file(suite_file)
             if "def" not in suite["config"]:
-                raise exception.ParamsError("def missed in suite file: {}!".format(suite_file))
+                raise exceptions.ParamsError("def missed in suite file: {}!".format(suite_file))
 
             call_func = suite["config"]["def"]
             function_meta = parse_function(call_func)
@@ -156,15 +157,15 @@ class TestcaseLoader(object):
         """
         api_items = FileUtils.load_file(file_path)
         if not isinstance(api_items, list):
-            raise exception.FileFormatError("API format error: {}".format(file_path))
+            raise exceptions.FileFormatError("API format error: {}".format(file_path))
 
         for api_item in api_items:
             if not isinstance(api_item, dict) or len(api_item) != 1:
-                raise exception.FileFormatError("API format error: {}".format(file_path))
+                raise exceptions.FileFormatError("API format error: {}".format(file_path))
 
             key, api_dict = api_item.popitem()
             if key != "api" or not isinstance(api_dict, dict) or "def" not in api_dict:
-                raise exception.FileFormatError("API format error: {}".format(file_path))
+                raise exceptions.FileFormatError("API format error: {}".format(file_path))
 
             api_def = api_dict.pop("def")
             function_meta = parse_function(api_def)
@@ -218,11 +219,11 @@ class TestcaseLoader(object):
         }
         for item in FileUtils.load_file(file_path):
             if not isinstance(item, dict) or len(item) != 1:
-                raise exception.FileFormatError("Testcase format error: {}".format(file_path))
+                raise exceptions.FileFormatError("Testcase format error: {}".format(file_path))
 
             key, test_block = item.popitem()
             if not isinstance(test_block, dict):
-                raise exception.FileFormatError("Testcase format error: {}".format(file_path))
+                raise exceptions.FileFormatError("Testcase format error: {}".format(file_path))
 
             if key == "config":
                 testset["config"].update(test_block)
@@ -261,7 +262,7 @@ class TestcaseLoader(object):
         def_args = block.get("function_meta").get("args", [])
 
         if len(call_args) != len(def_args):
-            raise exception.ParamsError("call args mismatch defined args!")
+            raise exceptions.ParamsError("call args mismatch defined args!")
 
         args_mapping = {}
         for index, item in enumerate(def_args):
@@ -289,10 +290,10 @@ class TestcaseLoader(object):
         if not block:
             err_msg = "{} not found!".format(name)
             if ref_type == "api":
-                raise exception.ApiNotFound(err_msg)
+                raise exceptions.ApiNotFound(err_msg)
             else:
                 # ref_type == "suite":
-                raise exception.SuiteNotFound(err_msg)
+                raise exceptions.SuiteNotFound(err_msg)
 
         return block
 
@@ -380,7 +381,7 @@ class TestcaseLoader(object):
                     testcases_list = [testset]
                 else:
                     testcases_list = []
-            except exception.FileFormatError:
+            except exceptions.FileFormatError:
                 testcases_list = []
 
         else:
@@ -407,7 +408,7 @@ def parse_validator(validator):
         }
     """
     if not isinstance(validator, dict):
-        raise exception.ParamsError("invalid validator: {}".format(validator))
+        raise exceptions.ParamsError("invalid validator: {}".format(validator))
 
     if "check" in validator and len(validator) > 1:
         # format1
@@ -418,7 +419,7 @@ def parse_validator(validator):
         elif "expected" in validator:
             expect_value = validator.get("expected")
         else:
-            raise exception.ParamsError("invalid validator: {}".format(validator))
+            raise exceptions.ParamsError("invalid validator: {}".format(validator))
 
         comparator = validator.get("comparator", "eq")
 
@@ -428,12 +429,12 @@ def parse_validator(validator):
         compare_values = validator[comparator]
 
         if not isinstance(compare_values, list) or len(compare_values) != 2:
-            raise exception.ParamsError("invalid validator: {}".format(validator))
+            raise exceptions.ParamsError("invalid validator: {}".format(validator))
 
         check_item, expect_value = compare_values
 
     else:
-        raise exception.ParamsError("invalid validator: {}".format(validator))
+        raise exceptions.ParamsError("invalid validator: {}".format(validator))
 
     return {
         "check": check_item,
@@ -627,7 +628,9 @@ def substitute_variables_with_mapping(content, mapping):
             # content is a variable
             content = value
         else:
-            content = content.replace(var, str(value))
+            if not isinstance(value, str):
+                value = builtin_str(value)
+            content = content.replace(var, value)
 
     return content
 
@@ -711,7 +714,7 @@ def parse_parameters(parameters, testset_path=None):
             # e.g. [{'app_version': '2.8.5'}, {'app_version': '2.8.6'}]
             # e.g. [{"username": "user1", "password": "111111"}, {"username": "user2", "password": "222222"}]
             if not isinstance(parsed_parameter_content, list):
-                raise exception.ParamsError("parameters syntax error!")
+                raise exceptions.ParamsError("parameters syntax error!")
 
             parameter_content_list = [
                 # get subset by parameter name
@@ -769,13 +772,13 @@ class TestcaseParser(object):
             if item_name in self.variables:
                 return self.variables[item_name]
         else:
-            raise exception.ParamsError("bind item should only be function or variable.")
+            raise exceptions.ParamsError("bind item should only be function or variable.")
 
         try:
             assert self.file_path is not None
             return utils.search_conf_item(self.file_path, item_type, item_name)
-        except (AssertionError, exception.FunctionNotFound):
+        except (AssertionError, exceptions.FunctionNotFound):
-            raise exception.ParamsError(
+            raise exceptions.ParamsError(
                 "{} is not defined in bind {}s!".format(item_name, item_type))
 
     def get_bind_function(self, func_name):
@@ -850,9 +853,12 @@ class TestcaseParser(object):
                 content = variable_value
             else:
                 # content contains one or several variables
+                if not isinstance(variable_value, str):
+                    variable_value = builtin_str(variable_value)
+
                 content = content.replace(
                     "${}".format(variable_name),
-                    str(variable_value), 1
+                    variable_value, 1
                 )
 
         return content

httprunner/utils.py

@@ -16,8 +16,8 @@ import types
 from datetime import datetime
 
 import yaml
-from httprunner import exception, logger
+from httprunner import exceptions, logger
-from httprunner.compat import OrderedDict, is_py2, is_py3
+from httprunner.compat import OrderedDict, basestring, is_py2, is_py3, str
 from requests.structures import CaseInsensitiveDict
 
 SECRET_KEY = "DebugTalk"
@@ -54,13 +54,13 @@ class FileUtils(object):
             # testcase file content is empty
             err_msg = u"Testcase file content is empty: {}".format(file_path)
             logger.log_error(err_msg)
-            raise exception.FileFormatError(err_msg)
+            raise exceptions.FileFormatError(err_msg)
 
         elif not isinstance(content, (list, dict)):
             # testcase file content does not match testcase format
             err_msg = u"Testcase file content format invalid: {}".format(file_path)
             logger.log_error(err_msg)
-            raise exception.FileFormatError(err_msg)
+            raise exceptions.FileFormatError(err_msg)
 
     @staticmethod
     def _load_yaml_file(yaml_file):
@@ -78,10 +78,10 @@ class FileUtils(object):
         with io.open(json_file, encoding='utf-8') as data_file:
             try:
                 json_content = json.load(data_file)
-            except exception.JSONDecodeError:
+            except exceptions.JSONDecodeError:
                 err_msg = u"JSONDecodeError: JSON file format error: {}".format(json_file)
                 logger.log_error(err_msg)
-                raise exception.FileFormatError(err_msg)
+                raise exceptions.FileFormatError(err_msg)
 
             FileUtils._check_format(json_file, json_content)
             return json_content
@@ -117,7 +117,7 @@ class FileUtils(object):
     @staticmethod
     def load_file(file_path):
         if not os.path.isfile(file_path):
-            raise exception.FileNotFoundError("{} does not exist.".format(file_path))
+            raise exceptions.FileNotFound("{} does not exist.".format(file_path))
 
         file_suffix = os.path.splitext(file_path)[1].lower()
         if file_suffix == '.json':
@@ -172,7 +172,7 @@ class FileUtils(object):
 
 def query_json(json_content, query, delimiter='.'):
     """ Do an xpath-like query with json_content.
-    @param (json_content) json_content
+    @param (dict/list/string) json_content
         json_content = {
             "ids": [1, 2, 3, 4],
             "person": {
@@ -186,23 +186,30 @@ def query_json(json_content, query, delimiter='.'):
         }
     @param (str) query
         "person.name.first_name"  =>  "Leo"
+        "person.name.first_name.0"  =>  "L"
         "person.cities.0"         =>  "Guangzhou"
     @return queried result
     """
-    if json_content == "":
+    raise_flag = False
-        raise exception.ResponseError("response content is empty!")
+    response_body = u"response body: {}\n".format(json_content)
-
     try:
         for key in query.split(delimiter):
-            if isinstance(json_content, list):
+            if isinstance(json_content, (list, basestring)):
                 json_content = json_content[int(key)]
-            elif isinstance(json_content, (dict, CaseInsensitiveDict)):
+            elif isinstance(json_content, dict):
                 json_content = json_content[key]
             else:
-                raise exception.ParseResponseError(
+                logger.log_error(
-                    "response content is in text format! failed to query key {}!".format(key))
+                    "invalid type value: {}({})".format(json_content, type(json_content)))
+                raise_flag = True
     except (KeyError, ValueError, IndexError):
-        raise exception.ParseResponseError("failed to query json when extracting response!")
+        raise_flag = True
+
+    if raise_flag:
+        err_msg = u"Failed to extract! => {}\n".format(query)
+        err_msg += response_body
+        logger.log_error(err_msg)
+        raise exceptions.ExtractFailure(err_msg)
 
     return json_content
 
@@ -333,9 +340,9 @@ def search_conf_item(start_path, item_type, item_name):
         # system root path
         err_msg = "{} not found in recursive upward path!".format(item_name)
         if item_type == "function":
-            raise exception.FunctionNotFound(err_msg)
+            raise exceptions.FunctionNotFound(err_msg)
         else:
-            raise exception.VariableNotFound(err_msg)
+            raise exceptions.VariableNotFound(err_msg)
 
     return search_conf_item(dir_path, item_type, item_name)
 
@@ -422,7 +429,7 @@ def override_variables_binds(variables, new_mapping):
     elif isinstance(variables, (OrderedDict, dict)):
         variables_ordered_dict = variables
     else:
-        raise exception.ParamsError("variables error!")
+        raise exceptions.ParamsError("variables error!")
 
     return update_ordered_dict(
         variables_ordered_dict,
@@ -507,7 +514,7 @@ def load_dot_env_file(path):
             return
     else:
         if not os.path.isfile(path):
-            raise exception.FileNotFoundError("env file not exist: {}".format(path))
+            raise exceptions.FileNotFound("env file not exist: {}".format(path))
 
     logger.log_info("Loading environment variables from {}".format(path))
     with io.open(path, 'r', encoding='utf-8') as fp:

tests/httpbin/hooks.yml

@@ -32,5 +32,5 @@
         - eq: ["status_code", 500]
         - eq: ["headers.content-type", "html/text"]
         - eq: [json.headers.Host, "127.0.0.1:8888"]
-        - eq: [content.headers.Host, "127.0.0.1:3458"]
+        - eq: [content.headers.Host, "127.0.0.1:8888"]
-        - eq: [text.headers.Host, "127.0.0.1:3458"]
+        - eq: [text.headers.Host, "127.0.0.1:8888"]

tests/test_context.py

@@ -2,7 +2,7 @@ import os
 import time
 
 import requests
-from httprunner import exception, response, runner, testcase
+from httprunner import exceptions, response, runner, testcase
 from httprunner.context import Context
 from httprunner.utils import FileUtils, gen_md5
 from tests.base import ApiServerUnittest
@@ -270,9 +270,8 @@ class VariableBindsUnittest(ApiServerUnittest):
         ]
         self.context.bind_variables(variables)
 
-        with self.assertRaises(exception.ValidationError):
+        with self.assertRaises(exceptions.ValidationFailure):
-            evaluated_validators = self.context.eval_validators(validators, resp_obj)
+            self.context.validate(validators, resp_obj)
-            self.context.validate(evaluated_validators)
 
         validators = [
             {"eq": ["$resp_status_code", 201]},
@@ -291,8 +290,7 @@ class VariableBindsUnittest(ApiServerUnittest):
         }
         self.context.bind_functions(functions)
 
-        evaluated_validators = self.context.eval_validators(validators, resp_obj)
+        self.context.validate(validators, resp_obj)
-        self.context.validate(evaluated_validators)
 
     def test_validate_exception(self):
         url = "http://127.0.0.1:5000/"
@@ -307,9 +305,8 @@ class VariableBindsUnittest(ApiServerUnittest):
         variables = []
         self.context.bind_variables(variables)
 
-        with self.assertRaises(exception.ParamsError):
+        with self.assertRaises(exceptions.ParamsError):
-            evaluated_validators = self.context.eval_validators(validators, resp_obj)
+            self.context.validate(validators, resp_obj)
-            self.context.validate(evaluated_validators)
 
         # expected value missed in variables mapping
         variables = [
@@ -317,6 +314,5 @@ class VariableBindsUnittest(ApiServerUnittest):
         ]
         self.context.bind_variables(variables)
 
-        with self.assertRaises(exception.ValidationError):
+        with self.assertRaises(exceptions.ValidationFailure):
-            evaluated_validators = self.context.eval_validators(validators, resp_obj)
+            self.context.validate(validators, resp_obj)
-            self.context.validate(evaluated_validators)

tests/test_httprunner.py

@@ -2,7 +2,7 @@ import os
 import shutil
 
 from httprunner import HttpRunner
-from httprunner.exception import FileNotFoundError
+from httprunner.exceptions import FileNotFound
 from tests.base import ApiServerUnittest
 
 
@@ -163,5 +163,5 @@ class TestHttpRunner(ApiServerUnittest):
         self.assertEqual(os.environ["UserName"], "debugtalk")
 
     def test_load_env_path_not_exist(self):
-        with self.assertRaises(FileNotFoundError):
+        with self.assertRaises(FileNotFound):
             HttpRunner(dot_env_path="not_exist.env").run(self.testset_path)

tests/test_response.py

@@ -1,6 +1,6 @@
 import requests
-from httprunner import exception, response, utils
+from httprunner import exceptions, response, utils
-from httprunner.compat import bytes
+from httprunner.compat import bytes, str
 from tests.base import ApiServerUnittest
 
 
@@ -27,7 +27,84 @@ class TestResponse(ApiServerUnittest):
         resp_obj = response.ResponseObject(resp)
         self.assertEqual(bytes, type(resp_obj.content))
 
-    def test_extract_response_json(self):
+    def test_extract_response_status_code(self):
+        resp = requests.get(url="http://127.0.0.1:3458/status/200")
+        resp_obj = response.ResponseObject(resp)
+
+        extract_binds_list = [
+            {"resp_status_code": "status_code"}
+        ]
+        extract_binds_dict = resp_obj.extract_response(extract_binds_list)
+
+        self.assertEqual(
+            extract_binds_dict["resp_status_code"],
+            200
+        )
+
+        extract_binds_list = [
+            {"resp_status_code": "status_code.xx"}
+        ]
+        with self.assertRaises(exceptions.ParamsError):
+            resp_obj.extract_response(extract_binds_list)
+
+    def test_extract_response_encoding_ok_reason_url(self):
+        resp = requests.get(url="http://127.0.0.1:3458/status/200")
+        resp_obj = response.ResponseObject(resp)
+
+        extract_binds_list = [
+            {"resp_encoding": "encoding"},
+            {"resp_ok": "ok"},
+            {"resp_reason": "reason"},
+            {"resp_url": "url"}
+        ]
+        extract_binds_dict = resp_obj.extract_response(extract_binds_list)
+
+        self.assertEqual(extract_binds_dict["resp_encoding"], "utf-8")
+        self.assertEqual(extract_binds_dict["resp_ok"], True)
+        self.assertEqual(extract_binds_dict["resp_reason"], "OK")
+        self.assertEqual(extract_binds_dict["resp_url"], "http://127.0.0.1:3458/status/200")
+
+        extract_binds_list = [{"resp_encoding": "encoding.xx"}]
+        with self.assertRaises(exceptions.ParamsError):
+            resp_obj.extract_response(extract_binds_list)
+
+        extract_binds_list = [{"resp_ok": "ok.xx"}]
+        with self.assertRaises(exceptions.ParamsError):
+            resp_obj.extract_response(extract_binds_list)
+
+        extract_binds_list = [{"resp_reason": "reason.xx"}]
+        with self.assertRaises(exceptions.ParamsError):
+            resp_obj.extract_response(extract_binds_list)
+
+        extract_binds_list = [{"resp_url": "url.xx"}]
+        with self.assertRaises(exceptions.ParamsError):
+            resp_obj.extract_response(extract_binds_list)
+
+    def test_extract_response_cookies(self):
+        resp = requests.get(
+            url="http://127.0.0.1:3458/cookies",
+            headers={
+                "accept": "application/json"
+            }
+        )
+        resp_obj = response.ResponseObject(resp)
+
+        extract_binds_list = [
+            {"resp_cookies": "cookies"}
+        ]
+        extract_binds_dict = resp_obj.extract_response(extract_binds_list)
+        self.assertEqual(
+            extract_binds_dict["resp_cookies"],
+            {}
+        )
+
+        extract_binds_list = [
+            {"resp_cookies": "cookies.xx"}
+        ]
+        with self.assertRaises(exceptions.ExtractFailure):
+            resp_obj.extract_response(extract_binds_list)
+
+    def test_extract_response_elapsed(self):
         resp = requests.post(
             url="http://127.0.0.1:3458/anything",
             json={
@@ -42,7 +119,68 @@ class TestResponse(ApiServerUnittest):
                 }
             }
         )
-        # resp.text
+        resp_obj = response.ResponseObject(resp)
+
+        extract_binds_list = [
+            {"resp_elapsed": "elapsed"}
+        ]
+        with self.assertRaises(exceptions.ParamsError):
+            resp_obj.extract_response(extract_binds_list)
+
+        extract_binds_list = [
+            {"resp_elapsed_microseconds": "elapsed.microseconds"},
+            {"resp_elapsed_seconds": "elapsed.seconds"},
+            {"resp_elapsed_days": "elapsed.days"},
+            {"resp_elapsed_total_seconds": "elapsed.total_seconds"}
+        ]
+        extract_binds_dict = resp_obj.extract_response(extract_binds_list)
+        self.assertGreater(extract_binds_dict["resp_elapsed_microseconds"], 1000)
+        self.assertEqual(extract_binds_dict["resp_elapsed_seconds"], 0)
+        self.assertEqual(extract_binds_dict["resp_elapsed_days"], 0)
+        self.assertGreater(extract_binds_dict["resp_elapsed_total_seconds"], 0)
+
+        extract_binds_list = [
+            {"resp_elapsed": "elapsed.years"}
+        ]
+        with self.assertRaises(exceptions.ParamsError):
+            resp_obj.extract_response(extract_binds_list)
+
+    def test_extract_response_headers(self):
+        resp = requests.get(url="http://127.0.0.1:3458/status/200")
+        resp_obj = response.ResponseObject(resp)
+
+        extract_binds_list = [
+            {"resp_headers": "headers"},
+            {"resp_headers_content_type": "headers.Content-Type"},
+            {"resp_headers_content_type_lowercase": "headers.content-type"}
+        ]
+        extract_binds_dict = resp_obj.extract_response(extract_binds_list)
+        self.assertIn("Content-Type", extract_binds_dict["resp_headers"])
+        self.assertIn("text/html", extract_binds_dict["resp_headers_content_type"])
+        self.assertIn("text/html", extract_binds_dict["resp_headers_content_type_lowercase"])
+
+        extract_binds_list = [
+            {"resp_headers_xxx": "headers.xxx"}
+        ]
+        with self.assertRaises(exceptions.ExtractFailure):
+            resp_obj.extract_response(extract_binds_list)
+
+    def test_extract_response_body_json(self):
+        resp = requests.post(
+            url="http://127.0.0.1:3458/anything",
+            json={
+                'success': False,
+                "person": {
+                    "name": {
+                        "first_name": "Leo",
+                        "last_name": "Lee",
+                    },
+                    "age": 29,
+                    "cities": ["Guangzhou", "Shenzhen"]
+                }
+            }
+        )
+        # resp.json()
         # {
         #     "args": {},
         #     "data": "{\"success\": false, \"person\": {\"name\": {\"first_name\": \"Leo\", \"last_name\": \"Lee\"}, \"age\": 29, \"cities\": [\"Guangzhou\", \"Shenzhen\"]}}",
@@ -77,7 +215,6 @@ class TestResponse(ApiServerUnittest):
         # }
 
         extract_binds_list = [
-            {"resp_status_code": "status_code"},
             {"resp_headers_content_type": "headers.content-type"},
             {"resp_content_body_success": "json.json.success"},
             {"resp_content_content_success": "content.json.success"},
@@ -88,10 +225,6 @@ class TestResponse(ApiServerUnittest):
         resp_obj = response.ResponseObject(resp)
         extract_binds_dict = resp_obj.extract_response(extract_binds_list)
 
-        self.assertEqual(
-            extract_binds_dict["resp_status_code"],
-            200
-        )
         self.assertEqual(
             extract_binds_dict["resp_headers_content_type"],
             "application/json"
@@ -117,6 +250,35 @@ class TestResponse(ApiServerUnittest):
             "Shenzhen"
         )
 
+    def test_extract_response_body_html(self):
+        resp = requests.get(url="http://127.0.0.1:3458/")
+        resp_obj = response.ResponseObject(resp)
+
+        extract_binds_list = [
+            {"resp_content": "content"}
+        ]
+        extract_binds_dict = resp_obj.extract_response(extract_binds_list)
+
+        self.assertIsInstance(extract_binds_dict["resp_content"], str)
+        self.assertIn("python-requests.org", extract_binds_dict["resp_content"])
+
+        extract_binds_list = [
+            {"resp_content": "content.xxx"}
+        ]
+        with self.assertRaises(exceptions.ExtractFailure):
+            resp_obj.extract_response(extract_binds_list)
+
+    def test_extract_response_others(self):
+        resp = requests.get(url="http://127.0.0.1:3458/status/200")
+        resp_obj = response.ResponseObject(resp)
+
+        extract_binds_list = [
+            {"resp_others_encoding": "encoding"},
+            {"resp_others_history": "history"}
+        ]
+        with self.assertRaises(exceptions.ParamsError):
+            resp_obj.extract_response(extract_binds_list)
+
     def test_extract_response_fail(self):
         resp = requests.post(
             url="http://127.0.0.1:3458/anything",
@@ -138,7 +300,7 @@ class TestResponse(ApiServerUnittest):
         ]
         resp_obj = response.ResponseObject(resp)
 
-        with self.assertRaises(exception.ParseResponseError):
+        with self.assertRaises(exceptions.ExtractFailure):
             resp_obj.extract_response(extract_binds_list)
 
         extract_binds_list = [
@@ -146,7 +308,7 @@ class TestResponse(ApiServerUnittest):
         ]
         resp_obj = response.ResponseObject(resp)
 
-        with self.assertRaises(exception.ParseResponseError):
+        with self.assertRaises(exceptions.ExtractFailure):
             resp_obj.extract_response(extract_binds_list)
 
     def test_extract_response_json_string(self):
@@ -202,7 +364,7 @@ class TestResponse(ApiServerUnittest):
             {"resp_content_key1": "LB123.*RB789"}
         ]
         resp_obj = response.ResponseObject(resp)
-        with self.assertRaises(exception.ParamsError):
+        with self.assertRaises(exceptions.ParamsError):
             resp_obj.extract_response(extract_binds_list)
 
     def test_extract_response_empty(self):
@@ -225,5 +387,5 @@ class TestResponse(ApiServerUnittest):
             {"resp_content_body": "content.data.def"}
         ]
         resp_obj = response.ResponseObject(resp)
-        with self.assertRaises(exception.ParseResponseError):
+        with self.assertRaises(exceptions.ExtractFailure):
             resp_obj.extract_response(extract_binds_list)

tests/test_runner.py

@@ -1,7 +1,7 @@
 import os
 import time
 
-from httprunner import HttpRunner, exception, runner
+from httprunner import HttpRunner, exceptions, runner
 from httprunner.testcase import TestcaseLoader
 from httprunner.utils import FileUtils, deep_update_dict
 from tests.base import ApiServerUnittest
@@ -66,7 +66,7 @@ class TestRunner(ApiServerUnittest):
             ]
         }
 
-        with self.assertRaises(exception.ValidationError):
+        with self.assertRaises(exceptions.ValidationFailure):
             self.test_runner.run_test(test)
 
     def test_run_testset_with_hooks(self):

tests/test_testcase.py

@@ -3,9 +3,7 @@ import time
 import unittest
 
 from httprunner import testcase
-from httprunner.exception import (ApiNotFound, FileFormatError,
+from httprunner.exceptions import ApiNotFound, ParamsError, SuiteNotFound
-                                  FileNotFoundError, ParamsError,
-                                  SuiteNotFound)
 from httprunner.testcase import TestcaseLoader
 
 

tests/test_utils.py

@@ -2,7 +2,7 @@ import os
 import shutil
 import unittest
 
-from httprunner import exception, utils
+from httprunner import exceptions, utils
 from httprunner.compat import OrderedDict
 from httprunner.utils import FileUtils
 from tests.base import ApiServerUnittest
@@ -16,7 +16,7 @@ class TestFileUtils(unittest.TestCase):
         with open(yaml_tmp_file, 'w') as f:
             f.write("")
 
-        with self.assertRaises(exception.FileFormatError):
+        with self.assertRaises(exceptions.FileFormatError):
             FileUtils._load_yaml_file(yaml_tmp_file)
 
         os.remove(yaml_tmp_file)
@@ -25,7 +25,7 @@ class TestFileUtils(unittest.TestCase):
         with open(yaml_tmp_file, 'w') as f:
             f.write("abc")
 
-        with self.assertRaises(exception.FileFormatError):
+        with self.assertRaises(exceptions.FileFormatError):
             FileUtils._load_yaml_file(yaml_tmp_file)
 
         os.remove(yaml_tmp_file)
@@ -37,7 +37,7 @@ class TestFileUtils(unittest.TestCase):
         with open(json_tmp_file, 'w') as f:
             f.write("")
 
-        with self.assertRaises(exception.FileFormatError):
+        with self.assertRaises(exceptions.FileFormatError):
             FileUtils._load_json_file(json_tmp_file)
 
         os.remove(json_tmp_file)
@@ -46,7 +46,7 @@ class TestFileUtils(unittest.TestCase):
         with open(json_tmp_file, 'w') as f:
             f.write("{}")
 
-        with self.assertRaises(exception.FileFormatError):
+        with self.assertRaises(exceptions.FileFormatError):
             FileUtils._load_json_file(json_tmp_file)
 
         os.remove(json_tmp_file)
@@ -55,14 +55,14 @@ class TestFileUtils(unittest.TestCase):
         with open(json_tmp_file, 'w') as f:
             f.write("abc")
 
-        with self.assertRaises(exception.FileFormatError):
+        with self.assertRaises(exceptions.FileFormatError):
             FileUtils._load_json_file(json_tmp_file)
 
         os.remove(json_tmp_file)
 
     def test_load_testcases_bad_filepath(self):
         testcase_file_path = os.path.join(os.getcwd(), 'tests/data/demo')
-        with self.assertRaises(exception.FileNotFoundError):
+        with self.assertRaises(exceptions.FileNotFound):
             FileUtils.load_file(testcase_file_path)
 
     def test_load_json_testcases(self):
@@ -163,11 +163,11 @@ class TestUtils(ApiServerUnittest):
         self.assertEqual(result, 3)
 
         query = "ids.str_key"
-        with self.assertRaises(exception.ParseResponseError):
+        with self.assertRaises(exceptions.ExtractFailure):
             utils.query_json(json_content, query)
 
         query = "ids.5"
-        with self.assertRaises(exception.ParseResponseError):
+        with self.assertRaises(exceptions.ExtractFailure):
             utils.query_json(json_content, query)
 
         query = "person.age"
@@ -175,7 +175,7 @@ class TestUtils(ApiServerUnittest):
         self.assertEqual(result, 29)
 
         query = "person.not_exist_key"
-        with self.assertRaises(exception.ParseResponseError):
+        with self.assertRaises(exceptions.ExtractFailure):
             utils.query_json(json_content, query)
 
         query = "person.cities.0"
@@ -186,16 +186,9 @@ class TestUtils(ApiServerUnittest):
         result = utils.query_json(json_content, query)
         self.assertEqual(result, "Leo")
 
-    def test_query_json_content_is_text(self):
+        query = "person.name.first_name.0"
-        json_content = ""
+        result = utils.query_json(json_content, query)
-        query = "key"
+        self.assertEqual(result, "L")
-        with self.assertRaises(exception.ResponseError):
-            utils.query_json(json_content, query)
-
-        json_content = "<html><body>content</body></html>"
-        query = "key"
-        with self.assertRaises(exception.ParseResponseError):
-            utils.query_json(json_content, query)
 
     def test_get_uniform_comparator(self):
         self.assertEqual(utils.get_uniform_comparator("eq"), "equals")
@@ -302,7 +295,7 @@ class TestUtils(ApiServerUnittest):
         self.assertIn("gen_md5", functions_dict)
         self.assertNotIn("urllib", functions_dict)
 
-        with self.assertRaises(exception.FileNotFoundError):
+        with self.assertRaises(exceptions.FileNotFoundError):
             utils.get_imported_module_from_file("tests/debugtalk2.py")
 
     def test_search_conf_function(self):
@@ -314,10 +307,10 @@ class TestUtils(ApiServerUnittest):
         self.assertTrue(utils.is_function(("_", gen_md5)))
         self.assertEqual(gen_md5("abc"), "900150983cd24fb0d6963f7d28e17f72")
 
-        with self.assertRaises(exception.FunctionNotFound):
+        with self.assertRaises(exceptions.FunctionNotFound):
             utils.search_conf_item("tests/data/subfolder/test.yml", "function", "func_not_exist")
 
-        with self.assertRaises(exception.FunctionNotFound):
+        with self.assertRaises(exceptions.FunctionNotFound):
             utils.search_conf_item("/user/local/bin", "function", "gen_md5")
 
     def test_search_conf_variable(self):
@@ -329,10 +322,10 @@ class TestUtils(ApiServerUnittest):
         self.assertTrue(utils.is_variable(("SECRET_KEY", SECRET_KEY)))
         self.assertEqual(SECRET_KEY, "DebugTalk")
 
-        with self.assertRaises(exception.VariableNotFound):
+        with self.assertRaises(exceptions.VariableNotFound):
             utils.search_conf_item("tests/data/subfolder/test.yml", "variable", "variable_not_exist")
 
-        with self.assertRaises(exception.VariableNotFound):
+        with self.assertRaises(exceptions.VariableNotFound):
             utils.search_conf_item("/user/local/bin", "variable", "SECRET_KEY")
 
     def test_is_variable(self):
@@ -443,7 +436,7 @@ class TestUtils(ApiServerUnittest):
 
         map_list = "invalid"
         override_mapping = {"a": 3, "c": 4}
-        with self.assertRaises(exception.ParamsError):
+        with self.assertRaises(exceptions.ParamsError):
             utils.override_variables_binds(map_list, override_mapping)
 
     def test_create_scaffold(self):