2.4.0

feat: validate with python script, ref #773

docs/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release History
 
+## 2.4.0 (2019-12-04)
+
+**Added**
+
+- feat: validate with python script, ref #773
+
 ## 2.3.3 (2019-12-04)
 
 **Fixed**

httprunner/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.3.3"
+__version__ = "2.4.0"
 __description__ = "One-stop solution for HTTP(S) testing."
 
 __all__ = ["__version__", "__description__"]

httprunner/context.py

@@ -13,11 +13,13 @@ class SessionContext(object):
         >>> context.update_session_variables(variables)
 
     """
+
     def __init__(self, variables=None):
         variables_mapping = utils.ensure_mapping_format(variables or {})
         self.session_variables_mapping = parser.parse_variables_mapping(variables_mapping)
+        self.test_variables_mapping = {}
         self.init_test_variables()
-        self.validation_results = []
+        self.validation_results = {}
 
     def init_test_variables(self, variables_mapping=None):
         """ init test variables, called when each test(api) starts.
@@ -77,7 +79,7 @@ class SessionContext(object):
 
         """
         if isinstance(check_item, (dict, list)) \
-            or isinstance(check_item, parser.LazyString):
+                or isinstance(check_item, parser.LazyString):
             # format 1/2/3
             check_value = self.eval_content(check_item)
         else:
@@ -101,7 +103,7 @@ class SessionContext(object):
     def validate(self, validators, resp_obj):
         """ make validation with comparators
         """
-        self.validation_results = []
+        self.validation_results = {}
         if not validators:
             return
 
@@ -111,6 +113,19 @@ class SessionContext(object):
         failures = []
 
         for validator in validators:
+
+            if isinstance(validator, dict) and validator.get("type") == "python_script":
+                validator_dict, ex = self.validate_script(validator["script"], resp_obj)
+                if ex:
+                    validate_pass = False
+                    failures.append(ex)
+
+                self.validation_results["validate_script"] = validator_dict
+                continue
+
+            if "validate_extractor" not in self.validation_results:
+                self.validation_results["validate_extractor"] = []
+
             # validator should be LazyFunction object
             if not isinstance(validator, parser.LazyFunction):
                 raise exceptions.ValidationFailure(
@@ -160,7 +175,7 @@ class SessionContext(object):
                 logger.log_error(validate_msg)
                 failures.append(validate_msg)
 
-            self.validation_results.append(validator_dict)
+            self.validation_results["validate_extractor"].append(validator_dict)
 
             # restore validator args, in case of running multiple times
             validator.update_args(validator_args)
@@ -168,3 +183,55 @@ class SessionContext(object):
         if not validate_pass:
             failures_string = "\n".join([failure for failure in failures])
             raise exceptions.ValidationFailure(failures_string)
+
+    def validate_script(self, script, resp_obj):
+        """ make validation with python script
+        """
+        validator_dict = {
+            "validate_script": "<br/>".join(script),
+            "check_result": "fail",
+            "exception": ""
+        }
+
+        script = "\n    ".join(script)
+        code = f"""
+# encoding: utf-8
+
+try:
+    {script}
+except Exception as ex:
+    import traceback
+    import sys
+    _type, _value, _tb = sys.exc_info()
+    # filename, lineno, name, line
+    _, _lineno, _, line_content = traceback.extract_tb(_tb, 1)[0]
+
+    line_no = _lineno - 4
+
+    c_exception = _type.__name__ + "\\n"
+    c_exception += "\\tError line number: " + str(line_no) + "\\n"
+    c_exception += "\\tError line content: " + str(line_content) + "\\n"
+
+    if _value.args:
+        c_exception += "\\tError description: " + str(_value)
+    else:
+        c_exception += "\\tError description: " + _type.__name__
+
+    raise _type(c_exception)
+"""
+        variables = {
+            "status_code": resp_obj.status_code,
+            "response_json": resp_obj.json,
+            "response": resp_obj
+        }
+        variables.update(self.test_variables_mapping)
+
+        try:
+            code = compile(code, '<string>', 'exec')
+            exec(code, variables)
+            validator_dict["check_result"] = "pass"
+            return validator_dict, ""
+        except Exception as ex:
+            validator_dict["check_result"] = "fail"
+            validator_dict["exception"] = "<br/>".join(str(ex).splitlines())
+            return validator_dict, str(ex)

httprunner/runner.py

@@ -62,7 +62,6 @@ class Runner(object):
         """
         self.verify = config.get("verify", True)
         self.export = config.get("export") or config.get("output", [])
-        self.validation_results = []
         config_variables = config.get("variables", {})
 
         # testcase setup hooks
@@ -86,7 +85,6 @@ class Runner(object):
         if not isinstance(self.http_client_session, HttpSession):
             return
 
-        self.validation_results = []
         self.http_client_session.init_meta_data()
 
     def __get_test_data(self):
@@ -96,7 +94,7 @@ class Runner(object):
             return
 
         meta_data = self.http_client_session.meta_data
-        meta_data["validators"] = self.validation_results
+        meta_data["validators"] = self.session_context.validation_results
         return meta_data
 
     def _handle_skip_feature(self, test_dict):
@@ -244,7 +242,8 @@ class Runner(object):
             raise exceptions.ParamsError(err_msg)
 
         logger.log_info("{method} {url}".format(method=method, url=parsed_url))
-        logger.log_debug("request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request))
+        logger.log_debug(
+            "request kwargs(raw): {kwargs}".format(kwargs=parsed_test_request))
 
         # request
         resp = self.http_client_session.request(
@@ -268,10 +267,19 @@ class Runner(object):
         self.session_context.update_session_variables(extracted_variables_mapping)
 
         # validate
+        # TODO: split validate from context
         validators = test_dict.get("validate") or test_dict.get("validators") or []
+        validate_script = test_dict.get("validate_script", [])
+        if validate_script:
+            validators.append({
+                "type": "python_script",
+                "script": validate_script
+            })
+
         try:
             self.session_context.validate(validators, resp_obj)
-        except (exceptions.ParamsError, exceptions.ValidationFailure, exceptions.ExtractFailure):
+        except (exceptions.ParamsError,
+                exceptions.ValidationFailure, exceptions.ExtractFailure):
             err_msg = "{} DETAILED REQUEST & RESPONSE {}\n".format("*" * 32, "*" * 32)
 
             # log request
@@ -294,9 +302,6 @@ class Runner(object):
 
             raise
 
-        finally:
-            self.validation_results = self.session_context.validation_results
-
     def _run_testcase(self, testcase_dict):
         """ run single testcase.
         """

httprunner/static/report_template.html

@@ -279,15 +279,17 @@
               {% endfor %}
 
               <h3>Validators:</h3>
-              <div style="overflow: auto">
+                <div style="overflow: auto">
-                  <table>
+                {% set validate_extractors = meta_data.validators.validate_extractor %}
+                {% if validate_extractors %}
+                <table>
                     <tr>
                       <th>check</th>
                       <th>comparator</th>
                       <th>expect value</th>
                       <th>actual value</th>
                     </tr>
-                    {% for validator in meta_data.validators %}
+                    {% for validator in validate_extractors %}
                     <tr>
                       {% if validator.check_result == "pass" %}
                       <td class="passed">
@@ -303,7 +305,27 @@
                       <td>{{validator.check_value | e}}</td>
                     </tr>
                     {% endfor %}
-                  </table>
+                </table>
+                {% endif %}
+
+                {% set validate_script = meta_data.validators.validate_script %}
+                {% if validate_script %}
+                    <table>
+                        <tr>
+                            <th>validate script</th><th>exception</th>
+                        </tr>
+                        <tr>
+                            <td>{{validate_script.validate_script | safe}}</td>
+                            {% if validate_script.check_result == "pass" %}
+                            <td class="passed">
+                            {% elif validate_script.check_result == "fail" %}
+                            <td class="failed">
+                            {% endif %}
+                            {{validate_script.exception}}
+                            </td>
+                        </tr>
+                    </table>
+                {% endif %}
               </div>
 
               <h3>Statistics:</h3>

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "httprunner"
-version = "2.3.3"
+version = "2.4.0"
 description = "One-stop solution for HTTP(S) testing."
 license = "Apache-2.0"
 readme = "README.md"

tests/httpbin/validate.yml

@@ -1,13 +1,34 @@
 - config:
     name: basic test with httpbin
-    request:
+    base_url: http://httpbin.org/
-        base_url: http://httpbin.org/
 
 - test:
-    name: headers
+    name: validate response with json path
     request:
-        url: /headers
+        url: /get
+        params:
+            a: 1
+            b: 2
         method: GET
     validate:
         - eq: ["status_code", 200]
-        - assert_status_code_is_200: ["status_code"]
+        - eq: ["json.args.a", '1']
+        - eq: ["json.args.b", '2']
+    validate_script:
+        - "assert status_code == 200"
+
+
+- test:
+    name: validate response with python script
+    request:
+        url: /get
+        params:
+            a: 1
+            b: 2
+        method: GET
+    validate:
+        - eq: ["status_code", 200]
+    validate_script:
+        - "assert status_code == 201"
+        - "a = response_json.get('args').get('a')"
+        - "assert a == '1'"

tests/test_api.py

@@ -289,6 +289,10 @@ class TestHttpRunner(ApiServerUnittest):
         self.assertEqual(summary["stat"]["testcases"]["total"], 2)
         self.assertEqual(summary["stat"]["teststeps"]["total"], 4)
 
+    def test_validate_script(self):
+        summary = self.runner.run("tests/httpbin/validate.yml")
+        self.assertFalse(summary["success"])
+
     def test_run_httprunner_with_hooks(self):
         testcase_file_path = os.path.join(
             os.getcwd(), 'tests/httpbin/hooks.yml')