diff --git a/docs/quickstart.md b/docs/quickstart.md index a199f946..b14772d4 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -66,8 +66,8 @@ Open your favorite text editor and you can write test cases like this. name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} ``` As you see, each API request is described in a `test` block. And in the `request` field, it describes the detail of HTTP request, includes url, method, headers and data, which are in line with the captured traffic. @@ -123,8 +123,8 @@ To fix this problem, we should correlate `token` field in the second API test ca extract: - token: content.token validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - test: name: create user which does not exist @@ -138,8 +138,8 @@ To fix this problem, we should correlate `token` field in the second API test ca name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} ``` As you see, the `token` field is no longer hardcoded, instead it is extracted from the first API request with `extract` mechanism. In the meanwhile, it is assigned to `token` variable, which can be referenced by the subsequent API requests. @@ -205,8 +205,8 @@ And then, we can revise our demo test case and reference the functions. Suppose extract: - token: content.token validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - test: name: create user which does not exist @@ -220,8 +220,8 @@ And then, we can revise our demo test case and reference the functions. Suppose name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} ``` In this revised test case, `variable reference` and `function invoke` mechanisms are both used. @@ -269,8 +269,8 @@ To handle this case, overall `config` block is supported in `HttpRunner`. If we extract: - token: content.token validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - test: name: create user which does not exist @@ -283,8 +283,8 @@ To handle this case, overall `config` block is supported in `HttpRunner`. If we name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} ``` As you see, we define variables in `config` block. Also, we can set `base_url` in `config` block, thereby we can specify relative path in each API request url. Besides, we can also set common fields in `config` `request`, such as `device_sn` in headers. diff --git a/docs/write-testcases.rst b/docs/write-testcases.rst index b8030098..c3ea5dce 100644 --- a/docs/write-testcases.rst +++ b/docs/write-testcases.rst @@ -40,8 +40,8 @@ And here is testset example of typical scenario: get `token` at the beginning, a extract: - token: content.token validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - test: name: create user which does not exist @@ -54,8 +54,8 @@ And here is testset example of typical scenario: get `token` at the beginning, a name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} Function invoke is supported in `YAML/JSON` format testcases, such as `gen_random_string` and `get_sign` above. This mechanism relies on the `debugtak.py` hot plugin, with which we can define functions in `debugtak.py` file, and then functions can be auto discovered and invoked in runtime. @@ -68,7 +68,7 @@ Comparator ``HttpRunner`` currently supports the following comparators. +---------------------------+---------------------------+-------------------------+--------------------------+ -| comparator | Description | A(check), B(expected) | examples | +| comparator | Description | A(check), B(expect) | examples | +===========================+===========================+=========================+==========================+ | ``eq``, ``==`` | value is equal | A == B | 9 eq 9 | +---------------------------+---------------------------+-------------------------+--------------------------+ @@ -171,11 +171,11 @@ There might be slight difference on list, cos we can use index to locate list it - content_type: headers.content-type - first_name: content.person.name.first_name validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "headers.content-type", "expected": "application/json"} - - {"check": "headers.content-length", "comparator": "gt", "expected": 40} - - {"check": "content.success", "comparator": "eq", "expected": True} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "headers.content-type", "expect": "application/json"} + - {"check": "headers.content-length", "comparator": "gt", "expect": 40} + - {"check": "content.success", "comparator": "eq", "expect": True} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} diff --git a/examples/quickstart-demo-rev-0.yml b/examples/quickstart-demo-rev-0.yml index 5580b17d..63dadbd6 100644 --- a/examples/quickstart-demo-rev-0.yml +++ b/examples/quickstart-demo-rev-0.yml @@ -11,8 +11,8 @@ json: sign: 19067cf712265eb5426db8d3664026c1ccea02b9 validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - test: name: create user which does not exist @@ -26,5 +26,5 @@ name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} diff --git a/examples/quickstart-demo-rev-1.yml b/examples/quickstart-demo-rev-1.yml index 0610615a..7bd3446d 100644 --- a/examples/quickstart-demo-rev-1.yml +++ b/examples/quickstart-demo-rev-1.yml @@ -13,8 +13,8 @@ extract: - token: content.token validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - test: name: create user which does not exist @@ -28,5 +28,5 @@ name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} diff --git a/examples/quickstart-demo-rev-2.yml b/examples/quickstart-demo-rev-2.yml index c41a37f8..f8e2ccf9 100644 --- a/examples/quickstart-demo-rev-2.yml +++ b/examples/quickstart-demo-rev-2.yml @@ -18,8 +18,8 @@ extract: - token: content.token validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - test: name: create user which does not exist @@ -33,5 +33,5 @@ name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} \ No newline at end of file + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} \ No newline at end of file diff --git a/examples/quickstart-demo-rev-3.yml b/examples/quickstart-demo-rev-3.yml index 952d2a59..7038d8e6 100644 --- a/examples/quickstart-demo-rev-3.yml +++ b/examples/quickstart-demo-rev-3.yml @@ -25,8 +25,8 @@ extract: - token: content.token validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - test: name: create user which does not exist @@ -39,5 +39,5 @@ name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} diff --git a/httprunner/response.py b/httprunner/response.py index d39be0b1..b59feae6 100644 --- a/httprunner/response.py +++ b/httprunner/response.py @@ -128,8 +128,8 @@ class ResponseObject(object): """ Bind named validators to value within the context. @param (list) validators [ - {"check": "status_code", "comparator": "eq", "expected": 201}, - {"check": "resp_body_success", "comparator": "eq", "expected": True} + {"check": "status_code", "comparator": "eq", "expect": 201}, + {"check": "resp_body_success", "comparator": "eq", "expect": True} ] @param (dict) variables_mapping { @@ -139,7 +139,7 @@ class ResponseObject(object): [ { "check": "status_code", - "comparator": "eq", "expected": 201, "value": 200 + "comparator": "eq", "expect": 201, "value": 200 } ] """ @@ -147,12 +147,15 @@ class ResponseObject(object): check_item = validator_dict.get("check") if not check_item: - raise exception.ParamsError("invalid check item in testcase validators!") + raise exception.ParamsError("check item invalid: {}".format(check_item)) - if "expected" not in validator_dict: - raise exception.ParamsError("expected item missed in testcase validators!") + if "expect" in validator_dict: + expect_value = validator_dict.get("expect") + elif "expected" in validator_dict: + expect_value = validator_dict.get("expected") + else: + raise exception.ParamsError("expected value missed in testcase validator!") - expected = validator_dict.get("expected") comparator = validator_dict.get("comparator", "eq") if check_item in variables_mapping: @@ -165,7 +168,7 @@ class ResponseObject(object): utils.match_expected( validator_dict["actual_value"], - expected, + expect_value, comparator, check_item ) diff --git a/tests/api/demo.yml b/tests/api/demo.yml index d256bc68..cd163775 100644 --- a/tests/api/demo.yml +++ b/tests/api/demo.yml @@ -11,8 +11,8 @@ json: sign: ${get_sign($user_agent, $device_sn, $os_platform, $app_version)} validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - api: def: create_user($uid, $user_name, $user_password, $token) diff --git a/tests/data/demo_testset_hardcode.json b/tests/data/demo_testset_hardcode.json index 08d72261..df7b220c 100644 --- a/tests/data/demo_testset_hardcode.json +++ b/tests/data/demo_testset_hardcode.json @@ -22,8 +22,8 @@ } ], "validate": [ - {"check": "status_code", "comparator": "eq", "expected": 200}, - {"check": "content.token", "comparator": "len_eq", "expected": 16} + {"check": "status_code", "comparator": "eq", "expect": 200}, + {"check": "content.token", "comparator": "len_eq", "expect": 16} ] } }, @@ -44,8 +44,8 @@ } }, "validate": [ - {"check": "status_code", "comparator": "eq", "expected": 201}, - {"check": "content.success", "comparator": "eq", "expected": true} + {"check": "status_code", "comparator": "eq", "expect": 201}, + {"check": "content.success", "comparator": "eq", "expect": true} ] } }, @@ -66,8 +66,8 @@ } }, "validate": [ - {"check": "status_code", "comparator": "eq", "expected": 500}, - {"check": "content.success", "comparator": "eq", "expected": false} + {"check": "status_code", "comparator": "eq", "expect": 500}, + {"check": "content.success", "comparator": "eq", "expect": false} ] } } diff --git a/tests/data/demo_testset_hardcode.yml b/tests/data/demo_testset_hardcode.yml index 7646e86f..fd869cb9 100644 --- a/tests/data/demo_testset_hardcode.yml +++ b/tests/data/demo_testset_hardcode.yml @@ -14,8 +14,8 @@ extract: - token: content.token validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - test: name: create user which does not exist @@ -30,8 +30,8 @@ name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} - test: name: create user which existed @@ -46,5 +46,5 @@ name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 500} - - {"check": "content.success", "comparator": "eq", "expected": false} \ No newline at end of file + - {"check": "status_code", "comparator": "eq", "expect": 500} + - {"check": "content.success", "comparator": "eq", "expect": false} \ No newline at end of file diff --git a/tests/data/demo_testset_layer.yml b/tests/data/demo_testset_layer.yml index 50328b6f..1b70b170 100644 --- a/tests/data/demo_testset_layer.yml +++ b/tests/data/demo_testset_layer.yml @@ -23,15 +23,15 @@ name: reset all users api: reset_all($token) validate: - - {"check": "status_code", "expected": 200} - - {"check": "content.success", "expected": true} + - {"check": "status_code", "expect": 200} + - {"check": "content.success", "expect": true} - test: name: get user that does not exist api: get_user(1000, $token) validate: - - {"check": "status_code", "expected": 404} - - {"check": "content.success", "expected": false} + - {"check": "status_code", "expect": 404} + - {"check": "content.success", "expect": false} - test: name: create user which does not exist @@ -40,16 +40,16 @@ - user_password: "123456" api: create_user(1000, $user_name, $user_password, $token) validate: - - {"check": "status_code", "expected": 201} - - {"check": "content.success", "expected": true} + - {"check": "status_code", "expect": 201} + - {"check": "content.success", "expect": true} - test: name: get user that has been created api: get_user(1000, $token) validate: - - {"check": "status_code", "expected": 200} - - {"check": "content.success", "expected": true} - - {"check": "content.data.password", "expected": "123456"} + - {"check": "status_code", "expect": 200} + - {"check": "content.success", "expect": true} + - {"check": "content.data.password", "expect": "123456"} - test: name: create user which exists @@ -58,8 +58,8 @@ - user_password: "123456" api: create_user(1000, $user_name, $user_password, $token) validate: - - {"check": "status_code", "expected": 500} - - {"check": "content.success", "expected": false} + - {"check": "status_code", "expect": 500} + - {"check": "content.success", "expect": false} - test: name: update user which exists @@ -68,37 +68,37 @@ - user_password: "654321" api: update_user(1000, $user_name, $user_password, $token) validate: - - {"check": "status_code", "expected": 200} - - {"check": "content.success", "expected": true} + - {"check": "status_code", "expect": 200} + - {"check": "content.success", "expect": true} - test: name: get user that has been updated api: get_user(1000, $token) validate: - - {"check": "status_code", "expected": 200} - - {"check": "content.success", "expected": true} - - {"check": "content.data.password", "expected": "654321"} + - {"check": "status_code", "expect": 200} + - {"check": "content.success", "expect": true} + - {"check": "content.data.password", "expect": "654321"} - test: name: get users api: get_users($token) validate: - - {"check": "status_code", "expected": 200} - - {"check": "content.count", "expected": 1} + - {"check": "status_code", "expect": 200} + - {"check": "content.count", "expect": 1} - test: name: delete user that exists api: delete_user(1000, $token) validate: - - {"check": "status_code", "expected": 200} - - {"check": "content.success", "expected": true} + - {"check": "status_code", "expect": 200} + - {"check": "content.success", "expect": true} - test: name: get users api: get_users($token) validate: - - {"check": "status_code", "expected": 200} - - {"check": "content.count", "expected": 0} + - {"check": "status_code", "expect": 200} + - {"check": "content.count", "expect": 0} - test: name: create user which has been deleted @@ -107,12 +107,12 @@ - user_password: "123456" api: create_user(1000, $user_name, $user_password, $token) validate: - - {"check": "status_code", "expected": 201} - - {"check": "content.success", "expected": true} + - {"check": "status_code", "expect": 201} + - {"check": "content.success", "expect": true} - test: name: get users api: get_users($token) validate: - - {"check": "status_code", "expected": 200} - - {"check": "content.count", "expected": 1} + - {"check": "status_code", "expect": 200} + - {"check": "content.count", "expect": 1} diff --git a/tests/data/demo_testset_template_import_functions.yml b/tests/data/demo_testset_template_import_functions.yml index d786ecdf..05a6806b 100644 --- a/tests/data/demo_testset_template_import_functions.yml +++ b/tests/data/demo_testset_template_import_functions.yml @@ -28,8 +28,8 @@ extract: - token: content.token validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - test: name: create user which does not exist @@ -45,8 +45,8 @@ name: $user_name password: $user_password validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} - test: name: create user which does not exist @@ -59,5 +59,5 @@ name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 500} - - {"check": "content.success", "comparator": "eq", "expected": false} + - {"check": "status_code", "comparator": "eq", "expect": 500} + - {"check": "content.success", "comparator": "eq", "expect": false} diff --git a/tests/data/demo_testset_template_lambda_functions.yml b/tests/data/demo_testset_template_lambda_functions.yml index 98321784..2610161c 100644 --- a/tests/data/demo_testset_template_lambda_functions.yml +++ b/tests/data/demo_testset_template_lambda_functions.yml @@ -38,8 +38,8 @@ extract: - token: content.token validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - test: name: create user which does not exist @@ -55,8 +55,8 @@ name: $user_name password: $user_password validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} - test: name: create user which does not exist @@ -69,5 +69,5 @@ name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 500} - - {"check": "content.success", "comparator": "eq", "expected": false} + - {"check": "status_code", "comparator": "eq", "expect": 500} + - {"check": "content.success", "comparator": "eq", "expect": false} diff --git a/tests/data/demo_testset_variables.yml b/tests/data/demo_testset_variables.yml index 1efd6bb9..987e3c45 100644 --- a/tests/data/demo_testset_variables.yml +++ b/tests/data/demo_testset_variables.yml @@ -29,8 +29,8 @@ extract: - token: content.token validate: - - {"check": "status_code", "comparator": "eq", "expected": 200} - - {"check": "content.token", "comparator": "len_eq", "expected": 16} + - {"check": "status_code", "comparator": "eq", "expect": 200} + - {"check": "content.token", "comparator": "len_eq", "expect": 16} - test: name: create user which does not exist @@ -46,8 +46,8 @@ name: $user_name password: $user_password validate: - - {"check": "status_code", "comparator": "eq", "expected": 201} - - {"check": "content.success", "comparator": "eq", "expected": true} + - {"check": "status_code", "comparator": "eq", "expect": 201} + - {"check": "content.success", "comparator": "eq", "expect": true} - test: name: create user which does not exist @@ -60,5 +60,5 @@ name: "user1" password: "123456" validate: - - {"check": "status_code", "comparator": "eq", "expected": 500} - - {"check": "content.success", "comparator": "eq", "expected": false} + - {"check": "status_code", "comparator": "eq", "expect": 500} + - {"check": "content.success", "comparator": "eq", "expect": false} diff --git a/tests/test_response.py b/tests/test_response.py index 97680553..3dd79ccd 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -232,8 +232,8 @@ class TestResponse(ApiServerUnittest): resp_obj = response.ResponseObject(resp) validators = [ - {"check": "resp_status_code", "comparator": "eq", "expected": 201}, - {"check": "resp_body_success", "comparator": "eq", "expected": True} + {"check": "resp_status_code", "comparator": "eq", "expect": 201}, + {"check": "resp_body_success", "comparator": "eq", "expect": True} ] variables_mapping = { "resp_status_code": 200, @@ -244,8 +244,8 @@ class TestResponse(ApiServerUnittest): resp_obj.validate(validators, variables_mapping) validators = [ - {"check": "resp_status_code", "comparator": "eq", "expected": 201}, - {"check": "resp_body_success", "comparator": "eq", "expected": True} + {"check": "resp_status_code", "comparator": "eq", "expect": 201}, + {"check": "resp_body_success", "comparator": "eq", "expect": True} ] variables_mapping = { "resp_status_code": 201, @@ -261,7 +261,7 @@ class TestResponse(ApiServerUnittest): # expected value missed in validators validators = [ - {"check": "status_code", "comparator": "eq", "expected": 201}, + {"check": "status_code", "comparator": "eq", "expect": 201}, {"check": "body_success", "comparator": "eq"} ] variables_mapping = {} @@ -270,7 +270,7 @@ class TestResponse(ApiServerUnittest): # expected value missed in variables mapping validators = [ - {"check": "resp_status_code", "comparator": "eq", "expected": 201}, + {"check": "resp_status_code", "comparator": "eq", "expect": 201}, {"check": "body_success", "comparator": "eq"} ] variables_mapping = { diff --git a/tests/test_runner.py b/tests/test_runner.py index d8f3d82e..ad530fa2 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -56,8 +56,8 @@ class TestRunner(ApiServerUnittest): {"token": "content.token"} ], "validate": [ - {"check": "status_code", "comparator": "eq", "expected": 205}, - {"check": "content.token", "comparator": "len_eq", "expected": 19} + {"check": "status_code", "expect": 205}, + {"check": "content.token", "comparator": "len_eq", "expect": 19} ] }