From 37be5c6060ac8fe183507ed70931914c651df605 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 5 Apr 2022 13:13:12 +0800 Subject: [PATCH 1/2] fix #1220: parse step url with base url --- hrp/parser.go | 20 ++++++++++++++++---- hrp/parser_test.go | 35 +++++++++++++++++++++++++++-------- httprunner/parser.py | 25 +++++++++++++++++-------- httprunner/parser_test.py | 27 ++++++++++++++++++++++++++- 4 files changed, 86 insertions(+), 21 deletions(-) diff --git a/hrp/parser.go b/hrp/parser.go index 3a7ea231..1e5c6b07 100644 --- a/hrp/parser.go +++ b/hrp/parser.go @@ -4,6 +4,7 @@ import ( builtinJSON "encoding/json" "fmt" "net/url" + "path" "reflect" "regexp" "strings" @@ -26,18 +27,29 @@ type Parser struct { } func buildURL(baseURL, stepURL string) string { - uConfig, err := url.Parse(baseURL) + uStep, err := url.Parse(stepURL) if err != nil { - log.Error().Str("baseURL", baseURL).Err(err).Msg("[buildURL] parse baseURL failed") + log.Error().Str("stepURL", stepURL).Err(err).Msg("[buildURL] parse url failed") return "" } - uStep, err := uConfig.Parse(stepURL) + // step url is absolute url + if uStep.Host != "" { + return stepURL + } + + // step url is relative, based on base url + uConfig, err := url.Parse(baseURL) if err != nil { - log.Error().Str("stepURL", stepURL).Err(err).Msg("[buildURL] parse stepURL failed") + log.Error().Str("baseURL", baseURL).Err(err).Msg("[buildURL] parse url failed") return "" } + // merge url + uStep.Scheme = uConfig.Scheme + uStep.Host = uConfig.Host + uStep.Path = path.Join(uConfig.Path, uStep.Path) + // base url missed return uStep.String() } diff --git a/hrp/parser_test.go b/hrp/parser_test.go index 64ba38d4..41b376b8 100644 --- a/hrp/parser_test.go +++ b/hrp/parser_test.go @@ -11,25 +11,44 @@ import ( func TestBuildURL(t *testing.T) { var url string + url = buildURL("https://postman-echo.com", "/get") - if url != "https://postman-echo.com/get" { - t.Fatalf("buildURL error, %s != 'https://postman-echo.com/get'", url) + if !assert.Equal(t, url, "https://postman-echo.com/get") { + t.Fail() + } + url = buildURL("https://postman-echo.com", "get") + if !assert.Equal(t, url, "https://postman-echo.com/get") { + t.Fail() + } + url = buildURL("https://postman-echo.com/", "/get") + if !assert.Equal(t, url, "https://postman-echo.com/get") { + t.Fail() } url = buildURL("https://postman-echo.com/abc/", "/get?a=1&b=2") - if url != "https://postman-echo.com/get?a=1&b=2" { - t.Fatalf("buildURL error, %s != 'https://postman-echo.com/get'", url) + if !assert.Equal(t, url, "https://postman-echo.com/abc/get?a=1&b=2") { + t.Fail() + } + url = buildURL("https://postman-echo.com/abc", "get?a=1&b=2") + if !assert.Equal(t, url, "https://postman-echo.com/abc/get?a=1&b=2") { + t.Fail() + } + + // omit query string in base url + url = buildURL("https://postman-echo.com/abc?x=6&y=9", "/get?a=1&b=2") + if !assert.Equal(t, url, "https://postman-echo.com/abc/get?a=1&b=2") { + t.Fail() } url = buildURL("", "https://postman-echo.com/get") - if url != "https://postman-echo.com/get" { - t.Fatalf("buildURL error, %s != 'https://postman-echo.com/get'", url) + if !assert.Equal(t, url, "https://postman-echo.com/get") { + t.Fail() } // notice: step request url > config base url url = buildURL("https://postman-echo.com", "https://httpbin.org/get") - if url != "https://httpbin.org/get" { - t.Fatalf("buildURL error, %s != 'https://httpbin.org/get'", url) + if !assert.Equal(t, url, "https://httpbin.org/get") { + t.Fail() } } diff --git a/httprunner/parser.py b/httprunner/parser.py index 9f32d721..192521da 100644 --- a/httprunner/parser.py +++ b/httprunner/parser.py @@ -3,6 +3,7 @@ import builtins import os import re from typing import Any, Callable, Dict, List, Set, Text +from urllib.parse import urljoin, urlparse from loguru import logger from sentry_sdk import capture_exception @@ -10,8 +11,6 @@ from sentry_sdk import capture_exception from httprunner import exceptions, loader, utils from httprunner.models import FunctionsMapping, VariablesMapping -absolute_http_url_regexp = re.compile(r"^https?://", re.I) - # use $$ to escape $ notation dolloar_regex_compile = re.compile(r"\$\$") # variable notation, e.g. ${var} or $var @@ -37,15 +36,25 @@ def parse_string_value(str_value: Text) -> Any: return str_value -def build_url(base_url, path): +def build_url(base_url, step_url): """ prepend url with base_url unless it's already an absolute URL """ - if absolute_http_url_regexp.match(path): - return path - elif base_url: - return "{}/{}".format(base_url.rstrip("/"), path.lstrip("/")) - else: + o_step_url = urlparse(step_url) + if o_step_url.netloc != "": + # step url is absolute url + return step_url + + # step url is relative, based on base url + o_base_url = urlparse(base_url) + if o_base_url.netloc == "": + # missed base url raise exceptions.ParamsError("base url missed!") + path = o_base_url.path.rstrip("/") + "/" + o_step_url.path.lstrip("/") + o_step_url = o_step_url._replace(scheme=o_base_url.scheme) \ + ._replace(netloc=o_base_url.netloc) \ + ._replace(path=path) + return o_step_url.geturl() + def regex_findall_variables(raw_string: Text) -> List[Text]: """ extract all variable names from content, which is in format $variable diff --git a/httprunner/parser_test.py b/httprunner/parser_test.py index 2a77af59..eaed0f78 100644 --- a/httprunner/parser_test.py +++ b/httprunner/parser_test.py @@ -3,11 +3,36 @@ import time import unittest from httprunner import parser -from httprunner.exceptions import VariableNotFound, FunctionNotFound +from httprunner.exceptions import FunctionNotFound, VariableNotFound from httprunner.loader import load_project_meta class TestParserBasic(unittest.TestCase): + + def test_build_url(self): + url = parser.build_url("https://postman-echo.com", "/get") + self.assertEqual(url, "https://postman-echo.com/get") + url = parser.build_url("https://postman-echo.com", "get") + self.assertEqual(url, "https://postman-echo.com/get") + url = parser.build_url("https://postman-echo.com/", "/get") + self.assertEqual(url, "https://postman-echo.com/get") + + url = parser.build_url("https://postman-echo.com/abc/", "/get?a=1&b=2") + self.assertEqual(url, "https://postman-echo.com/abc/get?a=1&b=2") + url = parser.build_url("https://postman-echo.com/abc/", "get?a=1&b=2") + self.assertEqual(url, "https://postman-echo.com/abc/get?a=1&b=2") + + # omit query string in base url + url = parser.build_url("https://postman-echo.com/abc?x=6&y=9", "/get?a=1&b=2") + self.assertEqual(url, "https://postman-echo.com/abc/get?a=1&b=2") + + url = parser.build_url("", "https://postman-echo.com/get") + self.assertEqual(url, "https://postman-echo.com/get") + + # notice: step request url > config base url + url = parser.build_url("https://postman-echo.com", "https://httpbin.org/get") + self.assertEqual(url, "https://httpbin.org/get") + def test_parse_variables_mapping(self): variables = {"varA": "$varB", "varB": "$varC", "varC": "123", "a": 1, "b": 2} parsed_variables = parser.parse_variables_mapping(variables) From 6d11f1264f130741637a33b9c0f9e2efa076f0d1 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 5 Apr 2022 13:41:52 +0800 Subject: [PATCH 2/2] fix: panic by data racing --- hrp/step_testcase.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/hrp/step_testcase.go b/hrp/step_testcase.go index ea412058..031133a5 100644 --- a/hrp/step_testcase.go +++ b/hrp/step_testcase.go @@ -44,25 +44,27 @@ func (s *StepTestCaseWithOptionalArgs) Struct() *TStep { } func (s *StepTestCaseWithOptionalArgs) Run(r *SessionRunner) (*StepResult, error) { - stepVariables, err := r.MergeStepVariables(s.step.Variables) - if err != nil { - return nil, err - } - s.step.Variables = stepVariables - stepResult := &StepResult{ Name: s.step.Name, StepType: stepTypeTestCase, Success: false, } - testcase := s.step.TestCase.(*TestCase) - // copy testcase to avoid data racing - copiedTestCase := &TestCase{} - if err := copier.Copy(copiedTestCase, testcase); err != nil { - log.Error().Err(err).Msg("copy testcase failed") + stepVariables, err := r.MergeStepVariables(s.step.Variables) + if err != nil { return stepResult, err } + + // copy step to avoid data racing + copiedStep := &TStep{} + if err := copier.Copy(copiedStep, s.step); err != nil { + log.Error().Err(err).Msg("copy step failed") + return stepResult, err + } + + copiedStep.Variables = stepVariables + copiedTestCase := copiedStep.TestCase.(*TestCase) + // override testcase config extendWithTestCase(s.step, copiedTestCase)