From 100c22b81f76572f4a2613e928fd892959d415ab Mon Sep 17 00:00:00 2001 From: buyuxiang <347586493@qq.com> Date: Tue, 5 Jul 2022 14:36:45 +0800 Subject: [PATCH] feat: support curl to case --- examples/data/curl/curl_examples.txt | 21 + examples/data/har2case/demo-quickstart.har | 223 --------- examples/data/har2case/demo.har | 148 ------ go.mod | 1 + go.sum | 2 + hrp/internal/builtin/utils.go | 35 ++ hrp/internal/convert/README.md | 2 +- hrp/internal/convert/converter.go | 55 ++- hrp/internal/convert/from_curl.go | 524 +++++++++++++++++++++ hrp/internal/convert/from_curl_test.go | 104 ++++ 10 files changed, 731 insertions(+), 384 deletions(-) create mode 100644 examples/data/curl/curl_examples.txt delete mode 100644 examples/data/har2case/demo-quickstart.har delete mode 100644 examples/data/har2case/demo.har create mode 100644 hrp/internal/convert/from_curl.go create mode 100644 hrp/internal/convert/from_curl_test.go diff --git a/examples/data/curl/curl_examples.txt b/examples/data/curl/curl_examples.txt new file mode 100644 index 00000000..d6aa3d9f --- /dev/null +++ b/examples/data/curl/curl_examples.txt @@ -0,0 +1,21 @@ +curl httpbin.org + +curl https://httpbin.org/get?key1=value1&key2=value2 + +curl -H "Content-Type: application/json" \ + -H "Authorization: Bearer b7d03a6947b217efb6f3ec3bd3504582" \ + -d '{"type":"A","name":"www","data":"162.10.66.0","priority":null,"port":null,"weight":null}' \ + "https://httpbin.org/post" + +curl -F "dummyName=dummyFile" -F file1=@file1.txt -F file2=@file2.txt https://httpbin.org/post + +curl https://httpbin.org/post \ + -d 'shipment[to_address][id]=adr_HrBKVA85' \ + -d 'shipment[from_address][id]=adr_VtuTOj7o' \ + -d 'shipment[parcel][id]=prcl_WDv2VzHp' \ + -d 'shipment[is_return]=true' \ + -d 'shipment[customs_info][id]=cstinfo_bl5sE20Y' + +curl https://httpbing.org/post -H "Content-Type: application/x-www-form-urlencoded" \ + --data "key1=value+1&key2=value%3A2" + diff --git a/examples/data/har2case/demo-quickstart.har b/examples/data/har2case/demo-quickstart.har deleted file mode 100644 index f4de4473..00000000 --- a/examples/data/har2case/demo-quickstart.har +++ /dev/null @@ -1,223 +0,0 @@ -{ - "log": { - "version": "1.2", - "creator": { - "name": "Charles Proxy", - "version": "4.2.1" - }, - "entries": [ - { - "startedDateTime": "2018-02-19T17:30:00.904+08:00", - "time": 3, - "request": { - "method": "POST", - "url": "http://127.0.0.1:5000/api/get-token", - "httpVersion": "HTTP/1.1", - "cookies": [], - "headers": [ - { - "name": "Host", - "value": "127.0.0.1:5000" - }, - { - "name": "User-Agent", - "value": "python-requests/2.18.4" - }, - { - "name": "Accept-Encoding", - "value": "gzip, deflate" - }, - { - "name": "Accept", - "value": "*/*" - }, - { - "name": "Connection", - "value": "keep-alive" - }, - { - "name": "device_sn", - "value": "FwgRiO7CNA50DSU" - }, - { - "name": "user_agent", - "value": "iOS/10.3" - }, - { - "name": "os_platform", - "value": "ios" - }, - { - "name": "app_version", - "value": "2.8.6" - }, - { - "name": "Content-Length", - "value": "52" - }, - { - "name": "Content-Type", - "value": "application/json" - } - ], - "queryString": [], - "postData": { - "mimeType": "application/json", - "text": "{\"sign\": \"958a05393efef0ac7c0fb80a7eac45e24fd40c27\"}" - }, - "headersSize": 299, - "bodySize": 52 - }, - "response": { - "_charlesStatus": "COMPLETE", - "status": 200, - "statusText": "OK", - "httpVersion": "HTTP/1.0", - "cookies": [], - "headers": [ - { - "name": "Content-Type", - "value": "application/json" - }, - { - "name": "Content-Length", - "value": "46" - }, - { - "name": "Server", - "value": "Werkzeug/0.14.1 Python/3.6.4" - }, - { - "name": "Date", - "value": "Mon, 19 Feb 2018 09:30:00 GMT" - }, - { - "name": "Proxy-Connection", - "value": "Close" - } - ], - "content": { - "size": 46, - "mimeType": "application/json", - "text": "eyJzdWNjZXNzIjogdHJ1ZSwgInRva2VuIjogImJhTkxYMXpoRllQMTFTZWIifQ\u003d\u003d", - "encoding": "base64" - }, - "headersSize": 175, - "bodySize": 46 - }, - "serverIPAddress": "127.0.0.1", - "cache": {}, - "timings": { - "dns": 1, - "connect": 0, - "ssl": -1, - "send": 0, - "wait": 1, - "receive": 1 - } - }, - { - "startedDateTime": "2018-02-19T17:30:00.911+08:00", - "time": 3, - "request": { - "method": "POST", - "url": "http://127.0.0.1:5000/api/users/1000", - "httpVersion": "HTTP/1.1", - "cookies": [], - "headers": [ - { - "name": "Host", - "value": "127.0.0.1:5000" - }, - { - "name": "User-Agent", - "value": "python-requests/2.18.4" - }, - { - "name": "Accept-Encoding", - "value": "gzip, deflate" - }, - { - "name": "Accept", - "value": "*/*" - }, - { - "name": "Connection", - "value": "keep-alive" - }, - { - "name": "device_sn", - "value": "FwgRiO7CNA50DSU" - }, - { - "name": "token", - "value": "baNLX1zhFYP11Seb" - }, - { - "name": "Content-Length", - "value": "39" - }, - { - "name": "Content-Type", - "value": "application/json" - } - ], - "queryString": [], - "postData": { - "mimeType": "application/json", - "text": "{\"name\": \"user1\", \"password\": \"123456\"}" - }, - "headersSize": 265, - "bodySize": 39 - }, - "response": { - "_charlesStatus": "COMPLETE", - "status": 201, - "statusText": "CREATED", - "httpVersion": "HTTP/1.0", - "cookies": [], - "headers": [ - { - "name": "Content-Type", - "value": "application/json" - }, - { - "name": "Content-Length", - "value": "54" - }, - { - "name": "Server", - "value": "Werkzeug/0.14.1 Python/3.6.4" - }, - { - "name": "Date", - "value": "Mon, 19 Feb 2018 09:30:00 GMT" - }, - { - "name": "Proxy-Connection", - "value": "Close" - } - ], - "content": { - "size": 54, - "mimeType": "application/json", - "text": "eyJzdWNjZXNzIjogdHJ1ZSwgIm1zZyI6ICJ1c2VyIGNyZWF0ZWQgc3VjY2Vzc2Z1bGx5LiJ9", - "encoding": "base64" - }, - "headersSize": 77, - "bodySize": 54 - }, - "serverIPAddress": "127.0.0.1", - "cache": {}, - "timings": { - "dns": 0, - "connect": 0, - "ssl": -1, - "send": 0, - "wait": 3, - "receive": 0 - } - } - ] - } -} \ No newline at end of file diff --git a/examples/data/har2case/demo.har b/examples/data/har2case/demo.har deleted file mode 100644 index f56e7450..00000000 --- a/examples/data/har2case/demo.har +++ /dev/null @@ -1,148 +0,0 @@ -{ - "log": { - "version": "1.2", - "creator": { - "name": "Charles Proxy", - "version": "4.2" - }, - "entries": [ - { - "startedDateTime": "2017-11-13T11:40:07.212+08:00", - "time": 35, - "request": { - "method": "POST", - "url": "https://httprunner.top/api/v1/Account/Login", - "httpVersion": "HTTP/1.1", - "cookies": [ - { - "name": "lang", - "value": "zh" - } - ], - "headers": [ - { - "name": "Host", - "value": "httprunner.top" - }, - { - "name": "Connection", - "value": "keep-alive" - }, - { - "name": "Content-Length", - "value": "50" - }, - { - "name": "Accept", - "value": "application/json" - }, - { - "name": "Origin", - "value": "https://httprunner.top" - }, - { - "name": "User-Agent", - "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" - }, - { - "name": "Content-Type", - "value": "application/json" - }, - { - "name": "Referer", - "value": "https://httprunner.top/login" - }, - { - "name": "Accept-Encoding", - "value": "gzip, deflate, br" - }, - { - "name": "Accept-Language", - "value": "en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4" - } - ], - "queryString": [], - "postData": { - "mimeType": "application/json", - "text": "{\"UserName\":\"test001\",\"Pwd\":\"123\",\"VerCode\":\"\"}" - }, - "headersSize": 640, - "bodySize": 50 - }, - "response": { - "_charlesStatus": "COMPLETE", - "status": 200, - "statusText": "OK", - "httpVersion": "HTTP/1.1", - "cookies": [ - { - "name": "lang", - "value": "zh", - "path": "/", - "domain": ".httprunner.top", - "expires": null, - "httpOnly": false, - "secure": false, - "comment": null, - "_maxAge": null - } - ], - "headers": [ - { - "name": "Date", - "value": "Mon, 13 Nov 2017 03:40:07 GMT" - }, - { - "name": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "name": "Content-Length", - "value": "71" - }, - { - "name": "Cache-Control", - "value": "no-cache" - }, - { - "name": "Pragma", - "value": "no-cache" - }, - { - "name": "Expires", - "value": "-1" - }, - { - "name": "Server", - "value": "Microsoft-IIS/8.5" - }, - { - "name": "X-AspNet-Version", - "value": "4.0.30319" - } - ], - "content": { - "size": 71, - "mimeType": "application/json; charset=utf-8", - "text": "eyJJc1N1Y2Nlc3MiOnRydWUsIkNvZGUiOjIwMCwiTWVzc2FnZSI6bnVsbCwiVmFsdWUiOnsiQmxuUmVzdWx0Ijp0cnVlfX0=", - "encoding": "base64" - }, - "redirectURL": null, - "headersSize": 0, - "bodySize": 71 - }, - "serverIPAddress": "192.168.1.169", - "cache": {}, - "timings": { - "dns": -1, - "connect": -1, - "ssl": -1, - "send": 6, - "wait": 28, - "receive": 1 - } - } - - ] - } -} \ No newline at end of file diff --git a/go.mod b/go.mod index 6d70a855..f4d02e95 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/fatih/color v1.13.0 github.com/getsentry/sentry-go v0.13.0 github.com/go-openapi/spec v0.20.6 + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/google/uuid v1.3.0 github.com/gorilla/websocket v1.4.1 github.com/httprunner/funplugin v0.5.0 diff --git a/go.sum b/go.sum index 3f819360..b7292409 100644 --- a/go.sum +++ b/go.sum @@ -216,6 +216,8 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/hrp/internal/builtin/utils.go b/hrp/internal/builtin/utils.go index 3b3a62b6..abea592e 100644 --- a/hrp/internal/builtin/utils.go +++ b/hrp/internal/builtin/utils.go @@ -1,6 +1,7 @@ package builtin import ( + "bufio" "bytes" "encoding/csv" builtinJSON "encoding/json" @@ -450,6 +451,40 @@ func ReadFile(path string) ([]byte, error) { return file, nil } +func ReadCmdLines(path string) ([]string, error) { + var err error + path, err = filepath.Abs(path) + if err != nil { + log.Error().Err(err).Str("path", path).Msg("convert absolute path failed") + return nil, err + } + file, err := os.Open(path) + if err != nil { + log.Error().Err(err).Str("path", path).Msg("open file failed") + return nil, err + } + defer file.Close() + + var line string + var lines []string + scanner := bufio.NewScanner(file) + // FIXME: resize scanner's capacity for lines over 64K + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if text == "" || text == "\n" { + continue + } + if strings.HasSuffix(text, "\\") { + line = line + strings.Trim(text, "\\") + continue + } + line = line + text + lines = append(lines, line) + line = "" + } + return lines, scanner.Err() +} + func GetFileNameWithoutExtension(path string) string { base := filepath.Base(path) ext := filepath.Ext(base) diff --git a/hrp/internal/convert/README.md b/hrp/internal/convert/README.md index 963957c3..90c0314f 100644 --- a/hrp/internal/convert/README.md +++ b/hrp/internal/convert/README.md @@ -73,7 +73,7 @@ cookies: | Postman | ✅ | ✅ | ❌ | ✅ | | JMeter | ❌ | ❌ | ❌ | ❌ | | Swagger | ❌ | ❌ | ❌ | ❌ | -| curl | ❌ | ❌ | ❌ | ❌ | +| curl | ✅ | ✅ | ❌ | ✅ | | Apache ab | ❌ | ❌ | ❌ | ❌ | | JSON | ✅ | ✅ | ❌ | ✅ | | YAML | ✅ | ✅ | ❌ | ✅ | diff --git a/hrp/internal/convert/converter.go b/hrp/internal/convert/converter.go index dd3f22b7..5e5a446a 100644 --- a/hrp/internal/convert/converter.go +++ b/hrp/internal/convert/converter.go @@ -3,7 +3,10 @@ package convert import ( _ "embed" "fmt" + "os" "path/filepath" + "strings" + "time" "github.com/pkg/errors" "github.com/rs/zerolog/log" @@ -58,18 +61,18 @@ func Run(outputType OutputType, outputDir, profilePath string, args []string) { }) var outputFiles []string - for _, path := range args { + for _, inputSample := range args { // loads source file and convert to TCase format - tCase, err := LoadTCase(path) + tCase, err := LoadTCase(inputSample) if err != nil { - log.Warn().Err(err).Str("path", path).Msg("convert source file failed") + log.Warn().Err(err).Str("input sample", inputSample).Msg("convert input sample failed") continue } caseConverter := &TCaseConverter{ - SourcePath: path, - OutputDir: outputDir, - TCase: tCase, + InputSample: inputSample, + OutputDir: outputDir, + TCase: tCase, } // override TCase with profile @@ -91,7 +94,7 @@ func Run(outputType OutputType, outputDir, profilePath string, args []string) { } if err != nil { log.Error().Err(err). - Str("source path", path). + Str("input sample", caseConverter.InputSample). Msg("convert case failed") continue } @@ -102,6 +105,14 @@ func Run(outputType OutputType, outputDir, profilePath string, args []string) { // LoadTCase loads source file and convert to TCase type func LoadTCase(path string) (*hrp.TCase, error) { + if strings.HasPrefix(path, "curl") { + // 'path' contains curl command + curlCase, err := LoadCurlCase(path) + if err != nil { + return nil, err + } + return curlCase, nil + } extName := filepath.Ext(path) if extName == "" { return nil, errors.New("file extension is not specified") @@ -155,6 +166,12 @@ func LoadTCase(path string) (*hrp.TCase, error) { return nil, errors.New("convert pytest is not implemented") case ".jmx": // TODO return nil, errors.New("convert JMeter jmx is not implemented") + case ".txt": + curlCase, err := LoadCurlCase(path) + if err != nil { + return nil, err + } + return curlCase, nil } return nil, fmt.Errorf("unsupported file type: %v", extName) @@ -162,17 +179,31 @@ func LoadTCase(path string) (*hrp.TCase, error) { // TCaseConverter holds the common properties of case converter type TCaseConverter struct { - SourcePath string - OutputDir string - TCase *hrp.TCase + InputSample string + OutputDir string + TCase *hrp.TCase } func (c *TCaseConverter) genOutputPath(suffix string) string { - outFileFullName := builtin.GetFileNameWithoutExtension(c.SourcePath) + "_test" + suffix + var outFileFullName string + if curlCmd := strings.TrimSpace(c.InputSample); strings.HasPrefix(curlCmd, "curl") { + outFileFullName = fmt.Sprintf("curl_%v_test_%v", time.Now().Format("20060102150405"), suffix) + if c.OutputDir != "" { + return filepath.Join(c.OutputDir, outFileFullName) + } else { + curWorkDir, err := os.Getwd() + if err != nil { + log.Error().Err(err).Msg("get current working direction failed") + os.Exit(1) + } + return filepath.Join(curWorkDir, outFileFullName) + } + } + outFileFullName = builtin.GetFileNameWithoutExtension(c.InputSample) + "_test" + suffix if c.OutputDir != "" { return filepath.Join(c.OutputDir, outFileFullName) } else { - return filepath.Join(filepath.Dir(c.SourcePath), outFileFullName) + return filepath.Join(filepath.Dir(c.InputSample), outFileFullName) } // TODO avoid outFileFullName conflict? } diff --git a/hrp/internal/convert/from_curl.go b/hrp/internal/convert/from_curl.go new file mode 100644 index 00000000..57b7fc18 --- /dev/null +++ b/hrp/internal/convert/from_curl.go @@ -0,0 +1,524 @@ +package convert + +import ( + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/google/shlex" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + + "github.com/httprunner/httprunner/v4/hrp" + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" + "github.com/httprunner/httprunner/v4/hrp/internal/json" +) + +const ( + originCmdKey = "_origin_cmd_key" + targetUrlKey = "_target_url_key" +) + +var curlOptionAliasMap = map[string]string{ + "-a": "--append", + "-A": "--user-agent", + "-b": "--cookie", + "-B": "--use-ascii", + "-c": "--cookie-jar", + "-C": "--continue-at", + "-d": "--data", + "-D": "--dump-header", + "-e": "--referer", + "-E": "--cert", + "-f": "--fail", + "-F": "--form", + "-g": "--globoff", + "-G": "--get", + "-h": "--help", + "-H": "--header", + "-i": "--include", + "-I": "--head", + "-j": "--junk-session-cookies", + "-J": "--remote-header-name", + "-k": "--insecure", + "-K": "--config", + "-l": "--list-only", + "-L": "--location", + "-m": "--max-time", + "-M": "--manual", + "-n": "--netrc", + "-N": "--no-buffer", + "-o": "--output", + "-O": "--remote-name", + "-p": "--proxytunnel", + "-P": "--ftp-port", + "-q": "--disable", + "-Q": "--quote", + "-r": "--range", + "-R": "--remote-time", + "-s": "--silent", + "-S": "--show-error", + "-t": "--telnet-option", + "-T": "--upload-file", + "-u": "--user", + "-U": "--proxy-user", + "-v": "--verbose", + "-V": "--version", + "-w": "--write-out", + "-x": "--proxy", + "-X": "--request", + "-Y": "--speed-limit", + "-y": "--speed-time", + "-z": "--time-cond", + "-Z": "--parallel", +} + +var curlOptionWhiteMap = map[string]struct{}{ + "--cookie": {}, + "--data": {}, + "--form": {}, + "--get": {}, + "--head": {}, + "--header": {}, + "--request": {}, +} + +var curlOptionWhiteList []string + +func init() { + for option := range curlOptionWhiteMap { + curlOptionWhiteList = append(curlOptionWhiteList, option) + } +} + +func LoadCurlCase(inputSample string) (*hrp.TCase, error) { + var err error + cmds, err := builtin.ReadCmdLines(inputSample) + if err != nil { + return nil, err + } + tCase := &hrp.TCase{ + Config: &hrp.TConfig{Name: "testcase converted from curl command"}, + } + for _, cmd := range cmds { + caseCurl, err := loadCaseCurl(cmd) + if err != nil { + return nil, err + } + tStep, err := caseCurl.toTStep() + if err != nil { + return nil, err + } + tCase.TestSteps = append(tCase.TestSteps, tStep) + } + err = tCase.MakeCompat() + if err != nil { + return nil, err + } + return tCase, nil +} + +func loadCaseCurl(cmd string) (CaseCurl, error) { + caseCurl := make(CaseCurl) + var err error + caseCurl, err = parseCaseCurl(cmd) + if err != nil { + return nil, errors.Wrap(err, "load curl command failed") + } + // deal with option alias, turn all options to long form + if err = caseCurl.toAlias(); err != nil { + return nil, errors.Wrap(err, "identify curl option alias failed") + } + // check if caseCurl contains unsupported args + if err = caseCurl.checkOptions(); err != nil { + return nil, errors.Wrap(err, "check curl option failed") + } + caseCurl.Set(originCmdKey, cmd) + return caseCurl, nil +} + +// parseCaseCurl parses command string to map, save command keyword and bool option as map key only. +// Otherwise, save option as map key and the following args([]string) as map value +func parseCaseCurl(cmd string) (CaseCurl, error) { + cmdWords, err := shlex.Split(cmd) + if err != nil { + return nil, err + } + + // deal with \n in the command string + //var cmdWords []string + //for _, w := range rawCmd { + // if w == "\n" { + // continue + // } + // cmdWords = append(cmdWords, strings.Trim(w, "\n)) + //} + + // parse the command string to map + res := make(CaseCurl) + var i int + if cmdWords[i] != "curl" { + return nil, errors.New("command not started with curl") + } + i++ + for i < len(cmdWords) { + if !strings.HasPrefix(cmdWords[i], "-") { + // save target url + res.Set(targetUrlKey, cmdWords[i]) + i++ + continue + } + option := cmdWords[i] + i++ + if i < len(cmdWords) && !strings.HasPrefix(cmdWords[i], "-") { + // option with only one following argument + res.Add(option, cmdWords[i]) + i++ + continue + } + // option with no argument, i.e. bool option, save key only + res[option] = nil + } + return res, nil +} + +type CaseCurl map[string][]string + +// GetFirst gets the first value associated with the given key. +// If there are no values associated with the key, GetFirst returns the empty string. +func (c CaseCurl) GetFirst(key string) string { + if c == nil { + return "" + } + vs := c[key] + if len(vs) == 0 { + return "" + } + return vs[0] +} + +func (c CaseCurl) Set(key, value string) { + c[key] = []string{value} +} + +func (c CaseCurl) Add(key, value string) { + c[key] = append(c[key], value) +} + +// HaveKey checks key existed or not +func (c CaseCurl) HaveKey(key string) bool { + if c == nil { + return false + } + _, ok := c[key] + return ok +} + +// HaveKeyWithPrefix checks key with prefix existed or not +func (c CaseCurl) HaveKeyWithPrefix(prefix string) bool { + if c == nil { + return false + } + for k := range c { + if strings.HasPrefix(k, prefix) { + return true + } + } + return false +} + +func (c CaseCurl) toAlias() error { + for option, args := range c { + if !strings.HasPrefix(option, "-") || strings.HasPrefix(option, "--") { + // not a short option like -X, pass + continue + } + longOption, ok := curlOptionAliasMap[option] + if !ok { + return errors.Errorf("unexpected curl option: %v", option) + } + // FIXME: need to copy args or not? + c[longOption] = args + delete(c, option) + } + return nil +} + +func (c CaseCurl) checkOptions() error { + for option := range c { + if option == originCmdKey || option == targetUrlKey { + continue + } + _, ok := curlOptionWhiteMap[option] + if !ok { + return errors.Errorf("option %v not supported yet. available options: %v", option, curlOptionWhiteList) + } + } + return nil +} + +func (c CaseCurl) ToTCase() (*hrp.TCase, error) { + testSteps, err := c.toTStep() + if err != nil { + return nil, err + } + tCase := &hrp.TCase{ + Config: &hrp.TConfig{Name: "testcase converted from curl command"}, + TestSteps: []*hrp.TStep{testSteps}, + } + err = tCase.MakeCompat() + if err != nil { + return nil, err + } + return tCase, nil +} + +func (c CaseCurl) toTStep() (*hrp.TStep, error) { + log.Info(). + Str("cmd", c.GetFirst(originCmdKey)). + Msg("convert teststep") + step := &stepFromCurl{ + TStep: &hrp.TStep{ + Request: &hrp.Request{}, + }, + } + if err := step.makeRequestName(c); err != nil { + return nil, err + } + if err := step.makeRequestMethod(c); err != nil { + return nil, err + } + if err := step.makeRequestURL(c); err != nil { + return nil, err + } + if err := step.makeRequestParams(c); err != nil { + return nil, err + } + if err := step.makeRequestHeaders(c); err != nil { + return nil, err + } + if err := step.makeRequestCookies(c); err != nil { + return nil, err + } + if err := step.makeRequestBody(c); err != nil { + return nil, err + } + return step.TStep, nil +} + +type stepFromCurl struct { + *hrp.TStep +} + +func (s *stepFromCurl) makeRequestName(c CaseCurl) error { + s.Name = c.GetFirst(originCmdKey) + return nil +} + +func (s *stepFromCurl) makeRequestMethod(c CaseCurl) error { + // default --get + s.Request.Method = http.MethodGet + if c.HaveKey("--data") || c.HaveKey("--form") { + s.Request.Method = http.MethodPost + } + if c.HaveKey("--head") { + s.Request.Method = http.MethodHead + } + if c.HaveKey("--request") { + s.Request.Method = hrp.HTTPMethod(strings.ToUpper(c.GetFirst("--request"))) + } + return nil +} + +func (s *stepFromCurl) makeRequestURL(c CaseCurl) error { + rawUrl := c.GetFirst(targetUrlKey) + if rawUrl == "" { + return errors.New("URL not found") + } + u, err := url.Parse(rawUrl) + if err != nil { + return errors.Wrap(err, "parse URL error") + } + // default protocol consistent with curl (http) + if u.Scheme == "" { + u.Scheme = "http" + } + s.Request.URL = fmt.Sprintf("%s://%s", u.Scheme, u.Host+u.Path) + return nil +} + +func (s *stepFromCurl) makeRequestParams(c CaseCurl) error { + s.Request.Params = make(map[string]interface{}) + rawUrl := c.GetFirst(targetUrlKey) + u, err := url.Parse(rawUrl) + if err != nil { + return errors.Wrap(err, "parse URL error") + } + s.Request.Params = make(map[string]interface{}) + queryValues := u.Query() + // query key may correspond to more than one value, get first query key only + for k := range queryValues { + s.Request.Params[k] = queryValues.Get(k) + } + return nil +} + +func (s *stepFromCurl) makeRequestHeaders(c CaseCurl) error { + s.Request.Headers = make(map[string]string) + headerList := c["--header"] + for _, headerExpr := range headerList { + if err := s.makeRequestHeader(headerExpr); err != nil { + return err + } + } + return nil +} + +func (s *stepFromCurl) makeRequestHeader(headerExpr string) error { + headerExpr = strings.TrimSpace(headerExpr) + if strings.HasPrefix(headerExpr, "@") { + return errors.Errorf("loading header from file not supported: %v", headerExpr) + } + if strings.TrimSpace(headerExpr) == ";" || strings.HasPrefix(strings.TrimSpace(headerExpr), ":") { + return errors.Errorf("invalid curl header format: %v", headerExpr) + } + if s.Request.Headers == nil { + s.Request.Headers = make(map[string]string) + } + if i := strings.Index(headerExpr, ":"); i != -1 { + headerKey := strings.TrimSpace(headerExpr[:i]) + var headerValue string + if i < len(headerExpr)-1 { + headerValue = strings.TrimSpace(headerExpr[i+1:]) + } else { + headerValue = "" + } + if strings.ToLower(headerKey) == "host" { + // headerExpr modifying internal header like "Host:" + log.Warn().Str("--header", headerExpr).Msg("modifying internal header not supported") + return nil + } + if headerValue != "" { + // normal headerExpr like "User-Agent: httprunner" + s.Request.Headers[headerKey] = headerValue + return nil + } + } + if i := strings.Index(headerExpr, ";"); i != -1 { + // headerExpr terminated with a semicolon like "X-Custom-Header;" + headerKey := strings.TrimSpace(headerExpr[:i]) + if strings.ToLower(headerKey) == "host" { + log.Warn().Str("--header", headerExpr).Msg("modifying internal header not supported") + return nil + } + s.Request.Headers[headerKey] = "" + return nil + } + log.Warn().Str("--header", headerExpr).Msg("pass meaningless curl header expression") + return nil +} + +func (s *stepFromCurl) makeRequestCookies(c CaseCurl) error { + s.Request.Cookies = make(map[string]string) + cookieList := c["--cookie"] + for _, cookieExpr := range cookieList { + if err := s.makeRequestCookie(cookieExpr); err != nil { + return err + } + } + return nil +} + +func (s *stepFromCurl) makeRequestCookie(cookieExpr string) error { + if !strings.Contains(cookieExpr, "=") { + return errors.Errorf("loading cookie from file not supported: %v", cookieExpr) + } + if s.Request.Cookies == nil { + s.Request.Cookies = make(map[string]string) + } + // deal with cookieExpr like "name1=value1; name2 = value2" + cookies := strings.Split(cookieExpr, ";") + for _, cookie := range cookies { + i := strings.Index(cookie, "=") + if i == -1 { + log.Warn().Str("--cookie", cookie).Msg("pass meaningless curl cookie expression") + continue + } + cookieKey := strings.TrimSpace(cookie[:i]) + var cookieValue string + if i < len(cookie)-1 { + cookieValue = strings.TrimSpace(cookie[i+1:]) + } else { + cookieValue = "" + } + s.Request.Cookies[cookieKey] = cookieValue + } + return nil +} + +func (s *stepFromCurl) makeRequestBody(c CaseCurl) error { + // check priority: --data > --form + dataList, dataExisted := c["--data"] + formList, formExisted := c["--form"] + if dataExisted { + if err := s.makeRequestData(dataList); err != nil { + return err + } + } else if formExisted { + if err := s.makeRequestForm(formList); err != nil { + return err + } + } + return nil +} + +func (s *stepFromCurl) makeRequestData(dataList []string) error { + dataMap := make(map[string]interface{}) + for _, dataExpr := range dataList { + if strings.HasPrefix(dataExpr, "@") { + return errors.Errorf("loading data from file not supported: %v", dataExpr) + } + var m map[string]interface{} + // --data may be json string, try to unmarshal to map first + err := json.Unmarshal([]byte(dataExpr), &m) + if err == nil { + for k, v := range m { + dataMap[k] = v + } + continue + } + dataValues, err := url.ParseQuery(dataExpr) + if err != nil { + return err + } + for dataKey := range dataValues { + dataMap[dataKey] = strings.Trim(dataValues.Get(dataKey), "\"'") + } + } + s.Request.Body = dataMap + return nil +} + +func (s *stepFromCurl) makeRequestForm(formList []string) error { + if s.Request.Upload == nil { + s.Request.Upload = make(map[string]interface{}) + } + for _, formExpr := range formList { + if !strings.Contains(formExpr, "=") { + return errors.Errorf("option --form: is badly used: %v", formExpr) + } + if i := strings.Index(formExpr, "="); i != -1 { + formKey := strings.TrimSpace(formExpr[:i]) + var formValue string + if i < len(formExpr)-1 { + formValue = strings.TrimSpace(formExpr[i+1:]) + } else { + formValue = "" + } + filePath := strings.TrimLeft(formValue, "@") + s.Request.Upload[formKey] = strings.Trim(filePath, "\"") + } + } + return nil +} diff --git a/hrp/internal/convert/from_curl_test.go b/hrp/internal/convert/from_curl_test.go new file mode 100644 index 00000000..a9b5f4f2 --- /dev/null +++ b/hrp/internal/convert/from_curl_test.go @@ -0,0 +1,104 @@ +package convert + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var curlPath = "../../../examples/data/curl/curl_examples.txt" + +func TestLoadCurlCase(t *testing.T) { + tCase, err := LoadCurlCase(curlPath) + if !assert.NoError(t, err) { + t.Fatal(err) + } + if !assert.Equal(t, 6, len(tCase.TestSteps)) { + t.Fatal() + } + + // curl httpbin.org + if !assert.Equal(t, "curl httpbin.org", tCase.TestSteps[0].Name) { + t.Fatal() + } + if !assert.EqualValues(t, "GET", tCase.TestSteps[0].Request.Method) { + t.Fatal() + } + if !assert.Equal(t, "http://httpbin.org", tCase.TestSteps[0].Request.URL) { + t.Fatal() + } + + // curl https://httpbin.org/get?key1=value1&key2=value2 + if !assert.Equal(t, "https://httpbin.org/get", tCase.TestSteps[1].Request.URL) { + t.Fatal() + } + if !assert.Equal(t, map[string]interface{}{ + "key1": "value1", + "key2": "value2", + }, tCase.TestSteps[1].Request.Params) { + t.Fatal() + } + + // curl -H "Content-Type: application/json" \ + // -H "Authorization: Bearer b7d03a6947b217efb6f3ec3bd3504582" \ + // -d '{"type":"A","name":"www","data":"162.10.66.0","priority":null,"port":null,"weight":null}' \ + // "https://httpbin.org/post" + if !assert.EqualValues(t, "POST", tCase.TestSteps[2].Request.Method) { + t.Fatal() + } + if !assert.Equal(t, map[string]string{ + "Authorization": "Bearer b7d03a6947b217efb6f3ec3bd3504582", + "Content-Type": "application/json", + }, tCase.TestSteps[2].Request.Headers) { + t.Fatal() + } + if !assert.Equal(t, map[string]interface{}{ + "data": "162.10.66.0", + "name": "www", + "port": nil, + "priority": nil, + "type": "A", + "weight": nil, + }, tCase.TestSteps[2].Request.Body) { + t.Fatal() + } + + // curl -F "dummyName=dummyFile" -F file1=@file1.txt -F file2=@file2.txt https://httpbin.org/post + if !assert.Equal(t, map[string]interface{}{ + "dummyName": "dummyFile", + "file1": "@file1.txt", + "file2": "@file2.txt", + }, tCase.TestSteps[3].Request.Upload) { + t.Fatal() + } + + // curl https://httpbin.org/post \ + // -d 'shipment[to_address][id]=adr_HrBKVA85' \ + // -d 'shipment[from_address][id]=adr_VtuTOj7o' \ + // -d 'shipment[parcel][id]=prcl_WDv2VzHp' \ + // -d 'shipment[is_return]=true' \ + // -d 'shipment[customs_info][id]=cstinfo_bl5sE20Y' + if !assert.Equal(t, map[string]interface{}{ + "shipment[customs_info][id]": "cstinfo_bl5sE20Y", + "shipment[from_address][id]": "adr_VtuTOj7o", + "shipment[is_return]": "true", + "shipment[parcel][id]": "prcl_WDv2VzHp", + "shipment[to_address][id]": "adr_HrBKVA85", + }, tCase.TestSteps[4].Request.Body) { + t.Fatal() + } + + // curl https://httpbing.org/post -H "Content-Type: application/x-www-form-urlencoded" \ + // --data "key1=value+1&key2=value%3A2" + if !assert.Equal(t, map[string]string{ + "Content-Type": "application/x-www-form-urlencoded", + }, tCase.TestSteps[5].Request.Headers) { + t.Fatal() + } + if !assert.Equal(t, map[string]interface{}{ + "key1": "value 1", + "key2": "value:2", + }, tCase.TestSteps[5].Request.Body) { + t.Fatal() + } +}