feat: hrp support uploading file

docs/CHANGELOG.md

@@ -6,6 +6,8 @@
 
 - change: set http request timeout default to 120s
 - fix: insert response cookies into request for redirect requests
+- feat: support uploading file by multipart/form-data
+- fix: failed to convert postman collection containing multipart/form-data requests to pytest
 
 ## v4.1.4 (2022-06-17)
 

examples/data/postman/intro.txt

@@ -0,0 +1 @@
+HttpRunner is an open source API testing tool that supports HTTP(S)/HTTP2/WebSocket/RPC network protocols, covering API testing, performance testing and digital experience monitoring (DEM) test types. Enjoy!

examples/data/postman/postman_collection.json

@@ -230,6 +230,16 @@
 									"value": "v3",
 									"type": "text",
 									"disabled": true
+								},
+								{
+									"key": "intro_key",
+									"type": "file",
+									"src": "intro.txt"
+								},
+								{
+									"key": "logo_key",
+									"type": "file",
+									"src": "logo.jpeg"
 								}
 							]
 						},

hrp/internal/builtin/function.go

@@ -1,25 +1,34 @@
 package builtin
 
 import (
+	"bytes"
 	"crypto/md5"
 	"encoding/hex"
+	"fmt"
 	"math"
 	"math/rand"
+	"mime/multipart"
 	"os"
+	"path/filepath"
 	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
 )
 
 var Functions = map[string]interface{}{
-	"get_timestamp":     getTimestamp,    // call without arguments
-	"sleep":             sleep,           // call with one argument
-	"gen_random_string": genRandomString, // call with one argument
-	"max":               math.Max,        // call with two arguments
-	"md5":               MD5,             // call with one argument
-	"parameterize":      loadFromCSV,
-	"P":                 loadFromCSV,
-	"environ":           os.Getenv,
-	"ENV":               os.Getenv,
-	"load_ws_message":   loadMessage,
+	"get_timestamp":          getTimestamp,    // call without arguments
+	"sleep":                  sleep,           // call with one argument
+	"gen_random_string":      genRandomString, // call with one argument
+	"max":                    math.Max,        // call with two arguments
+	"md5":                    MD5,             // call with one argument
+	"parameterize":           loadFromCSV,
+	"P":                      loadFromCSV,
+	"environ":                os.Getenv,
+	"ENV":                    os.Getenv,
+	"load_ws_message":        loadMessage,
+	"multipart_encoder":      multipartEncoder,
+	"multipart_content_type": multipartContentType,
 }
 
 func init() {
@@ -50,3 +59,60 @@ func MD5(str string) string {
 	hasher.Write([]byte(str))
 	return hex.EncodeToString(hasher.Sum(nil))
 }
+
+type TFormWriter struct {
+	Writer  *multipart.Writer
+	Payload *bytes.Buffer
+}
+
+func multipartEncoder(formMap map[string]interface{}) *TFormWriter {
+	payload := &bytes.Buffer{}
+	writer := multipart.NewWriter(payload)
+	for formKey, formValue := range formMap {
+		formValueString := fmt.Sprintf("%v", formValue)
+		if err := writeFormDataFile(writer, formKey, formValueString); err == nil {
+			// form value is a file path
+			continue
+		}
+		// form value is not a file path, write as raw string
+		if err := writer.WriteField(formKey, formValueString); err != nil {
+			log.Info().Err(err).Msgf("failed to write field: %v=%v, ignore", formKey, formValue)
+		}
+	}
+	if err := writer.Close(); err != nil {
+	}
+	return &TFormWriter{
+		Writer:  writer,
+		Payload: payload,
+	}
+}
+
+func writeFormDataFile(writer *multipart.Writer, fName, fPath string) error {
+	var err error
+	fPath, err = filepath.Abs(fPath)
+	if err != nil {
+		log.Error().Err(err).Str("path", fPath).Msg("convert absolute path failed")
+		return err
+	}
+	if !IsFilePathExists(fPath) {
+		return errors.Errorf("file %v not existed", fPath)
+	}
+	file, err := os.ReadFile(fPath)
+	if err != nil {
+		log.Error().Err(err).Str("path", fPath).Msg("read file failed")
+		return err
+	}
+	formFile, err := writer.CreateFormFile(fName, filepath.Base(fPath))
+	if err != nil {
+		return err
+	}
+	_, err = formFile.Write(file)
+	return err
+}
+
+func multipartContentType(w *TFormWriter) string {
+	if w.Writer == nil {
+		return ""
+	}
+	return w.Writer.FormDataContentType()
+}

hrp/internal/convert/converter_postman.go

@@ -1,13 +1,8 @@
 package convert
 
 import (
-	"bytes"
 	"fmt"
-	"io"
-	"mime/multipart"
 	"net/url"
-	"os"
-	"path/filepath"
 	"reflect"
 	"strings"
 
@@ -158,7 +153,7 @@ func (c *ConverterPostman) ToYAML() (string, error) {
 }
 
 func (c *ConverterPostman) ToGoTest() (string, error) {
-	//TODO implement me
+	// TODO implement me
 	return "", errors.New("convert from postman to gotest scripts is not supported yet")
 }
 
@@ -208,7 +203,7 @@ func (c *ConverterPostman) prepareTestSteps(casePostman *CasePostman) ([]*hrp.TS
 
 	var steps []*hrp.TStep
 	for _, item := range itemList {
-		step, err := c.prepareTestStep(&item, steps)
+		step, err := c.prepareTestStep(&item)
 		if err != nil {
 			return nil, err
 		}
@@ -234,7 +229,7 @@ func extractItemList(item TItem, itemList *[]TItem) {
 	}
 }
 
-func (c *ConverterPostman) prepareTestStep(item *TItem, steps []*hrp.TStep) (*hrp.TStep, error) {
+func (c *ConverterPostman) prepareTestStep(item *TItem) (*hrp.TStep, error) {
 	log.Info().
 		Str("method", item.Request.Method).
 		Str("url", item.Request.URL.Raw).
@@ -265,7 +260,7 @@ func (c *ConverterPostman) prepareTestStep(item *TItem, steps []*hrp.TStep) (*hr
 	if err := step.makeRequestCookies(item); err != nil {
 		return nil, err
 	}
-	if err := step.makeRequestBody(item, steps); err != nil {
+	if err := step.makeRequestBody(item); err != nil {
 		return nil, err
 	}
 	return &step.TStep, nil
@@ -373,7 +368,7 @@ func (s *stepFromPostman) parseRequestCookiesMap(cookies string) {
 	}
 }
 
-func (s *stepFromPostman) makeRequestBody(item *TItem, steps []*hrp.TStep) error {
+func (s *stepFromPostman) makeRequestBody(item *TItem) error {
 	mode := item.Request.Body.Mode
 	if mode == "" {
 		return nil
@@ -382,7 +377,7 @@ func (s *stepFromPostman) makeRequestBody(item *TItem, steps []*hrp.TStep) error
 	case enumBodyRaw:
 		return s.makeRequestBodyRaw(item)
 	case enumBodyFormData:
-		return s.makeRequestBodyFormData(item, steps)
+		return s.makeRequestBodyFormData(item)
 	case enumBodyUrlEncoded:
 		return s.makeRequestBodyUrlEncoded(item)
 	case enumBodyFile, enumBodyGraphQL:
@@ -424,49 +419,22 @@ func (s *stepFromPostman) makeRequestBodyRaw(item *TItem) (err error) {
 	return
 }
 
-func (s *stepFromPostman) makeRequestBodyFormData(item *TItem, steps []*hrp.TStep) (err error) {
-	defer func() {
-		if err != nil {
-			err = errors.Wrap(err, "make request body form-data failed")
-		}
-	}()
-	payload := &bytes.Buffer{}
-	writer := multipart.NewWriter(payload)
+func (s *stepFromPostman) makeRequestBodyFormData(item *TItem) error {
+	s.Request.Upload = make(map[string]interface{})
 	for _, field := range item.Request.Body.FormData {
 		if field.Disabled {
 			continue
 		}
 		// form data could be text or file
 		if field.Type == enumFieldTypeText {
-			err = writer.WriteField(field.Key, field.Value)
-			if err != nil {
-				return
-			}
+			s.Request.Upload[field.Key] = field.Value
 		} else if field.Type == enumFieldTypeFile {
-			err = writeFormDataFile(writer, &field)
-			if err != nil {
-				return
-			}
+			s.Request.Upload[field.Key] = field.Src
+		} else {
+			return errors.Errorf("make request body form data failed: unexpect field type: %v", field.Type)
 		}
 	}
-	err = writer.Close()
-	s.Request.Body = payload.String()
-	s.Request.Headers["Content-Type"] = writer.FormDataContentType()
-	return
-}
-
-func writeFormDataFile(writer *multipart.Writer, field *TField) error {
-	file, err := os.Open(field.Src)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-	formFile, err := writer.CreateFormFile(field.Key, filepath.Base(field.Src))
-	if err != nil {
-		return err
-	}
-	_, err = io.Copy(formFile, file)
-	return err
+	return nil
 }
 
 func (s *stepFromPostman) makeRequestBodyUrlEncoded(item *TItem) error {
@@ -482,7 +450,7 @@ func (s *stepFromPostman) makeRequestBodyUrlEncoded(item *TItem) error {
 	return nil
 }
 
-// TODO makeValidate from example response
+// TODO makeValidate from test scripts
 func (s *stepFromPostman) makeValidate(item *TItem) error {
 	return nil
 }

hrp/internal/convert/converter_postman_test.go

@@ -71,11 +71,8 @@ func TestMakeTestCaseFromCollection(t *testing.T) {
 	if !assert.Equal(t, "v1", tCase.TestSteps[0].Request.Params["k1"]) {
 		t.Fatal()
 	}
-	// check cookies (pass, postman collection doesn't contains cookies)
+	// check cookies (pass, postman collection doesn't contain cookies)
 	// check headers
-	if !assert.Contains(t, tCase.TestSteps[1].Request.Headers["Content-Type"], "multipart/form-data") {
-		t.Fatal()
-	}
 	if !assert.Equal(t, "application/x-www-form-urlencoded", tCase.TestSteps[2].Request.Headers["Content-Type"]) {
 		t.Fatal()
 	}
@@ -92,9 +89,6 @@ func TestMakeTestCaseFromCollection(t *testing.T) {
 	if !assert.Equal(t, nil, tCase.TestSteps[0].Request.Body) {
 		t.Fatal()
 	}
-	if !assert.NotEmpty(t, tCase.TestSteps[1].Request.Body) {
-		t.Fatal()
-	}
 	if !assert.Equal(t, map[string]string{"k1": "v1", "k2": "v2"}, tCase.TestSteps[2].Request.Body) {
 		t.Fatal()
 	}
@@ -121,9 +115,6 @@ func TestMakeTestCaseWithProfileOverride(t *testing.T) {
 		if step.Request.Method == "GET" && !assert.Len(t, step.Request.Headers, 1) {
 			t.Fatal()
 		}
-		if step.Request.Method == "POST" && !assert.Len(t, step.Request.Headers, 2) {
-			t.Fatal()
-		}
 		if !assert.Equal(t, "all original headers will be overridden", step.Request.Headers["Header1"]) {
 			t.Fatal()
 		}

hrp/step_request.go

@@ -52,6 +52,7 @@ type Request struct {
 	Timeout        float32                `json:"timeout,omitempty" yaml:"timeout,omitempty"`
 	AllowRedirects bool                   `json:"allow_redirects,omitempty" yaml:"allow_redirects,omitempty"`
 	Verify         bool                   `json:"verify,omitempty" yaml:"verify,omitempty"`
+	Upload         map[string]interface{} `json:"upload,omitempty" yaml:"upload,omitempty"`
 }
 
 func newRequestBuilder(parser *Parser, config *TConfig, stepRequest *Request) *requestBuilder {
@@ -246,6 +247,8 @@ func (r *requestBuilder) prepareBody(stepVariables map[string]interface{}) error
 		dataBytes = vv
 	case bytes.Buffer:
 		dataBytes = vv.Bytes()
+	case *builtin.TFormWriter:
+		dataBytes = vv.Payload.Bytes()
 	default: // unexpected body type
 		return errors.New("unexpected request body type")
 	}
@@ -256,6 +259,27 @@ func (r *requestBuilder) prepareBody(stepVariables map[string]interface{}) error
 	return nil
 }
 
+func prepareUpload(parser *Parser, step *TStep) (err error) {
+	if step.Request.Upload == nil {
+		return
+	}
+	step.Request.Upload, err = parser.ParseVariables(step.Request.Upload)
+	if err != nil {
+		return
+	}
+	if step.Variables == nil {
+		step.Variables = make(map[string]interface{})
+	}
+	step.Variables["m_upload"] = step.Request.Upload
+	step.Variables["m_encoder"] = fmt.Sprintf("${multipart_encoder($m_upload)}")
+	if step.Request.Headers == nil {
+		step.Request.Headers = make(map[string]string)
+	}
+	step.Request.Headers["Content-Type"] = "${multipart_content_type($m_encoder)}"
+	step.Request.Body = "$m_encoder"
+	return
+}
+
 func runStepRequest(r *SessionRunner, step *TStep) (stepResult *StepResult, err error) {
 	stepResult = &StepResult{
 		Name:        step.Name,
@@ -271,6 +295,11 @@ func runStepRequest(r *SessionRunner, step *TStep) (stepResult *StepResult, err
 		}
 	}()
 
+	err = prepareUpload(r.parser, step)
+	if err != nil {
+		return
+	}
+
 	// override step variables
 	stepVariables, err := r.MergeStepVariables(step.Variables)
 	if err != nil {
@@ -778,6 +807,12 @@ func (s *StepRequestWithOptionalArgs) WithBody(body interface{}) *StepRequestWit
 	return s
 }
 
+// WithUpload sets HTTP request body for uploading file(s).
+func (s *StepRequestWithOptionalArgs) WithUpload(upload map[string]interface{}) *StepRequestWithOptionalArgs {
+	s.step.Request.Upload = upload
+	return s
+}
+
 // TeardownHook adds a teardown hook for current teststep.
 func (s *StepRequestWithOptionalArgs) TeardownHook(hook string) *StepRequestWithOptionalArgs {
 	s.step.TeardownHooks = append(s.step.TeardownHooks, hook)