From 9a0247af0d0b9b4073629321e49814b3638ffbcc Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 11 Jun 2022 23:26:14 +0800 Subject: [PATCH 01/12] bump version to v4.1.3 --- docs/CHANGELOG.md | 4 ++++ hrp/internal/version/VERSION | 2 +- httprunner/__init__.py | 2 +- pyproject.toml | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index a19ddf85..d7cf60ef 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History +## v4.1.3 (2022-06-12) + + + ## v4.1.2 (2022-06-09) - feat: add Dockerfile diff --git a/hrp/internal/version/VERSION b/hrp/internal/version/VERSION index b572ab0b..8a58a0dc 100644 --- a/hrp/internal/version/VERSION +++ b/hrp/internal/version/VERSION @@ -1 +1 @@ -v4.1.2 \ No newline at end of file +v4.1.3 \ No newline at end of file diff --git a/httprunner/__init__.py b/httprunner/__init__.py index aba58277..6d1c0ef7 100644 --- a/httprunner/__init__.py +++ b/httprunner/__init__.py @@ -1,4 +1,4 @@ -__version__ = "v4.1.2" +__version__ = "v4.1.3" __description__ = "One-stop solution for HTTP(S) testing." diff --git a/pyproject.toml b/pyproject.toml index b1a8da31..2eb2a102 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "httprunner" -version = "v4.1.2" +version = "v4.1.3" description = "One-stop solution for HTTP(S) testing." license = "Apache-2.0" readme = "README.md" From f24c4538908ace4eb09ae9cebdd4681c0c9de049 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Sat, 11 Jun 2022 23:46:04 +0800 Subject: [PATCH 02/12] fix: pip upgrade httprunner when installing hrp --- docs/CHANGELOG.md | 2 +- scripts/install.sh | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d7cf60ef..f6956a8e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,7 +2,7 @@ ## v4.1.3 (2022-06-12) - +- fix: pip upgrade httprunner when installing hrp ## v4.1.2 (2022-06-09) diff --git a/scripts/install.sh b/scripts/install.sh index 686feffa..23dde6fb 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -135,6 +135,13 @@ function main() { hrp -v echo "$ hrp -h" hrp -h + echo + + if [[ -f $HOME/.hrp/venv/bin/pip3 ]]; then + echoInfo "Upgrade httprunner..." + echo "$ $HOME/.hrp/venv/bin/pip3 install --upgrade httprunner==$version" + $HOME/.hrp/venv/bin/pip3 install --upgrade httprunner==$version + fi } main From 182d2fd5d8a0f7deebd812541931c7817d2dedfd Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 13 Jun 2022 14:02:36 +0800 Subject: [PATCH 03/12] feat #1342: support specify custom python3 venv --- docs/CHANGELOG.md | 5 +- docs/cmd/hrp.md | 2 +- docs/cmd/hrp_boom.md | 2 +- docs/cmd/hrp_build.md | 2 +- docs/cmd/hrp_convert.md | 2 +- docs/cmd/hrp_pytest.md | 2 +- docs/cmd/hrp_run.md | 2 +- docs/cmd/hrp_startproject.md | 2 +- docs/cmd/hrp_wiki.md | 2 +- go.mod | 2 +- go.sum | 4 +- hrp/boomer.go | 9 +- hrp/build.go | 26 +--- hrp/cmd/boom.go | 3 + hrp/cmd/build.go | 7 + hrp/cmd/convert.go | 8 ++ hrp/cmd/pytest.go | 7 + hrp/cmd/root.go | 2 + hrp/cmd/run.go | 3 + hrp/cmd/scaffold.go | 2 +- hrp/internal/builtin/utils.go | 125 +++++++++++++----- hrp/internal/builtin/utils_unix.go | 77 +++++++++++ hrp/internal/builtin/utils_windows.go | 100 ++++++++++++++ hrp/internal/convert/converter_gotest.go | 7 +- hrp/internal/convert/converter_json.go | 14 +- hrp/internal/convert/converter_pytest.go | 13 +- hrp/internal/pytest/main.go | 13 +- hrp/internal/scaffold/examples_test.go | 8 +- hrp/internal/scaffold/main.go | 16 ++- .../templates/plugin/debugtalkGoTemplate | 4 +- .../templates/plugin/debugtalkPythonTemplate | 1 + .../templates/plugin/debugtalk_gen.go | 2 +- hrp/internal/version/init.go | 2 - hrp/plugin.go | 18 ++- hrp/runner.go | 10 +- 35 files changed, 381 insertions(+), 123 deletions(-) create mode 100644 hrp/internal/builtin/utils_unix.go create mode 100644 hrp/internal/builtin/utils_windows.go diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f6956a8e..a0c4ae05 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,7 +1,10 @@ # Release History -## v4.1.3 (2022-06-12) +## v4.1.3 (2022-06-13) +**go version** + +- feat #1342: support specify custom python3 venv - fix: pip upgrade httprunner when installing hrp ## v4.1.2 (2022-06-09) diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index 7a58c565..810a22ad 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -37,4 +37,4 @@ Copyright 2017 debugtalk * [hrp startproject](hrp_startproject.md) - create a scaffold project * [hrp wiki](hrp_wiki.md) - visit https://httprunner.com -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_boom.md b/docs/cmd/hrp_boom.md index a13c1168..7a9cfbca 100644 --- a/docs/cmd/hrp_boom.md +++ b/docs/cmd/hrp_boom.md @@ -42,4 +42,4 @@ hrp boom [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_build.md b/docs/cmd/hrp_build.md index 8b886d77..73dccf0b 100644 --- a/docs/cmd/hrp_build.md +++ b/docs/cmd/hrp_build.md @@ -28,4 +28,4 @@ hrp build $path ... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_convert.md b/docs/cmd/hrp_convert.md index 8105d41e..4160891c 100644 --- a/docs/cmd/hrp_convert.md +++ b/docs/cmd/hrp_convert.md @@ -22,4 +22,4 @@ hrp convert $path... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_pytest.md b/docs/cmd/hrp_pytest.md index 70be0989..b48bf18e 100644 --- a/docs/cmd/hrp_pytest.md +++ b/docs/cmd/hrp_pytest.md @@ -16,4 +16,4 @@ hrp pytest $path ... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index ad37cb40..cc859ed3 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -35,4 +35,4 @@ hrp run $path... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_startproject.md b/docs/cmd/hrp_startproject.md index 05c015da..78953953 100644 --- a/docs/cmd/hrp_startproject.md +++ b/docs/cmd/hrp_startproject.md @@ -21,4 +21,4 @@ hrp startproject $project_name [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_wiki.md b/docs/cmd/hrp_wiki.md index dcfd35ea..7283ba32 100644 --- a/docs/cmd/hrp_wiki.md +++ b/docs/cmd/hrp_wiki.md @@ -16,4 +16,4 @@ hrp wiki [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/go.mod b/go.mod index 633c73b7..6d70a855 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/go-openapi/spec v0.20.6 github.com/google/uuid v1.3.0 github.com/gorilla/websocket v1.4.1 - github.com/httprunner/funplugin v0.4.9 + github.com/httprunner/funplugin v0.5.0 github.com/jinzhu/copier v0.3.2 github.com/jmespath/go-jmespath v0.4.0 github.com/json-iterator/go v1.1.12 diff --git a/go.sum b/go.sum index 466fe49c..3f819360 100644 --- a/go.sum +++ b/go.sum @@ -253,8 +253,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/httprunner/funplugin v0.4.9 h1:gmF1sP5D4/nvdocqgOAyT3GpVDz3fL4ErZ17WHo8x9U= -github.com/httprunner/funplugin v0.4.9/go.mod h1:vPyeJIfbpGe0epZZtAV0wCn16gLY9+imSw/zfxq0Lcc= +github.com/httprunner/funplugin v0.5.0 h1:Laoe8URu71qeyST9wvRtGSkDWc8Y3T1IrnvFSTHmO84= +github.com/httprunner/funplugin v0.5.0/go.mod h1:vPyeJIfbpGe0epZZtAV0wCn16gLY9+imSw/zfxq0Lcc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= diff --git a/hrp/boomer.go b/hrp/boomer.go index caef351c..839ebd15 100644 --- a/hrp/boomer.go +++ b/hrp/boomer.go @@ -29,9 +29,16 @@ type HRPBoomer struct { pluginsMutex *sync.RWMutex // avoid data race } -func (b *HRPBoomer) SetClientTransport() { +func (b *HRPBoomer) SetClientTransport() *HRPBoomer { // set client transport for high concurrency load testing b.hrpRunner.SetClientTransport(b.GetSpawnCount(), b.GetDisableKeepAlive(), b.GetDisableCompression()) + return b +} + +// SetPython3Venv specifies python3 venv. +func (b *HRPBoomer) SetPython3Venv(venv string) *HRPBoomer { + b.hrpRunner.SetPython3Venv(venv) + return b } // Run starts to run load test for one or multiple testcases. diff --git a/hrp/build.go b/hrp/build.go index 3184fa3c..94692317 100644 --- a/hrp/build.go +++ b/hrp/build.go @@ -12,16 +12,15 @@ import ( "strings" "text/template" + "github.com/httprunner/funplugin/shared" "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/httprunner/funplugin/shared" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/version" ) const ( - funppy = `import funppy` fungo = `"github.com/httprunner/funplugin/fungo"` regexPythonFunctionName = `def ([a-zA-Z_]\w*)\(.*\)` regexGoImports = `import \(([\s\S]*?)\)` @@ -38,7 +37,6 @@ var goTemplate string type TemplateContent struct { Version string // hrp version - Fun string // funplugin package Regexps *Regexps // match import/function Imports []string // python/go import FromImports []string // python from...import... @@ -81,7 +79,7 @@ func (t *TemplateContent) parseGoContent(path string) error { } // import fungo package if !builtin.Contains(t.Imports, fungo) { - t.Imports = append(t.Imports, t.Fun) + t.Imports = append(t.Imports, fungo) } // parse function name @@ -155,15 +153,11 @@ func (t *TemplateContent) parsePyContent(path string) error { // function content t.Functions = append(t.Functions, strings.Trim(content, "\n")) - // import funppy - if !builtin.Contains(t.Imports, t.Fun) { - t.Imports = append(t.Imports, t.Fun) - } return nil } -func (t *TemplateContent) genDebugTalk(path string, templ string) error { - file, err := os.Create(path) +func (t *TemplateContent) genDebugTalk(output string, templ string) error { + file, err := os.Create(output) if err != nil { log.Error().Err(err).Msg("open file failed") return err @@ -178,9 +172,9 @@ func (t *TemplateContent) genDebugTalk(path string, templ string) error { } err = writer.Flush() if err == nil { - log.Info().Str("path", path).Msg("generate debugtalk success") + log.Info().Str("output", output).Msg("generate debugtalk success") } else { - log.Error().Str("path", path).Msg("generate debugtalk failed") + log.Error().Str("output", output).Msg("generate debugtalk failed") } return err } @@ -189,7 +183,6 @@ func (t *TemplateContent) genDebugTalk(path string, templ string) error { func buildGo(path string, output string) error { templateContent := &TemplateContent{ Version: version.VERSION, - Fun: fungo, Regexps: &Regexps{ Import: regexp.MustCompile(regexGoImport), Imports: regexp.MustCompile(regexGoImports), @@ -260,7 +253,6 @@ func buildGo(path string, output string) error { func buildPy(path string, output string) error { templateContent := &TemplateContent{ Version: version.VERSION, - Fun: funppy, Regexps: &Regexps{ FunctionName: regexp.MustCompile(regexPythonFunctionName), }, @@ -271,12 +263,6 @@ func buildPy(path string, output string) error { return err } - // ensure installation of packages - _, err = builtin.EnsurePython3Venv(templateContent.Packages...) - if err != nil { - return err - } - // check the syntax of debugtalk.py err = builtin.CheckPythonScriptSyntax(path) if err != nil { diff --git a/hrp/cmd/boom.go b/hrp/cmd/boom.go index 5ae038e1..ca48d13b 100644 --- a/hrp/cmd/boom.go +++ b/hrp/cmd/boom.go @@ -54,6 +54,9 @@ var boomCmd = &cobra.Command{ hrpBoomer.SetDisableKeepAlive(boomArgs.DisableKeepalive) hrpBoomer.SetDisableCompression(boomArgs.DisableCompression) hrpBoomer.SetClientTransport() + if venv != "" { + hrpBoomer.SetPython3Venv(venv) + } hrpBoomer.EnableCPUProfile(boomArgs.CPUProfile, boomArgs.CPUProfileDuration) hrpBoomer.EnableMemoryProfile(boomArgs.MemoryProfile, boomArgs.MemoryProfileDuration) hrpBoomer.EnableGracefulQuit() diff --git a/hrp/cmd/build.go b/hrp/cmd/build.go index 3c8848e3..9a8c2bb8 100644 --- a/hrp/cmd/build.go +++ b/hrp/cmd/build.go @@ -1,9 +1,11 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/httprunner/httprunner/v4/hrp" + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" ) var buildCmd = &cobra.Command{ @@ -17,6 +19,11 @@ var buildCmd = &cobra.Command{ setLogLevel(logLevel) }, RunE: func(cmd *cobra.Command, args []string) error { + err := builtin.PrepareVenv(venv) + if err != nil { + log.Error().Err(err).Msg("prepare python3 venv failed") + return err + } return hrp.BuildPlugin(args[0], output) }, } diff --git a/hrp/cmd/convert.go b/hrp/cmd/convert.go index 83ba88bb..ee43c2de 100644 --- a/hrp/cmd/convert.go +++ b/hrp/cmd/convert.go @@ -3,8 +3,10 @@ package cmd import ( "errors" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/convert" ) @@ -32,6 +34,12 @@ var convertCmd = &cobra.Command{ if toPyTestFlag { flagCount++ outputType = convert.OutputTypePyTest + + err := builtin.PrepareVenv(venv) + if err != nil { + log.Error().Err(err).Msg("prepare python3 venv failed") + return err + } } if flagCount > 1 { return errors.New("please specify at most one conversion flag") diff --git a/hrp/cmd/pytest.go b/hrp/cmd/pytest.go index fd7732fb..4d49b2be 100644 --- a/hrp/cmd/pytest.go +++ b/hrp/cmd/pytest.go @@ -1,8 +1,10 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/pytest" ) @@ -15,6 +17,11 @@ var pytestCmd = &cobra.Command{ }, DisableFlagParsing: true, // allow to pass any args to pytest RunE: func(cmd *cobra.Command, args []string) error { + err := builtin.PrepareVenv(venv) + if err != nil { + log.Error().Err(err).Msg("prepare python3 venv failed") + return err + } return pytest.RunPytest(args) }, } diff --git a/hrp/cmd/root.go b/hrp/cmd/root.go index 2909ea15..698d0598 100644 --- a/hrp/cmd/root.go +++ b/hrp/cmd/root.go @@ -48,6 +48,7 @@ Copyright 2017 debugtalk`, var ( logLevel string logJSON bool + venv string ) // Execute adds all child commands to the root command and sets flags appropriately. @@ -55,6 +56,7 @@ var ( func Execute() { rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", "INFO", "set log level") rootCmd.PersistentFlags().BoolVar(&logJSON, "log-json", false, "set log to json format") + rootCmd.PersistentFlags().StringVar(&venv, "venv", "", "specify python3 venv path") if err := rootCmd.Execute(); err != nil { os.Exit(1) diff --git a/hrp/cmd/run.go b/hrp/cmd/run.go index 500e595b..5c7bcd90 100644 --- a/hrp/cmd/run.go +++ b/hrp/cmd/run.go @@ -41,6 +41,9 @@ var runCmd = &cobra.Command{ if pluginLogOn { runner.SetPluginLogOn() } + if venv != "" { + runner.SetPython3Venv(venv) + } if proxyUrl != "" { runner.SetProxyUrl(proxyUrl) } diff --git a/hrp/cmd/scaffold.go b/hrp/cmd/scaffold.go index c51281ed..86830725 100644 --- a/hrp/cmd/scaffold.go +++ b/hrp/cmd/scaffold.go @@ -34,7 +34,7 @@ var scaffoldCmd = &cobra.Command{ pluginType = scaffold.Py // default } - err := scaffold.CreateScaffold(args[0], pluginType, force) + err := scaffold.CreateScaffold(args[0], pluginType, venv, force) if err != nil { log.Error().Err(err).Msg("create scaffold project failed") os.Exit(1) diff --git a/hrp/internal/builtin/utils.go b/hrp/internal/builtin/utils.go index 93a4b64b..0fd52db5 100644 --- a/hrp/internal/builtin/utils.go +++ b/hrp/internal/builtin/utils.go @@ -13,12 +13,13 @@ import ( "strconv" "strings" - "github.com/httprunner/funplugin/shared" + "github.com/httprunner/funplugin/fungo" "github.com/pkg/errors" "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" "github.com/httprunner/httprunner/v4/hrp/internal/json" + "github.com/httprunner/httprunner/v4/hrp/internal/version" ) func Dump2JSON(data interface{}, path string) error { @@ -88,24 +89,104 @@ func FormatResponse(raw interface{}) interface{} { return formattedResponse } -func EnsurePython3Venv(packages ...string) (string, error) { - // create python venv +var Python3Executable string = "python3" // system default python3 + +func PrepareVenv(venv string) error { + defer func() { + log.Info().Str("Python3Executable", Python3Executable).Msg("set python3 executable path") + }() + + // specify python3 venv + if venv != "" { + Python3Executable = getPython3Executable(venv) + return nil + } + + // not specify python3 venv, create default venv + python3, err := createDefaultPython3Venv() + if err != nil { + return errors.Wrap(err, "create default python3 venv failed") + } + Python3Executable = python3 + + return nil +} + +// create default python3 venv located in $HOME/.hrp/venv +func createDefaultPython3Venv() (string, error) { + log.Info().Msg("create default python3 venv at $HOME/.hrp/venv") home, err := os.UserHomeDir() if err != nil { return "", errors.Wrap(err, "get user home dir failed") } - venvDir := filepath.Join(home, ".hrp", "venv") - python3, err := shared.EnsurePython3Venv(venvDir, packages...) - if err != nil { - return "", errors.Wrap(err, "ensure python3 venv failed") + venv := filepath.Join(home, ".hrp", "venv") + packages := []string{ + fmt.Sprintf("funppy==%s", fungo.Version), + fmt.Sprintf("httprunner==%s", version.VERSION), + } + return EnsurePython3Venv(venv, packages...) +} + +func ExecPython3Command(cmdName string, args ...string) error { + args = append([]string{"-m", cmdName}, args...) + return ExecCommand(Python3Executable, args...) +} + +func InstallPythonPackage(python3 string, pkg string) (err error) { + var pkgName string + if strings.Contains(pkg, "==") { + // funppy==0.4.2 + pkgInfo := strings.Split(pkg, "==") + pkgName = pkgInfo[0] + } else if strings.Contains(pkg, ">=") { + // httprunner>=4.0.0-beta + pkgInfo := strings.Split(pkg, ">=") + pkgName = pkgInfo[0] + } else { + pkgName = pkg } - return python3, nil + defer func() { + if err == nil { + // check package version + if out, err := exec.Command( + python3, "-c", fmt.Sprintf("import %s; print(%s.__version__)", pkgName, pkgName), + ).Output(); err == nil { + log.Info(). + Str("name", pkgName). + Str("version", strings.TrimSpace(string(out))). + Msg("python package is ready") + } + } + }() + + // check if package installed + err = exec.Command(python3, "-c", fmt.Sprintf("import %s", pkgName)).Run() + if err == nil { + return nil + } + + // check if pip available + err = ExecCommand(python3, "-m", "pip", "--version") + if err != nil { + log.Warn().Msg("pip is not available") + return errors.Wrap(err, "pip is not available") + } + + log.Info().Str("package", pkg).Msg("installing python package") + + // install package + err = ExecCommand(python3, "-m", "pip", "install", "--upgrade", pkg, + "--quiet", "--disable-pip-version-check") + if err != nil { + return errors.Wrap(err, "pip install package failed") + } + + return nil } func CheckPythonScriptSyntax(path string) error { - err := ExecCommand("python3", "-m", "py_compile", path) - return err + return ExecCommand("python3", "-m", "py_compile", path) } func ExecCommandInDir(cmd *exec.Cmd, dir string) error { @@ -125,30 +206,6 @@ func ExecCommandInDir(cmd *exec.Cmd, dir string) error { return nil } -func ExecCommand(cmdName string, args ...string) error { - cmd := exec.Command(cmdName, args...) - log.Info().Str("cmd", cmd.String()).Msg("exec command") - - // print output with colors - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - // add cmd dir path to PATH - PATH := fmt.Sprintf("%s:%s", filepath.Dir(cmdName), os.Getenv("PATH")) - if err := os.Setenv("PATH", PATH); err != nil { - log.Error().Err(err).Msg("failed to add cmd dir path to $PATH") - return err - } - - err := cmd.Run() - if err != nil { - log.Error().Err(err).Msg("exec command failed") - return err - } - - return err -} - func CreateFolder(folderPath string) error { log.Info().Str("path", folderPath).Msg("create folder") err := os.MkdirAll(folderPath, os.ModePerm) diff --git a/hrp/internal/builtin/utils_unix.go b/hrp/internal/builtin/utils_unix.go new file mode 100644 index 00000000..510ac0c0 --- /dev/null +++ b/hrp/internal/builtin/utils_unix.go @@ -0,0 +1,77 @@ +// +build darwin linux + +package builtin + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + +func getPython3Executable(venvDir string) string { + return filepath.Join(venvDir, "bin", "python3") +} + +// EnsurePython3Venv ensures python3 venv for hashicorp python plugin +// venvDir should be directory path of target venv +func EnsurePython3Venv(venvDir string, packages ...string) (python3 string, err error) { + python3 = getPython3Executable(venvDir) + + log.Info(). + Str("python3", python3). + Interface("packages", packages). + Msg("ensure python3 venv") + + // check if python3 venv is available + if err := exec.Command(python3, "--version").Run(); err != nil { + // python3 venv not available, create one + // check if system python3 is available + if err := ExecCommand("python3", "--version"); err != nil { + return "", errors.Wrap(err, "python3 not found") + } + + // check if .venv exists + if _, err := os.Stat(venvDir); err == nil { + // .venv exists, remove first + if err := ExecCommand("rm", "-rf", venvDir); err != nil { + return "", errors.Wrap(err, "remove existed venv failed") + } + } + + // create python3 .venv + if err := ExecCommand("python3", "-m", "venv", venvDir); err != nil { + return "", errors.Wrap(err, "create python3 venv failed") + } + } + + // install default python packages + for _, pkg := range packages { + err := InstallPythonPackage(python3, pkg) + if err != nil { + return "", errors.Wrap(err, fmt.Sprintf("pip install %s failed", pkg)) + } + } + + return python3, nil +} + +func ExecCommand(cmdName string, args ...string) error { + cmd := exec.Command(cmdName, args...) + log.Info().Str("cmd", cmd.String()).Msg("exec command") + + // print output with colors + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + log.Error().Err(err).Msg("exec command failed") + return err + } + + return nil +} diff --git a/hrp/internal/builtin/utils_windows.go b/hrp/internal/builtin/utils_windows.go new file mode 100644 index 00000000..8501f082 --- /dev/null +++ b/hrp/internal/builtin/utils_windows.go @@ -0,0 +1,100 @@ +// +build windows + +package builtin + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + +func getPython3Executable(venvDir string) string { + return filepath.Join(venvDir, "Scripts", "python3.exe") +} + +// EnsurePython3Venv ensures python3 venv for hashicorp python plugin +// venvDir should be directory path of target venv +func EnsurePython3Venv(venvDir string, packages ...string) (python3 string, err error) { + python3 = getPython3Executable(venvDir) + log.Info(). + Str("python3", python3). + Interface("packages", packages). + Msg("ensure python3 venv") + + systemPython := "python3" + + // check if python3 venv is available + if err := exec.Command("cmd", "/c", python3, "--version").Run(); err != nil { + // python3 venv not available, create one + // check if system python3 is available + if err := ExecCommand(systemPython, "--version"); err != nil { + if err := ExecCommand("python", "--version"); err != nil { + return "", errors.Wrap(err, "python3 not found") + } + systemPython = "python" + } + + // check if .venv exists + if _, err := os.Stat(venvDir); err == nil { + // .venv exists, remove first + if err := ExecCommand("del", "/q", venvDir); err != nil { + return "", errors.Wrap(err, "remove existed venv failed") + } + } + + // create python3 .venv + // notice: --symlinks should be specified for windows + // https://github.com/actions/virtual-environments/issues/2690 + if err := ExecCommand(systemPython, "-m", "venv", "--symlinks", venvDir); err != nil { + // fix: failed to symlink on Windows + log.Warn().Msg("failed to create python3 .venv by using --symlinks, try to use --copies") + if err := ExecCommand(systemPython, "-m", "venv", "--copies", venvDir); err != nil { + return "", errors.Wrap(err, "create python3 venv failed") + } + } + + // fix: python3 doesn't exist in .venv on Windows + if _, err := os.Stat(python3); err != nil { + log.Warn().Msg("python3 doesn't exist, try to link python") + err := os.Link(filepath.Join(venvDir, "Scripts", "python.exe"), python3) + if err != nil { + return "", errors.Wrap(err, "python3 doesn't exist in .venv") + } + } + } + + // install default python packages + for _, pkg := range packages { + err := InstallPythonPackage(python3, pkg) + if err != nil { + return "", errors.Wrap(err, fmt.Sprintf("pip install %s failed", pkg)) + } + } + + return python3, nil +} + +func ExecCommand(cmdName string, args ...string) error { + // "cmd /c" carries out the command specified by string and then stops + // refer: https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/cmd + cmdStr := fmt.Sprintf("%s %s", cmdName, strings.Join(args, " ")) + cmd := exec.Command("cmd", "/c", cmdStr) + log.Info().Str("cmd", cmd.String()).Msg("exec command") + + // print output with colors + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + log.Error().Err(err).Msg("exec command failed") + return err + } + + return nil +} diff --git a/hrp/internal/convert/converter_gotest.go b/hrp/internal/convert/converter_gotest.go index 863da231..04897c76 100644 --- a/hrp/internal/convert/converter_gotest.go +++ b/hrp/internal/convert/converter_gotest.go @@ -48,12 +48,7 @@ func convert2GoTestScripts(paths ...string) error { } // format pytest scripts with black - python3, err := builtin.EnsurePython3Venv("black") - if err != nil { - return err - } - args := append([]string{"-m", "black"}, pytestPaths...) - return builtin.ExecCommand(python3, args...) + return builtin.ExecPython3Command("black", pytestPaths...) } //go:embed testcase.tmpl diff --git a/hrp/internal/convert/converter_json.go b/hrp/internal/convert/converter_json.go index fc380142..f55a7f5c 100644 --- a/hrp/internal/convert/converter_json.go +++ b/hrp/internal/convert/converter_json.go @@ -1,13 +1,10 @@ package convert import ( - "fmt" - "github.com/pkg/errors" "github.com/httprunner/httprunner/v4/hrp" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" - "github.com/httprunner/httprunner/v4/hrp/internal/version" ) func NewConverterJSON(converter *TCaseConverter) *ConverterJSON { @@ -51,7 +48,7 @@ func (c *ConverterJSON) ToYAML() (string, error) { } func (c *ConverterJSON) ToGoTest() (string, error) { - //TODO implement me + // TODO implement me return "", errors.New("convert from json testcase to gotest scripts is not supported yet") } @@ -60,13 +57,8 @@ func (c *ConverterJSON) ToPyTest() (string, error) { } func (c *ConverterJSON) MakePyTestScript() (string, error) { - httprunner := fmt.Sprintf("httprunner>=%s", version.HttpRunnerMinVersion) - python3, err := builtin.EnsurePython3Venv(httprunner) - if err != nil { - return "", err - } - args := append([]string{"-m", "httprunner", "make"}, c.converter.InputPath) - err = builtin.ExecCommand(python3, args...) + args := append([]string{"make"}, c.converter.InputPath) + err := builtin.ExecPython3Command("httprunner", args...) if err != nil { return "", err } diff --git a/hrp/internal/convert/converter_pytest.go b/hrp/internal/convert/converter_pytest.go index 8c094900..75c1490f 100644 --- a/hrp/internal/convert/converter_pytest.go +++ b/hrp/internal/convert/converter_pytest.go @@ -1,19 +1,10 @@ package convert import ( - "fmt" - "github.com/httprunner/httprunner/v4/hrp/internal/builtin" - "github.com/httprunner/httprunner/v4/hrp/internal/version" ) func convert2PyTestScripts(paths ...string) error { - httprunner := fmt.Sprintf("httprunner>=%s", version.HttpRunnerMinVersion) - python3, err := builtin.EnsurePython3Venv(httprunner) - if err != nil { - return err - } - - args := append([]string{"-m", "httprunner", "make"}, paths...) - return builtin.ExecCommand(python3, args...) + args := append([]string{"make"}, paths...) + return builtin.ExecPython3Command("httprunner", args...) } diff --git a/hrp/internal/pytest/main.go b/hrp/internal/pytest/main.go index a1c0c88c..dde5f0ae 100644 --- a/hrp/internal/pytest/main.go +++ b/hrp/internal/pytest/main.go @@ -1,11 +1,8 @@ package pytest import ( - "fmt" - "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/sdk" - "github.com/httprunner/httprunner/v4/hrp/internal/version" ) func RunPytest(args []string) error { @@ -14,12 +11,6 @@ func RunPytest(args []string) error { Action: "hrp pytest", }) - httprunner := fmt.Sprintf("httprunner>=%s", version.HttpRunnerMinVersion) - python3, err := builtin.EnsurePython3Venv(httprunner) - if err != nil { - return err - } - - args = append([]string{"-m", "httprunner", "run"}, args...) - return builtin.ExecCommand(python3, args...) + args = append([]string{"run"}, args...) + return builtin.ExecPython3Command("httprunner", args...) } diff --git a/hrp/internal/scaffold/examples_test.go b/hrp/internal/scaffold/examples_test.go index d480ffb6..0590994d 100644 --- a/hrp/internal/scaffold/examples_test.go +++ b/hrp/internal/scaffold/examples_test.go @@ -6,25 +6,25 @@ import ( func TestGenDemoExamples(t *testing.T) { dir := "../../../examples/demo-with-go-plugin" - err := CreateScaffold(dir, Go, true) + err := CreateScaffold(dir, Go, "", true) if err != nil { t.Fatal() } dir = "../../../examples/demo-with-py-plugin" - err = CreateScaffold(dir, Py, true) + err = CreateScaffold(dir, Py, "", true) if err != nil { t.Fatal() } dir = "../../../examples/demo-without-plugin" - err = CreateScaffold(dir, Ignore, true) + err = CreateScaffold(dir, Ignore, "", true) if err != nil { t.Fatal() } dir = "../../../examples/demo-empty-project" - err = CreateScaffold(dir, Empty, true) + err = CreateScaffold(dir, Empty, "", true) if err != nil { t.Fatal() } diff --git a/hrp/internal/scaffold/main.go b/hrp/internal/scaffold/main.go index 04ad8ac5..ed9405a6 100644 --- a/hrp/internal/scaffold/main.go +++ b/hrp/internal/scaffold/main.go @@ -8,6 +8,7 @@ import ( "path/filepath" "time" + "github.com/httprunner/funplugin/fungo" "github.com/pkg/errors" "github.com/rs/zerolog/log" @@ -51,7 +52,7 @@ func CopyFile(templateFile, targetFile string) error { return nil } -func CreateScaffold(projectName string, pluginType PluginType, force bool) error { +func CreateScaffold(projectName string, pluginType PluginType, venv string, force bool) error { // report event sdk.SendEvent(sdk.EventTracking{ Category: "Scaffold", @@ -165,7 +166,7 @@ func CreateScaffold(projectName string, pluginType PluginType, force bool) error // create debugtalk function plugin switch pluginType { case Py: - return createPythonPlugin(projectName) + return createPythonPlugin(projectName, venv) case Go: return createGoPlugin(projectName) } @@ -194,7 +195,7 @@ func createGoPlugin(projectName string) error { return nil } -func createPythonPlugin(projectName string) error { +func createPythonPlugin(projectName, venv string) error { log.Info().Msg("start to create hashicorp python plugin") // create debugtalk.py @@ -204,7 +205,14 @@ func createPythonPlugin(projectName string) error { return errors.Wrap(err, "copy file failed") } - _, err = builtin.EnsurePython3Venv("funppy") + if venv == "" { + venv = filepath.Join(projectName, ".venv") + } + packages := []string{ + fmt.Sprintf("funppy==%s", fungo.Version), + fmt.Sprintf("httprunner==%s", version.VERSION), + } + _, err = builtin.EnsurePython3Venv(venv, packages...) if err != nil { return err } diff --git a/hrp/internal/scaffold/templates/plugin/debugtalkGoTemplate b/hrp/internal/scaffold/templates/plugin/debugtalkGoTemplate index d5b096bc..be50fa2c 100644 --- a/hrp/internal/scaffold/templates/plugin/debugtalkGoTemplate +++ b/hrp/internal/scaffold/templates/plugin/debugtalkGoTemplate @@ -2,12 +2,12 @@ package main import ( - "github.com/httprunner/funplugin/fungo" + "github.com/httprunner/funplugin/fungo" ) func main() { {{- range $functionName := .FunctionNames }} - fungo.Register("{{ $functionName }}", {{ $functionName }}) + fungo.Register("{{ $functionName }}", {{ $functionName }}) {{- end }} fungo.Serve() } diff --git a/hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate b/hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate index dc16657a..b892dd30 100644 --- a/hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate +++ b/hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate @@ -12,6 +12,7 @@ {{ end }} if __name__ == "__main__": + import funppy {{- range $functionName := .FunctionNames }} funppy.register("{{ $functionName }}", {{ $functionName }}) {{- end }} diff --git a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go index 1e6ceac9..df7727c3 100644 --- a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go +++ b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go @@ -1,4 +1,4 @@ -// NOTE: Generated By hrp v4.1.2, DO NOT EDIT! +// NOTE: Generated By hrp v4.1.3, DO NOT EDIT! package main import ( diff --git a/hrp/internal/version/init.go b/hrp/internal/version/init.go index 836ffca7..4887463b 100644 --- a/hrp/internal/version/init.go +++ b/hrp/internal/version/init.go @@ -6,5 +6,3 @@ import ( //go:embed VERSION var VERSION string - -const HttpRunnerMinVersion = "v4.1.0" diff --git a/hrp/plugin.go b/hrp/plugin.go index 5053b2e5..91bdedd8 100644 --- a/hrp/plugin.go +++ b/hrp/plugin.go @@ -11,6 +11,7 @@ import ( "github.com/httprunner/funplugin" "github.com/rs/zerolog/log" + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/sdk" ) @@ -25,7 +26,7 @@ const ( const projectInfoFile = "proj.json" // used for ensuring root project -func initPlugin(path string, logOn bool) (plugin funplugin.IPlugin, err error) { +func initPlugin(path, venv string, logOn bool) (plugin funplugin.IPlugin, err error) { // plugin file not found if path == "" { return nil, nil @@ -35,6 +36,8 @@ func initPlugin(path string, logOn bool) (plugin funplugin.IPlugin, err error) { return nil, nil } + pluginOptions := []funplugin.Option{funplugin.WithLogOn(logOn)} + if strings.HasSuffix(pluginPath, ".py") { // register funppy plugin genPyPluginPath := filepath.Join(filepath.Dir(pluginPath), PluginPySourceGenFile) @@ -44,10 +47,21 @@ func initPlugin(path string, logOn bool) (plugin funplugin.IPlugin, err error) { return nil, nil } pluginPath = genPyPluginPath + + // priority: specified > projectDir/.venv > $HOME/.hrp/venv + if venv == "" && builtin.IsFolderPathExists(filepath.Join(filepath.Dir(pluginPath), ".venv")) { + venv = filepath.Join(filepath.Dir(pluginPath), ".venv") + } + err = builtin.PrepareVenv(venv) + if err != nil { + log.Error().Err(err).Msg("prepare python3 venv failed") + return + } + pluginOptions = append(pluginOptions, funplugin.WithPython3(builtin.Python3Executable)) } // found plugin file - plugin, err = funplugin.Init(pluginPath, funplugin.WithLogOn(logOn)) + plugin, err = funplugin.Init(pluginPath, pluginOptions...) if err != nil { log.Error().Err(err).Msgf("init plugin failed: %s", pluginPath) return diff --git a/hrp/runner.go b/hrp/runner.go index 8290b1bf..46eb2af8 100644 --- a/hrp/runner.go +++ b/hrp/runner.go @@ -58,6 +58,7 @@ type HRPRunner struct { httpStatOn bool requestsLogOn bool pluginLogOn bool + venv string saveTests bool genHTMLReport bool httpClient *http.Client @@ -116,6 +117,13 @@ func (r *HRPRunner) SetPluginLogOn() *HRPRunner { return r } +// SetPython3Venv specifies python3 venv. +func (r *HRPRunner) SetPython3Venv(venv string) *HRPRunner { + log.Info().Str("venv", venv).Msg("[init] SetPython3Venv") + r.venv = venv + return r +} + // SetProxyUrl configures the proxy URL, which is usually used to capture HTTP packets for debugging. func (r *HRPRunner) SetProxyUrl(proxyUrl string) *HRPRunner { log.Info().Str("proxyUrl", proxyUrl).Msg("[init] SetProxyUrl") @@ -235,7 +243,7 @@ func (r *HRPRunner) newCaseRunner(testcase *TestCase) (*testCaseRunner, error) { } // init parser plugin - plugin, err := initPlugin(testcase.Config.Path, r.pluginLogOn) + plugin, err := initPlugin(testcase.Config.Path, r.venv, r.pluginLogOn) if err != nil { return nil, errors.Wrap(err, "init plugin failed") } From 9de49a1d299b1adf7a194f9741a00bfb19d7546c Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 13 Jun 2022 23:27:25 +0800 Subject: [PATCH 04/12] refactor: build plugin --- docs/CHANGELOG.md | 2 + examples/demo-with-go-plugin/proj.json | 2 +- .../demo-with-go-plugin/testcases/demo.json | 2 +- .../demo-with-py-plugin/.debugtalk_gen.py | 62 +---- examples/demo-with-py-plugin/proj.json | 2 +- .../demo-with-py-plugin/testcases/demo.json | 2 +- hrp/build.go | 238 ++++++------------ hrp/internal/builtin/utils.go | 4 - .../templates/plugin/.debugtalk_gen.py | 62 +---- .../templates/plugin/debugtalkPythonTemplate | 15 +- .../templates/plugin/debugtalk_gen.go | 2 +- 11 files changed, 93 insertions(+), 300 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index a0c4ae05..1669752f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,6 +5,8 @@ **go version** - feat #1342: support specify custom python3 venv +- feat: support python3 venv priority, specified > projectDir/.venv > $HOME/.hrp/venv +- refactor: build plugin mechanism - fix: pip upgrade httprunner when installing hrp ## v4.1.2 (2022-06-09) diff --git a/examples/demo-with-go-plugin/proj.json b/examples/demo-with-go-plugin/proj.json index 477b6be7..48b9a7c2 100644 --- a/examples/demo-with-go-plugin/proj.json +++ b/examples/demo-with-go-plugin/proj.json @@ -1,5 +1,5 @@ { "project_name": "demo-with-go-plugin", - "create_time": "2022-06-09T23:39:14.781583+08:00", + "create_time": "2022-06-13T23:28:24.920066+08:00", "hrp_version": "v4.1.2" } diff --git a/examples/demo-with-go-plugin/testcases/demo.json b/examples/demo-with-go-plugin/testcases/demo.json index 1bb63ed8..8e50e2aa 100644 --- a/examples/demo-with-go-plugin/testcases/demo.json +++ b/examples/demo-with-go-plugin/testcases/demo.json @@ -173,4 +173,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/examples/demo-with-py-plugin/.debugtalk_gen.py b/examples/demo-with-py-plugin/.debugtalk_gen.py index 880c145f..653fed67 100644 --- a/examples/demo-with-py-plugin/.debugtalk_gen.py +++ b/examples/demo-with-py-plugin/.debugtalk_gen.py @@ -1,67 +1,15 @@ # NOTE: Generated By hrp v4.1.2, DO NOT EDIT! -import logging -import time -import funppy +import sys +import os -from typing import List +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -def get_user_agent(): - return "hrp/funppy" - - -def sleep(n_secs): - time.sleep(n_secs) - - -def sum(*args): - result = 0 - for arg in args: - result += arg - return result - - -def sum_ints(*args: List[int]) -> int: - result = 0 - for arg in args: - result += arg - return result - - -def sum_two_int(a: int, b: int) -> int: - return a + b - - -def sum_two_string(a: str, b: str) -> str: - return a + b - - -def sum_strings(*args: List[str]) -> str: - result = "" - for arg in args: - result += arg - return result - - -def concatenate(*args: List[str]) -> str: - result = "" - for arg in args: - result += str(arg) - return result - - -def setup_hook_example(name): - logging.warning("setup_hook_example") - return f"setup_hook_example: {name}" - - -def teardown_hook_example(name): - logging.warning("teardown_hook_example") - return f"teardown_hook_example: {name}" +from debugtalk import * if __name__ == "__main__": + import funppy funppy.register("get_user_agent", get_user_agent) funppy.register("sleep", sleep) funppy.register("sum", sum) diff --git a/examples/demo-with-py-plugin/proj.json b/examples/demo-with-py-plugin/proj.json index 1cf81b3a..457cef97 100644 --- a/examples/demo-with-py-plugin/proj.json +++ b/examples/demo-with-py-plugin/proj.json @@ -1,5 +1,5 @@ { "project_name": "demo-with-py-plugin", - "create_time": "2022-06-09T23:39:14.922843+08:00", + "create_time": "2022-06-13T23:28:25.254595+08:00", "hrp_version": "v4.1.2" } diff --git a/examples/demo-with-py-plugin/testcases/demo.json b/examples/demo-with-py-plugin/testcases/demo.json index 1bb63ed8..8e50e2aa 100644 --- a/examples/demo-with-py-plugin/testcases/demo.json +++ b/examples/demo-with-py-plugin/testcases/demo.json @@ -173,4 +173,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/hrp/build.go b/hrp/build.go index 94692317..6c97a207 100644 --- a/hrp/build.go +++ b/hrp/build.go @@ -4,13 +4,12 @@ import ( "bufio" _ "embed" "fmt" - "io" + "html/template" "os" "os/exec" "path/filepath" "regexp" "strings" - "text/template" "github.com/httprunner/funplugin/shared" "github.com/pkg/errors" @@ -20,156 +19,57 @@ import ( "github.com/httprunner/httprunner/v4/hrp/internal/version" ) -const ( - fungo = `"github.com/httprunner/funplugin/fungo"` - regexPythonFunctionName = `def ([a-zA-Z_]\w*)\(.*\)` - regexGoImports = `import \(([\s\S]*?)\)` - regexGoImport = `import (\"[\s\S]*\")` - regexGoFunctionName = `func ([A-Z][a-zA-Z_]\w*)\(.*\)` - regexGoFunctionContent = `func [\s\S]*?\n}` -) - //go:embed internal/scaffold/templates/plugin/debugtalkPythonTemplate var pyTemplate string //go:embed internal/scaffold/templates/plugin/debugtalkGoTemplate var goTemplate string -type TemplateContent struct { +// regex for finding all function names +var ( + regexPyFunctionName = regexp.MustCompile(`def ([a-zA-Z_]\w*)\(.*\)`) + regexGoFunctionName = regexp.MustCompile(`func ([A-Z][a-zA-Z_]\w*)\(.*\)`) +) + +type pluginTemplateContent struct { Version string // hrp version - Regexps *Regexps // match import/function - Imports []string // python/go import - FromImports []string // python from...import... - Functions []string // python/go function - FunctionNames []string // function name set by user - Packages []string // python packages + FunctionNames []string // function names } -type Regexps struct { - Import *regexp.Regexp - Imports *regexp.Regexp - FunctionName *regexp.Regexp - FunctionContent *regexp.Regexp // including function define and body -} - -func (t *TemplateContent) parseGoContent(path string) error { - log.Info().Str("path", path).Msg("start to parse debugtalk.go") +func findAllFunctionNames(t *regexp.Regexp, path string) (functionNames []string, err error) { + log.Info().Str("path", path).Msg("find all function names from plugin file") content, err := os.ReadFile(path) if err != nil { log.Error().Err(err).Msg("failed to read file") - return err - } - originalContent := string(content) - - // parse imports - importSlice := t.Regexps.Imports.FindAllStringSubmatch(originalContent, -1) - if len(importSlice) != 0 { - imports := strings.Replace(importSlice[0][1], "\t", "", -1) - for _, elem := range strings.Split(imports, "\n") { - t.Imports = append(t.Imports, strings.TrimSpace(elem)) - } - } - // parse import - importSlice = t.Regexps.Import.FindAllStringSubmatch(originalContent, -1) - if len(importSlice) != 0 { - for _, elem := range importSlice { - t.Imports = append(t.Imports, strings.TrimSpace(elem[1])) - } - } - // import fungo package - if !builtin.Contains(t.Imports, fungo) { - t.Imports = append(t.Imports, fungo) + return nil, errors.Wrap(err, "read file failed") } - // parse function name - functionNameSlice := t.Regexps.FunctionName.FindAllStringSubmatch(originalContent, -1) + // find all function names + functionNameSlice := t.FindAllStringSubmatch(string(content), -1) for _, elem := range functionNameSlice { name := strings.Trim(elem[1], " ") - if name == "main" { - continue - } - t.FunctionNames = append(t.FunctionNames, name) + functionNames = append(functionNames, name) } - // parse function content - functionContentSlice := t.Regexps.FunctionContent.FindAllStringSubmatch(originalContent, -1) - for _, f := range functionContentSlice { - if strings.Contains(f[0], "func main") { - continue - } - t.Functions = append(t.Functions, strings.Trim(f[0], "\n")) - } - return nil + return } -func (t *TemplateContent) parsePyContent(path string) error { - file, err := os.Open(path) - if err != nil { - log.Error().Err(err).Str("path", path).Msg("failed to open file") - return err - } - defer file.Close() - - r := bufio.NewReader(file) - - // record content excluding import and main - content := "" - - // parse python content line by line - for { - l, _, err := r.ReadLine() - if err == io.EOF { - break - } - line := string(l) - - if strings.HasPrefix(line, "import") { - t.Imports = append(t.Imports, strings.Trim(line, " ")) - // e.g. import module as md - // import package.module - t.Packages = append(t.Packages, strings.Split(strings.Split(line, " ")[1], ".")[0]) - } else if strings.HasPrefix(line, "from") { - t.FromImports = append(t.FromImports, strings.Trim(line, " ")) - // e.g. from package.module import function - // from module import function - // from package import module - t.Packages = append(t.Packages, strings.Split(strings.Split(line, " ")[1], ".")[0]) - } else { - // no parse content at under of `if __name__ == "__main__"` - if strings.HasPrefix(line, "if __name__") { - break - } - if strings.HasPrefix(line, "def") { - functionNameSlice := t.Regexps.FunctionName.FindAllStringSubmatch(line, -1) - if len(functionNameSlice) == 0 { - continue - } - t.FunctionNames = append(t.FunctionNames, functionNameSlice[0][1]) - } - content += line + "\n" - } - } - // function content - t.Functions = append(t.Functions, strings.Trim(content, "\n")) - - return nil -} - -func (t *TemplateContent) genDebugTalk(output string, templ string) error { +func generate(data *pluginTemplateContent, tmpl, output string) error { file, err := os.Create(output) if err != nil { - log.Error().Err(err).Msg("open file failed") + log.Error().Err(err).Msg("open output file failed") return err } defer file.Close() + writer := bufio.NewWriter(file) - tmpl := template.Must(template.New("debugtalk").Parse(templ)) - err = tmpl.Execute(writer, t) + err = template.Must(template.New("debugtalk").Parse(tmpl)).Execute(writer, data) if err != nil { - log.Error().Err(err).Msg("execute applies a parsed template to the specified data object failed") + log.Error().Err(err).Msg("execute template parsing failed") return err } + err = writer.Flush() if err == nil { log.Info().Str("output", output).Msg("generate debugtalk success") @@ -181,35 +81,35 @@ func (t *TemplateContent) genDebugTalk(output string, templ string) error { // buildGo builds debugtalk.go to debugtalk.bin func buildGo(path string, output string) error { - templateContent := &TemplateContent{ - Version: version.VERSION, - Regexps: &Regexps{ - Import: regexp.MustCompile(regexGoImport), - Imports: regexp.MustCompile(regexGoImports), - FunctionName: regexp.MustCompile(regexGoFunctionName), - FunctionContent: regexp.MustCompile(regexGoFunctionContent), - }, + functionNames, err := findAllFunctionNames(regexGoFunctionName, path) + if err != nil { + return errors.Wrap(err, "find all function names failed") + } + // filter main and init function + var filteredFunctionNames []string + for _, name := range functionNames { + if name == "main" || name == "init" { + continue + } + filteredFunctionNames = append(filteredFunctionNames, name) + } + + templateContent := &pluginTemplateContent{ + Version: version.VERSION, + FunctionNames: filteredFunctionNames, } pluginDir := filepath.Dir(path) + err = generate(templateContent, goTemplate, filepath.Join(pluginDir, PluginGoSourceGenFile)) + if err != nil { + return errors.Wrap(err, "generate hashicorp plugin failed") + } // check go sdk in tempDir if err := builtin.ExecCommandInDir(exec.Command("go", "version"), pluginDir); err != nil { return errors.Wrap(err, "go sdk not installed") } - // parse debugtalk.go in pluginDir - err := templateContent.parseGoContent(path) - if err != nil { - return err - } - - // generate debugtalk.go in pluginDir - err = templateContent.genDebugTalk(filepath.Join(pluginDir, PluginGoSourceGenFile), goTemplate) - if err != nil { - return err - } - if !builtin.IsFilePathExists(filepath.Join(pluginDir, "go.mod")) { // create go mod if err := builtin.ExecCommandInDir(exec.Command("go", "mod", "init", "main"), pluginDir); err != nil { @@ -220,64 +120,66 @@ func buildGo(path string, output string) error { // funplugin version should be locked funplugin := fmt.Sprintf("github.com/httprunner/funplugin@%s", shared.Version) if err := builtin.ExecCommandInDir(exec.Command("go", "get", funplugin), pluginDir); err != nil { - return err + return errors.Wrap(err, "go get funplugin failed") } } // add missing and remove unused modules if err := builtin.ExecCommandInDir(exec.Command("go", "mod", "tidy"), pluginDir); err != nil { - return err + return errors.Wrap(err, "go mod tidy failed") } + // specify output file path if output == "" { dir, _ := os.Getwd() output = filepath.Join(dir, PluginHashicorpGoBuiltFile) } else if builtin.IsFolderPathExists(output) { output = filepath.Join(output, PluginHashicorpGoBuiltFile) } - outputPath, err := filepath.Abs(output) - if err != nil { - return err - } + outputPath, _ := filepath.Abs(output) - // build plugin debugtalk.bin + // build go plugin to debugtalk.bin cmd := exec.Command("go", "build", "-o", outputPath, PluginGoSourceGenFile, filepath.Base(path)) if err := builtin.ExecCommandInDir(cmd, pluginDir); err != nil { - return err + return errors.Wrap(err, "go build plugin failed") } - log.Info().Str("output", outputPath).Str("plugin", path).Msg("build plugin successfully") + log.Info().Str("output", outputPath).Str("plugin", path).Msg("build go plugin successfully") return nil } // buildPy completes funppy information in debugtalk.py func buildPy(path string, output string) error { - templateContent := &TemplateContent{ - Version: version.VERSION, - Regexps: &Regexps{ - FunctionName: regexp.MustCompile(regexPythonFunctionName), - }, - } - - err := templateContent.parsePyContent(path) - if err != nil { - return err - } - // check the syntax of debugtalk.py - err = builtin.CheckPythonScriptSyntax(path) + err := builtin.ExecCommand("python3", "-m", "py_compile", path) + if err != nil { + return errors.Wrap(err, "python plugin syntax invalid") + } + + functionNames, err := findAllFunctionNames(regexPyFunctionName, path) if err != nil { return err } + templateContent := &pluginTemplateContent{ + Version: version.VERSION, + FunctionNames: functionNames, + } - // generate .debugtalk_gen.py + // specify output file path if output == "" { dir, _ := os.Getwd() output = filepath.Join(dir, PluginPySourceGenFile) } else if builtin.IsFolderPathExists(output) { output = filepath.Join(output, PluginPySourceGenFile) } - err = templateContent.genDebugTalk(output, pyTemplate) - return err + + // generate .debugtalk_gen.py + err = generate(templateContent, pyTemplate, output) + if err != nil { + return err + } + + log.Info().Str("output", output).Str("plugin", path).Msg("build python plugin successfully") + return nil } func BuildPlugin(path string, output string) (err error) { @@ -291,7 +193,7 @@ func BuildPlugin(path string, output string) (err error) { return errors.New("type error, expected .py or .go") } if err != nil { - log.Error().Err(err).Str("arg", path).Msg("build plugin failed") + log.Error().Err(err).Str("path", path).Msg("build plugin failed") os.Exit(1) } return nil diff --git a/hrp/internal/builtin/utils.go b/hrp/internal/builtin/utils.go index 0fd52db5..fe702f97 100644 --- a/hrp/internal/builtin/utils.go +++ b/hrp/internal/builtin/utils.go @@ -185,10 +185,6 @@ func InstallPythonPackage(python3 string, pkg string) (err error) { return nil } -func CheckPythonScriptSyntax(path string) error { - return ExecCommand("python3", "-m", "py_compile", path) -} - func ExecCommandInDir(cmd *exec.Cmd, dir string) error { log.Info().Str("cmd", cmd.String()).Str("dir", dir).Msg("exec command") cmd.Dir = dir diff --git a/hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py b/hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py index 880c145f..653fed67 100644 --- a/hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py +++ b/hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py @@ -1,67 +1,15 @@ # NOTE: Generated By hrp v4.1.2, DO NOT EDIT! -import logging -import time -import funppy +import sys +import os -from typing import List +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -def get_user_agent(): - return "hrp/funppy" - - -def sleep(n_secs): - time.sleep(n_secs) - - -def sum(*args): - result = 0 - for arg in args: - result += arg - return result - - -def sum_ints(*args: List[int]) -> int: - result = 0 - for arg in args: - result += arg - return result - - -def sum_two_int(a: int, b: int) -> int: - return a + b - - -def sum_two_string(a: str, b: str) -> str: - return a + b - - -def sum_strings(*args: List[str]) -> str: - result = "" - for arg in args: - result += arg - return result - - -def concatenate(*args: List[str]) -> str: - result = "" - for arg in args: - result += str(arg) - return result - - -def setup_hook_example(name): - logging.warning("setup_hook_example") - return f"setup_hook_example: {name}" - - -def teardown_hook_example(name): - logging.warning("teardown_hook_example") - return f"teardown_hook_example: {name}" +from debugtalk import * if __name__ == "__main__": + import funppy funppy.register("get_user_agent", get_user_agent) funppy.register("sleep", sleep) funppy.register("sum", sum) diff --git a/hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate b/hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate index b892dd30..61123599 100644 --- a/hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate +++ b/hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate @@ -1,15 +1,12 @@ # NOTE: Generated By hrp {{ .Version }}, DO NOT EDIT! -{{ range $import := .Imports }} -{{- $import}} -{{ end }} -{{ range $fromImport := .FromImports }} -{{- $fromImport}} -{{ end }} +import sys +import os + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from debugtalk import * -{{ range $function := .Functions }} -{{- $function }} -{{ end }} if __name__ == "__main__": import funppy diff --git a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go index df7727c3..1e6ceac9 100644 --- a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go +++ b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go @@ -1,4 +1,4 @@ -// NOTE: Generated By hrp v4.1.3, DO NOT EDIT! +// NOTE: Generated By hrp v4.1.2, DO NOT EDIT! package main import ( From a62bc1d399f882963200fbf726c3784629e8b972 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Mon, 13 Jun 2022 23:54:47 +0800 Subject: [PATCH 05/12] feat: check if specified python3 venv is invalid --- docs/CHANGELOG.md | 4 ++++ hrp/internal/builtin/utils.go | 13 +++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 1669752f..a84beb29 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,10 @@ - refactor: build plugin mechanism - fix: pip upgrade httprunner when installing hrp +**python version** + +fix: unexpected changes in step variables + ## v4.1.2 (2022-06-09) - feat: add Dockerfile diff --git a/hrp/internal/builtin/utils.go b/hrp/internal/builtin/utils.go index fe702f97..9f6d2b92 100644 --- a/hrp/internal/builtin/utils.go +++ b/hrp/internal/builtin/utils.go @@ -92,13 +92,14 @@ func FormatResponse(raw interface{}) interface{} { var Python3Executable string = "python3" // system default python3 func PrepareVenv(venv string) error { - defer func() { - log.Info().Str("Python3Executable", Python3Executable).Msg("set python3 executable path") - }() - // specify python3 venv if venv != "" { - Python3Executable = getPython3Executable(venv) + python3 := getPython3Executable(venv) + if !IsFilePathExists(python3) { + return errors.New("specified python3 venv is invalid") + } + Python3Executable = python3 + log.Info().Str("Python3Executable", Python3Executable).Msg("set python3 executable path") return nil } @@ -108,7 +109,7 @@ func PrepareVenv(venv string) error { return errors.Wrap(err, "create default python3 venv failed") } Python3Executable = python3 - + log.Info().Str("Python3Executable", Python3Executable).Msg("set python3 executable path") return nil } From 2bb71526434c3f4a05712506b22d53960837c0df Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 14 Jun 2022 00:40:53 +0800 Subject: [PATCH 06/12] feat: assert python3 package is ready with specified version --- docs/CHANGELOG.md | 5 +- examples/demo-with-go-plugin/proj.json | 4 +- .../demo-with-py-plugin/.debugtalk_gen.py | 2 +- examples/demo-with-py-plugin/proj.json | 4 +- hrp/internal/builtin/utils.go | 82 ++++++++++--------- hrp/internal/scaffold/main.go | 3 +- .../templates/plugin/.debugtalk_gen.py | 2 +- .../templates/plugin/debugtalk_gen.go | 2 +- hrp/plugin.go | 8 ++ 9 files changed, 63 insertions(+), 49 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index a84beb29..60611491 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,12 +6,13 @@ - feat #1342: support specify custom python3 venv - feat: support python3 venv priority, specified > projectDir/.venv > $HOME/.hrp/venv -- refactor: build plugin mechanism +- feat: assert python3 package is ready with specified version +- refactor: build plugin mechanism, cancel automatic installation of dependencies - fix: pip upgrade httprunner when installing hrp **python version** -fix: unexpected changes in step variables +- fix: unexpected changes in step variables ## v4.1.2 (2022-06-09) diff --git a/examples/demo-with-go-plugin/proj.json b/examples/demo-with-go-plugin/proj.json index 48b9a7c2..bebbd982 100644 --- a/examples/demo-with-go-plugin/proj.json +++ b/examples/demo-with-go-plugin/proj.json @@ -1,5 +1,5 @@ { "project_name": "demo-with-go-plugin", - "create_time": "2022-06-13T23:28:24.920066+08:00", - "hrp_version": "v4.1.2" + "create_time": "2022-06-13T23:42:06.381463+08:00", + "hrp_version": "v4.1.3" } diff --git a/examples/demo-with-py-plugin/.debugtalk_gen.py b/examples/demo-with-py-plugin/.debugtalk_gen.py index 653fed67..7199a757 100644 --- a/examples/demo-with-py-plugin/.debugtalk_gen.py +++ b/examples/demo-with-py-plugin/.debugtalk_gen.py @@ -1,4 +1,4 @@ -# NOTE: Generated By hrp v4.1.2, DO NOT EDIT! +# NOTE: Generated By hrp v4.1.3, DO NOT EDIT! import sys import os diff --git a/examples/demo-with-py-plugin/proj.json b/examples/demo-with-py-plugin/proj.json index 457cef97..1a5c6c7b 100644 --- a/examples/demo-with-py-plugin/proj.json +++ b/examples/demo-with-py-plugin/proj.json @@ -1,5 +1,5 @@ { "project_name": "demo-with-py-plugin", - "create_time": "2022-06-13T23:28:25.254595+08:00", - "hrp_version": "v4.1.2" + "create_time": "2022-06-13T23:42:06.751991+08:00", + "hrp_version": "v4.1.3" } diff --git a/hrp/internal/builtin/utils.go b/hrp/internal/builtin/utils.go index 9f6d2b92..c8d8fcc0 100644 --- a/hrp/internal/builtin/utils.go +++ b/hrp/internal/builtin/utils.go @@ -104,7 +104,17 @@ func PrepareVenv(venv string) error { } // not specify python3 venv, create default venv - python3, err := createDefaultPython3Venv() + home, err := os.UserHomeDir() + if err != nil { + return errors.Wrap(err, "get user home dir failed") + } + venv = filepath.Join(home, ".hrp", "venv") + log.Info().Str("venv", venv).Msg("create default python3 venv") + packages := []string{ + fmt.Sprintf("funppy==%s", fungo.Version), + fmt.Sprintf("httprunner==%s", version.VERSION), + } + python3, err := EnsurePython3Venv(venv, packages...) if err != nil { return errors.Wrap(err, "create default python3 venv failed") } @@ -113,56 +123,50 @@ func PrepareVenv(venv string) error { return nil } -// create default python3 venv located in $HOME/.hrp/venv -func createDefaultPython3Venv() (string, error) { - log.Info().Msg("create default python3 venv at $HOME/.hrp/venv") - home, err := os.UserHomeDir() - if err != nil { - return "", errors.Wrap(err, "get user home dir failed") - } - venv := filepath.Join(home, ".hrp", "venv") - packages := []string{ - fmt.Sprintf("funppy==%s", fungo.Version), - fmt.Sprintf("httprunner==%s", version.VERSION), - } - return EnsurePython3Venv(venv, packages...) -} - func ExecPython3Command(cmdName string, args ...string) error { args = append([]string{"-m", cmdName}, args...) return ExecCommand(Python3Executable, args...) } +func AssertPythonPackage(python3 string, pkgName, pkgVersion string) error { + out, err := exec.Command( + python3, "-c", fmt.Sprintf("import %s; print(%s.__version__)", pkgName, pkgName), + ).Output() + if err != nil { + return fmt.Errorf("python package %s not found", pkgName) + } + + // do not check version if pkgVersion is empty + if pkgVersion == "" { + log.Info().Str("name", pkgName).Msg("python package is ready") + return nil + } + + // check package version equality + version := strings.TrimSpace(string(out)) + if fmt.Sprintf("v%s", version) != pkgVersion { + return fmt.Errorf("python package %s version %s not matched, please upgrade to %s", + pkgName, version, pkgVersion) + } + + log.Info().Str("name", pkgName).Str("version", pkgVersion).Msg("python package is ready") + return nil +} + func InstallPythonPackage(python3 string, pkg string) (err error) { - var pkgName string + var pkgName, pkgVersion string if strings.Contains(pkg, "==") { - // funppy==0.4.2 + // funppy==0.5.0 pkgInfo := strings.Split(pkg, "==") pkgName = pkgInfo[0] - } else if strings.Contains(pkg, ">=") { - // httprunner>=4.0.0-beta - pkgInfo := strings.Split(pkg, ">=") - pkgName = pkgInfo[0] + pkgVersion = pkgInfo[1] } else { + // funppy pkgName = pkg } - defer func() { - if err == nil { - // check package version - if out, err := exec.Command( - python3, "-c", fmt.Sprintf("import %s; print(%s.__version__)", pkgName, pkgName), - ).Output(); err == nil { - log.Info(). - Str("name", pkgName). - Str("version", strings.TrimSpace(string(out))). - Msg("python package is ready") - } - } - }() - - // check if package installed - err = exec.Command(python3, "-c", fmt.Sprintf("import %s", pkgName)).Run() + // check if package installed and version matched + err = AssertPythonPackage(python3, pkgName, pkgVersion) if err == nil { return nil } @@ -183,7 +187,7 @@ func InstallPythonPackage(python3 string, pkg string) (err error) { return errors.Wrap(err, "pip install package failed") } - return nil + return AssertPythonPackage(python3, pkgName, pkgVersion) } func ExecCommandInDir(cmd *exec.Cmd, dir string) error { diff --git a/hrp/internal/scaffold/main.go b/hrp/internal/scaffold/main.go index ed9405a6..46e21b76 100644 --- a/hrp/internal/scaffold/main.go +++ b/hrp/internal/scaffold/main.go @@ -8,10 +8,10 @@ import ( "path/filepath" "time" - "github.com/httprunner/funplugin/fungo" "github.com/pkg/errors" "github.com/rs/zerolog/log" + "github.com/httprunner/funplugin/fungo" "github.com/httprunner/httprunner/v4/hrp" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/sdk" @@ -208,6 +208,7 @@ func createPythonPlugin(projectName, venv string) error { if venv == "" { venv = filepath.Join(projectName, ".venv") } + log.Info().Str("venv", venv).Msg("create python3 venv") packages := []string{ fmt.Sprintf("funppy==%s", fungo.Version), fmt.Sprintf("httprunner==%s", version.VERSION), diff --git a/hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py b/hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py index 653fed67..7199a757 100644 --- a/hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py +++ b/hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py @@ -1,4 +1,4 @@ -# NOTE: Generated By hrp v4.1.2, DO NOT EDIT! +# NOTE: Generated By hrp v4.1.3, DO NOT EDIT! import sys import os diff --git a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go index 1e6ceac9..df7727c3 100644 --- a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go +++ b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go @@ -1,4 +1,4 @@ -// NOTE: Generated By hrp v4.1.2, DO NOT EDIT! +// NOTE: Generated By hrp v4.1.3, DO NOT EDIT! package main import ( diff --git a/hrp/plugin.go b/hrp/plugin.go index 91bdedd8..2af298c3 100644 --- a/hrp/plugin.go +++ b/hrp/plugin.go @@ -9,6 +9,7 @@ import ( "syscall" "github.com/httprunner/funplugin" + "github.com/httprunner/funplugin/fungo" "github.com/rs/zerolog/log" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" @@ -57,6 +58,13 @@ func initPlugin(path, venv string, logOn bool) (plugin funplugin.IPlugin, err er log.Error().Err(err).Msg("prepare python3 venv failed") return } + // check if funppy is ready in python3 venv + err = builtin.AssertPythonPackage(builtin.Python3Executable, "funppy", fungo.Version) + if err != nil { + log.Error().Err(err).Str("venv", venv).Msg("python package funppy is not ready") + os.Exit(1) + } + pluginOptions = append(pluginOptions, funplugin.WithPython3(builtin.Python3Executable)) } From d91eba501ca84c4ee322dc01e35f8fc1076dc580 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 14 Jun 2022 10:35:33 +0800 Subject: [PATCH 07/12] change: venv priority: specified > /Users/debugtalk/.hrp/venv --- docs/CHANGELOG.md | 5 ++--- hrp/internal/builtin/utils.go | 2 +- hrp/internal/scaffold/main.go | 8 ++++++-- hrp/plugin.go | 5 +---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 60611491..db8a18f8 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,11 +1,10 @@ # Release History -## v4.1.3 (2022-06-13) +## v4.1.3 (2022-06-14) **go version** -- feat #1342: support specify custom python3 venv -- feat: support python3 venv priority, specified > projectDir/.venv > $HOME/.hrp/venv +- feat #1342: support specify custom python3 venv, priority is greater than $HOME/.hrp/venv - feat: assert python3 package is ready with specified version - refactor: build plugin mechanism, cancel automatic installation of dependencies - fix: pip upgrade httprunner when installing hrp diff --git a/hrp/internal/builtin/utils.go b/hrp/internal/builtin/utils.go index c8d8fcc0..f145989c 100644 --- a/hrp/internal/builtin/utils.go +++ b/hrp/internal/builtin/utils.go @@ -178,7 +178,7 @@ func InstallPythonPackage(python3 string, pkg string) (err error) { return errors.Wrap(err, "pip is not available") } - log.Info().Str("package", pkg).Msg("installing python package") + log.Info().Str("pkgName", pkgName).Str("pkgVersion", pkgVersion).Msg("installing python package") // install package err = ExecCommand(python3, "-m", "pip", "install", "--upgrade", pkg, diff --git a/hrp/internal/scaffold/main.go b/hrp/internal/scaffold/main.go index 46e21b76..59557128 100644 --- a/hrp/internal/scaffold/main.go +++ b/hrp/internal/scaffold/main.go @@ -8,10 +8,10 @@ import ( "path/filepath" "time" + "github.com/httprunner/funplugin/fungo" "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/httprunner/funplugin/fungo" "github.com/httprunner/httprunner/v4/hrp" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/sdk" @@ -206,7 +206,11 @@ func createPythonPlugin(projectName, venv string) error { } if venv == "" { - venv = filepath.Join(projectName, ".venv") + home, err := os.UserHomeDir() + if err != nil { + return errors.Wrap(err, "get user home dir failed") + } + venv = filepath.Join(home, ".hrp", "venv") } log.Info().Str("venv", venv).Msg("create python3 venv") packages := []string{ diff --git a/hrp/plugin.go b/hrp/plugin.go index 2af298c3..bc7d0241 100644 --- a/hrp/plugin.go +++ b/hrp/plugin.go @@ -49,10 +49,7 @@ func initPlugin(path, venv string, logOn bool) (plugin funplugin.IPlugin, err er } pluginPath = genPyPluginPath - // priority: specified > projectDir/.venv > $HOME/.hrp/venv - if venv == "" && builtin.IsFolderPathExists(filepath.Join(filepath.Dir(pluginPath), ".venv")) { - venv = filepath.Join(filepath.Dir(pluginPath), ".venv") - } + // priority: specified > $HOME/.hrp/venv err = builtin.PrepareVenv(venv) if err != nil { log.Error().Err(err).Msg("prepare python3 venv failed") From c00dab17b3cb6e14e09a19388ec8c63a70e6c769 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 14 Jun 2022 11:22:33 +0800 Subject: [PATCH 08/12] refactor: EnsurePython3Venv --- docs/cmd/hrp.md | 2 +- docs/cmd/hrp_boom.md | 2 +- docs/cmd/hrp_build.md | 2 +- docs/cmd/hrp_convert.md | 2 +- docs/cmd/hrp_pytest.md | 2 +- docs/cmd/hrp_run.md | 2 +- docs/cmd/hrp_startproject.md | 2 +- docs/cmd/hrp_wiki.md | 2 +- examples/demo-with-go-plugin/proj.json | 2 +- examples/demo-with-py-plugin/proj.json | 2 +- hrp/cmd/build.go | 7 ---- hrp/cmd/convert.go | 9 ++++- hrp/cmd/pytest.go | 10 ++++- hrp/internal/builtin/utils.go | 48 +++++++++--------------- hrp/internal/builtin/utils_unix.go | 13 +++---- hrp/internal/builtin/utils_windows.go | 1 + hrp/internal/convert/converter_pytest.go | 9 ----- hrp/internal/scaffold/examples_test.go | 4 +- hrp/internal/scaffold/main.go | 8 ---- hrp/plugin.go | 19 ++++------ 20 files changed, 60 insertions(+), 88 deletions(-) diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index 810a22ad..70279830 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -37,4 +37,4 @@ Copyright 2017 debugtalk * [hrp startproject](hrp_startproject.md) - create a scaffold project * [hrp wiki](hrp_wiki.md) - visit https://httprunner.com -###### Auto generated by spf13/cobra on 13-Jun-2022 +###### Auto generated by spf13/cobra on 14-Jun-2022 diff --git a/docs/cmd/hrp_boom.md b/docs/cmd/hrp_boom.md index 7a9cfbca..802d8f82 100644 --- a/docs/cmd/hrp_boom.md +++ b/docs/cmd/hrp_boom.md @@ -42,4 +42,4 @@ hrp boom [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 13-Jun-2022 +###### Auto generated by spf13/cobra on 14-Jun-2022 diff --git a/docs/cmd/hrp_build.md b/docs/cmd/hrp_build.md index 73dccf0b..3202ee3e 100644 --- a/docs/cmd/hrp_build.md +++ b/docs/cmd/hrp_build.md @@ -28,4 +28,4 @@ hrp build $path ... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 13-Jun-2022 +###### Auto generated by spf13/cobra on 14-Jun-2022 diff --git a/docs/cmd/hrp_convert.md b/docs/cmd/hrp_convert.md index 4160891c..a1cf7236 100644 --- a/docs/cmd/hrp_convert.md +++ b/docs/cmd/hrp_convert.md @@ -22,4 +22,4 @@ hrp convert $path... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 13-Jun-2022 +###### Auto generated by spf13/cobra on 14-Jun-2022 diff --git a/docs/cmd/hrp_pytest.md b/docs/cmd/hrp_pytest.md index b48bf18e..3ceb2d62 100644 --- a/docs/cmd/hrp_pytest.md +++ b/docs/cmd/hrp_pytest.md @@ -16,4 +16,4 @@ hrp pytest $path ... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 13-Jun-2022 +###### Auto generated by spf13/cobra on 14-Jun-2022 diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index cc859ed3..ffbe28fc 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -35,4 +35,4 @@ hrp run $path... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 13-Jun-2022 +###### Auto generated by spf13/cobra on 14-Jun-2022 diff --git a/docs/cmd/hrp_startproject.md b/docs/cmd/hrp_startproject.md index 78953953..e608327d 100644 --- a/docs/cmd/hrp_startproject.md +++ b/docs/cmd/hrp_startproject.md @@ -21,4 +21,4 @@ hrp startproject $project_name [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 13-Jun-2022 +###### Auto generated by spf13/cobra on 14-Jun-2022 diff --git a/docs/cmd/hrp_wiki.md b/docs/cmd/hrp_wiki.md index 7283ba32..196d0bba 100644 --- a/docs/cmd/hrp_wiki.md +++ b/docs/cmd/hrp_wiki.md @@ -16,4 +16,4 @@ hrp wiki [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 13-Jun-2022 +###### Auto generated by spf13/cobra on 14-Jun-2022 diff --git a/examples/demo-with-go-plugin/proj.json b/examples/demo-with-go-plugin/proj.json index bebbd982..03850cc4 100644 --- a/examples/demo-with-go-plugin/proj.json +++ b/examples/demo-with-go-plugin/proj.json @@ -1,5 +1,5 @@ { "project_name": "demo-with-go-plugin", - "create_time": "2022-06-13T23:42:06.381463+08:00", + "create_time": "2022-06-14T11:34:25.941159+08:00", "hrp_version": "v4.1.3" } diff --git a/examples/demo-with-py-plugin/proj.json b/examples/demo-with-py-plugin/proj.json index 1a5c6c7b..3cb92bd4 100644 --- a/examples/demo-with-py-plugin/proj.json +++ b/examples/demo-with-py-plugin/proj.json @@ -1,5 +1,5 @@ { "project_name": "demo-with-py-plugin", - "create_time": "2022-06-13T23:42:06.751991+08:00", + "create_time": "2022-06-14T11:34:26.276891+08:00", "hrp_version": "v4.1.3" } diff --git a/hrp/cmd/build.go b/hrp/cmd/build.go index 9a8c2bb8..3c8848e3 100644 --- a/hrp/cmd/build.go +++ b/hrp/cmd/build.go @@ -1,11 +1,9 @@ package cmd import ( - "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/httprunner/httprunner/v4/hrp" - "github.com/httprunner/httprunner/v4/hrp/internal/builtin" ) var buildCmd = &cobra.Command{ @@ -19,11 +17,6 @@ var buildCmd = &cobra.Command{ setLogLevel(logLevel) }, RunE: func(cmd *cobra.Command, args []string) error { - err := builtin.PrepareVenv(venv) - if err != nil { - log.Error().Err(err).Msg("prepare python3 venv failed") - return err - } return hrp.BuildPlugin(args[0], output) }, } diff --git a/hrp/cmd/convert.go b/hrp/cmd/convert.go index ee43c2de..3611274c 100644 --- a/hrp/cmd/convert.go +++ b/hrp/cmd/convert.go @@ -2,12 +2,14 @@ package cmd import ( "errors" + "fmt" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/convert" + "github.com/httprunner/httprunner/v4/hrp/internal/version" ) var convertCmd = &cobra.Command{ @@ -35,9 +37,12 @@ var convertCmd = &cobra.Command{ flagCount++ outputType = convert.OutputTypePyTest - err := builtin.PrepareVenv(venv) + packages := []string{ + fmt.Sprintf("httprunner==%s", version.VERSION), + } + _, err := builtin.EnsurePython3Venv(venv, packages...) if err != nil { - log.Error().Err(err).Msg("prepare python3 venv failed") + log.Error().Err(err).Msg("python3 venv is not ready") return err } } diff --git a/hrp/cmd/pytest.go b/hrp/cmd/pytest.go index 4d49b2be..2e1933ad 100644 --- a/hrp/cmd/pytest.go +++ b/hrp/cmd/pytest.go @@ -1,11 +1,14 @@ package cmd import ( + "fmt" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/pytest" + "github.com/httprunner/httprunner/v4/hrp/internal/version" ) var pytestCmd = &cobra.Command{ @@ -17,9 +20,12 @@ var pytestCmd = &cobra.Command{ }, DisableFlagParsing: true, // allow to pass any args to pytest RunE: func(cmd *cobra.Command, args []string) error { - err := builtin.PrepareVenv(venv) + packages := []string{ + fmt.Sprintf("httprunner==%s", version.VERSION), + } + _, err := builtin.EnsurePython3Venv(venv, packages...) if err != nil { - log.Error().Err(err).Msg("prepare python3 venv failed") + log.Error().Err(err).Msg("python3 venv is not ready") return err } return pytest.RunPytest(args) diff --git a/hrp/internal/builtin/utils.go b/hrp/internal/builtin/utils.go index f145989c..0d05339a 100644 --- a/hrp/internal/builtin/utils.go +++ b/hrp/internal/builtin/utils.go @@ -13,13 +13,11 @@ import ( "strconv" "strings" - "github.com/httprunner/funplugin/fungo" "github.com/pkg/errors" "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" "github.com/httprunner/httprunner/v4/hrp/internal/json" - "github.com/httprunner/httprunner/v4/hrp/internal/version" ) func Dump2JSON(data interface{}, path string) error { @@ -89,43 +87,31 @@ func FormatResponse(raw interface{}) interface{} { return formattedResponse } -var Python3Executable string = "python3" // system default python3 +var python3Executable string = "python3" // system default python3 -func PrepareVenv(venv string) error { - // specify python3 venv - if venv != "" { - python3 := getPython3Executable(venv) - if !IsFilePathExists(python3) { - return errors.New("specified python3 venv is invalid") +// EnsurePython3Venv ensures python3 venv with specified packages +// venv should be directory path of target venv +func EnsurePython3Venv(venv string, packages ...string) (python3 string, err error) { + // priority: specified > $HOME/.hrp/venv + if venv == "" { + home, err := os.UserHomeDir() + if err != nil { + return "", errors.Wrap(err, "get user home dir failed") } - Python3Executable = python3 - log.Info().Str("Python3Executable", Python3Executable).Msg("set python3 executable path") - return nil + venv = filepath.Join(home, ".hrp", "venv") } - - // not specify python3 venv, create default venv - home, err := os.UserHomeDir() + python3, err = ensurePython3Venv(venv, packages...) if err != nil { - return errors.Wrap(err, "get user home dir failed") + return "", errors.Wrap(err, "prepare python3 venv failed") } - venv = filepath.Join(home, ".hrp", "venv") - log.Info().Str("venv", venv).Msg("create default python3 venv") - packages := []string{ - fmt.Sprintf("funppy==%s", fungo.Version), - fmt.Sprintf("httprunner==%s", version.VERSION), - } - python3, err := EnsurePython3Venv(venv, packages...) - if err != nil { - return errors.Wrap(err, "create default python3 venv failed") - } - Python3Executable = python3 - log.Info().Str("Python3Executable", Python3Executable).Msg("set python3 executable path") - return nil + python3Executable = python3 + log.Info().Str("Python3Executable", python3Executable).Msg("set python3 executable path") + return python3, nil } func ExecPython3Command(cmdName string, args ...string) error { args = append([]string{"-m", cmdName}, args...) - return ExecCommand(Python3Executable, args...) + return ExecCommand(python3Executable, args...) } func AssertPythonPackage(python3 string, pkgName, pkgVersion string) error { @@ -144,7 +130,7 @@ func AssertPythonPackage(python3 string, pkgName, pkgVersion string) error { // check package version equality version := strings.TrimSpace(string(out)) - if fmt.Sprintf("v%s", version) != pkgVersion { + if strings.TrimLeft(version, "v") != strings.TrimLeft(pkgVersion, "v") { return fmt.Errorf("python package %s version %s not matched, please upgrade to %s", pkgName, version, pkgVersion) } diff --git a/hrp/internal/builtin/utils_unix.go b/hrp/internal/builtin/utils_unix.go index 510ac0c0..042b2869 100644 --- a/hrp/internal/builtin/utils_unix.go +++ b/hrp/internal/builtin/utils_unix.go @@ -1,3 +1,4 @@ +//go:build darwin || linux // +build darwin linux package builtin @@ -16,10 +17,8 @@ func getPython3Executable(venvDir string) string { return filepath.Join(venvDir, "bin", "python3") } -// EnsurePython3Venv ensures python3 venv for hashicorp python plugin -// venvDir should be directory path of target venv -func EnsurePython3Venv(venvDir string, packages ...string) (python3 string, err error) { - python3 = getPython3Executable(venvDir) +func ensurePython3Venv(venv string, packages ...string) (python3 string, err error) { + python3 = getPython3Executable(venv) log.Info(). Str("python3", python3). @@ -35,15 +34,15 @@ func EnsurePython3Venv(venvDir string, packages ...string) (python3 string, err } // check if .venv exists - if _, err := os.Stat(venvDir); err == nil { + if _, err := os.Stat(venv); err == nil { // .venv exists, remove first - if err := ExecCommand("rm", "-rf", venvDir); err != nil { + if err := ExecCommand("rm", "-rf", venv); err != nil { return "", errors.Wrap(err, "remove existed venv failed") } } // create python3 .venv - if err := ExecCommand("python3", "-m", "venv", venvDir); err != nil { + if err := ExecCommand("python3", "-m", "venv", venv); err != nil { return "", errors.Wrap(err, "create python3 venv failed") } } diff --git a/hrp/internal/builtin/utils_windows.go b/hrp/internal/builtin/utils_windows.go index 8501f082..db1c2b59 100644 --- a/hrp/internal/builtin/utils_windows.go +++ b/hrp/internal/builtin/utils_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package builtin diff --git a/hrp/internal/convert/converter_pytest.go b/hrp/internal/convert/converter_pytest.go index 75c1490f..233bcded 100644 --- a/hrp/internal/convert/converter_pytest.go +++ b/hrp/internal/convert/converter_pytest.go @@ -1,10 +1 @@ package convert - -import ( - "github.com/httprunner/httprunner/v4/hrp/internal/builtin" -) - -func convert2PyTestScripts(paths ...string) error { - args := append([]string{"make"}, paths...) - return builtin.ExecPython3Command("httprunner", args...) -} diff --git a/hrp/internal/scaffold/examples_test.go b/hrp/internal/scaffold/examples_test.go index 0590994d..4ed02cf1 100644 --- a/hrp/internal/scaffold/examples_test.go +++ b/hrp/internal/scaffold/examples_test.go @@ -1,6 +1,7 @@ package scaffold import ( + "path/filepath" "testing" ) @@ -12,7 +13,8 @@ func TestGenDemoExamples(t *testing.T) { } dir = "../../../examples/demo-with-py-plugin" - err = CreateScaffold(dir, Py, "", true) + venv := filepath.Join(dir, ".venv") + err = CreateScaffold(dir, Py, venv, true) if err != nil { t.Fatal() } diff --git a/hrp/internal/scaffold/main.go b/hrp/internal/scaffold/main.go index 59557128..f27a035a 100644 --- a/hrp/internal/scaffold/main.go +++ b/hrp/internal/scaffold/main.go @@ -205,14 +205,6 @@ func createPythonPlugin(projectName, venv string) error { return errors.Wrap(err, "copy file failed") } - if venv == "" { - home, err := os.UserHomeDir() - if err != nil { - return errors.Wrap(err, "get user home dir failed") - } - venv = filepath.Join(home, ".hrp", "venv") - } - log.Info().Str("venv", venv).Msg("create python3 venv") packages := []string{ fmt.Sprintf("funppy==%s", fungo.Version), fmt.Sprintf("httprunner==%s", version.VERSION), diff --git a/hrp/plugin.go b/hrp/plugin.go index bc7d0241..a2343358 100644 --- a/hrp/plugin.go +++ b/hrp/plugin.go @@ -49,20 +49,17 @@ func initPlugin(path, venv string, logOn bool) (plugin funplugin.IPlugin, err er } pluginPath = genPyPluginPath - // priority: specified > $HOME/.hrp/venv - err = builtin.PrepareVenv(venv) - if err != nil { - log.Error().Err(err).Msg("prepare python3 venv failed") - return + packages := []string{ + fmt.Sprintf("funppy==%s", fungo.Version), } - // check if funppy is ready in python3 venv - err = builtin.AssertPythonPackage(builtin.Python3Executable, "funppy", fungo.Version) + python3, err := builtin.EnsurePython3Venv(venv, packages...) if err != nil { - log.Error().Err(err).Str("venv", venv).Msg("python package funppy is not ready") - os.Exit(1) + log.Error().Err(err). + Interface("packages", packages). + Msg("python3 venv is not ready") + return nil, err } - - pluginOptions = append(pluginOptions, funplugin.WithPython3(builtin.Python3Executable)) + pluginOptions = append(pluginOptions, funplugin.WithPython3(python3)) } // found plugin file From 5a24bc8c27f280477dc1cb7091b6b24ac885d6d0 Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 14 Jun 2022 13:22:07 +0800 Subject: [PATCH 09/12] fix #1352: avoid conversion to exponential notation --- docs/CHANGELOG.md | 1 + examples/demo-with-go-plugin/proj.json | 2 +- .../testcases/requests.json | 4 ++-- .../testcases/requests.yml | 4 ++-- examples/demo-with-py-plugin/proj.json | 2 +- .../testcases/requests.json | 4 ++-- .../testcases/requests.yml | 4 ++-- .../templates/testcases/demo_requests.json | 4 ++-- .../templates/testcases/demo_requests.yml | 4 ++-- .../templates/testcases/demo_requests_test.py | 4 ++-- hrp/parser.go | 19 +++++++++++-------- hrp/parser_test.go | 2 ++ hrp/step_request.go | 6 +++--- 13 files changed, 33 insertions(+), 27 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index db8a18f8..9262a3f3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,7 @@ - feat: assert python3 package is ready with specified version - refactor: build plugin mechanism, cancel automatic installation of dependencies - fix: pip upgrade httprunner when installing hrp +- fix #1352: avoid conversion to exponential notation **python version** diff --git a/examples/demo-with-go-plugin/proj.json b/examples/demo-with-go-plugin/proj.json index 03850cc4..fac542f7 100644 --- a/examples/demo-with-go-plugin/proj.json +++ b/examples/demo-with-go-plugin/proj.json @@ -1,5 +1,5 @@ { "project_name": "demo-with-go-plugin", - "create_time": "2022-06-14T11:34:25.941159+08:00", + "create_time": "2022-06-14T13:19:05.262039+08:00", "hrp_version": "v4.1.3" } diff --git a/examples/demo-with-go-plugin/testcases/requests.json b/examples/demo-with-go-plugin/testcases/requests.json index 75c464f9..4c01ec74 100644 --- a/examples/demo-with-go-plugin/testcases/requests.json +++ b/examples/demo-with-go-plugin/testcases/requests.json @@ -22,7 +22,7 @@ "variables": { "foo1": "${ENV(USERNAME)}", "foo2": "bar21", - "sum_v": "${sum_two_int(1, 2)}" + "sum_v": "${sum_two_int(10000000, 20000000)}" }, "request": { "method": "GET", @@ -52,7 +52,7 @@ { "check": "body.args.sum_v", "assert": "equal", - "expect": "3", + "expect": "30000000", "msg": "check body.args.sum_v" }, { diff --git a/examples/demo-with-go-plugin/testcases/requests.yml b/examples/demo-with-go-plugin/testcases/requests.yml index 1db4e4d1..5922ab12 100644 --- a/examples/demo-with-go-plugin/testcases/requests.yml +++ b/examples/demo-with-go-plugin/testcases/requests.yml @@ -16,7 +16,7 @@ teststeps: variables: foo1: ${ENV(USERNAME)} foo2: bar21 - sum_v: "${sum_two_int(1, 2)}" + sum_v: "${sum_two_int(10000000, 20000000)}" request: method: GET url: $base_url/get @@ -29,7 +29,7 @@ teststeps: validate: - eq: ["status_code", 200] - eq: ["body.args.foo1", "debugtalk"] - - eq: ["body.args.sum_v", "3"] + - eq: ["body.args.sum_v", "30000000"] - eq: ["body.args.foo2", "bar21"] - name: post raw text diff --git a/examples/demo-with-py-plugin/proj.json b/examples/demo-with-py-plugin/proj.json index 3cb92bd4..e0ae1816 100644 --- a/examples/demo-with-py-plugin/proj.json +++ b/examples/demo-with-py-plugin/proj.json @@ -1,5 +1,5 @@ { "project_name": "demo-with-py-plugin", - "create_time": "2022-06-14T11:34:26.276891+08:00", + "create_time": "2022-06-14T13:19:05.680344+08:00", "hrp_version": "v4.1.3" } diff --git a/examples/demo-with-py-plugin/testcases/requests.json b/examples/demo-with-py-plugin/testcases/requests.json index 75c464f9..4c01ec74 100644 --- a/examples/demo-with-py-plugin/testcases/requests.json +++ b/examples/demo-with-py-plugin/testcases/requests.json @@ -22,7 +22,7 @@ "variables": { "foo1": "${ENV(USERNAME)}", "foo2": "bar21", - "sum_v": "${sum_two_int(1, 2)}" + "sum_v": "${sum_two_int(10000000, 20000000)}" }, "request": { "method": "GET", @@ -52,7 +52,7 @@ { "check": "body.args.sum_v", "assert": "equal", - "expect": "3", + "expect": "30000000", "msg": "check body.args.sum_v" }, { diff --git a/examples/demo-with-py-plugin/testcases/requests.yml b/examples/demo-with-py-plugin/testcases/requests.yml index 1db4e4d1..5922ab12 100644 --- a/examples/demo-with-py-plugin/testcases/requests.yml +++ b/examples/demo-with-py-plugin/testcases/requests.yml @@ -16,7 +16,7 @@ teststeps: variables: foo1: ${ENV(USERNAME)} foo2: bar21 - sum_v: "${sum_two_int(1, 2)}" + sum_v: "${sum_two_int(10000000, 20000000)}" request: method: GET url: $base_url/get @@ -29,7 +29,7 @@ teststeps: validate: - eq: ["status_code", 200] - eq: ["body.args.foo1", "debugtalk"] - - eq: ["body.args.sum_v", "3"] + - eq: ["body.args.sum_v", "30000000"] - eq: ["body.args.foo2", "bar21"] - name: post raw text diff --git a/hrp/internal/scaffold/templates/testcases/demo_requests.json b/hrp/internal/scaffold/templates/testcases/demo_requests.json index 75c464f9..4c01ec74 100644 --- a/hrp/internal/scaffold/templates/testcases/demo_requests.json +++ b/hrp/internal/scaffold/templates/testcases/demo_requests.json @@ -22,7 +22,7 @@ "variables": { "foo1": "${ENV(USERNAME)}", "foo2": "bar21", - "sum_v": "${sum_two_int(1, 2)}" + "sum_v": "${sum_two_int(10000000, 20000000)}" }, "request": { "method": "GET", @@ -52,7 +52,7 @@ { "check": "body.args.sum_v", "assert": "equal", - "expect": "3", + "expect": "30000000", "msg": "check body.args.sum_v" }, { diff --git a/hrp/internal/scaffold/templates/testcases/demo_requests.yml b/hrp/internal/scaffold/templates/testcases/demo_requests.yml index 1db4e4d1..5922ab12 100644 --- a/hrp/internal/scaffold/templates/testcases/demo_requests.yml +++ b/hrp/internal/scaffold/templates/testcases/demo_requests.yml @@ -16,7 +16,7 @@ teststeps: variables: foo1: ${ENV(USERNAME)} foo2: bar21 - sum_v: "${sum_two_int(1, 2)}" + sum_v: "${sum_two_int(10000000, 20000000)}" request: method: GET url: $base_url/get @@ -29,7 +29,7 @@ teststeps: validate: - eq: ["status_code", 200] - eq: ["body.args.foo1", "debugtalk"] - - eq: ["body.args.sum_v", "3"] + - eq: ["body.args.sum_v", "30000000"] - eq: ["body.args.foo2", "bar21"] - name: post raw text diff --git a/hrp/internal/scaffold/templates/testcases/demo_requests_test.py b/hrp/internal/scaffold/templates/testcases/demo_requests_test.py index dd65d1f0..30d7f022 100644 --- a/hrp/internal/scaffold/templates/testcases/demo_requests_test.py +++ b/hrp/internal/scaffold/templates/testcases/demo_requests_test.py @@ -26,7 +26,7 @@ class TestCaseDemoRequests(HttpRunner): Step( RunRequest("get with params") .with_variables( - **{"foo1": "bar11", "foo2": "bar21", "sum_v": "${sum_two_int(1, 2)}"} + **{"foo1": "bar11", "foo2": "bar21", "sum_v": "${sum_two_int(10000000, 20000000)}"} ) .get("/get") .with_params(**{"foo1": "$foo1", "foo2": "$foo2", "sum_v": "$sum_v"}) @@ -36,7 +36,7 @@ class TestCaseDemoRequests(HttpRunner): .validate() .assert_equal("status_code", 200) .assert_equal("body.args.foo1", "bar11") - .assert_equal("body.args.sum_v", "3") + .assert_equal("body.args.sum_v", "30000000") .assert_equal("body.args.foo2", "bar21") ), Step( diff --git a/hrp/parser.go b/hrp/parser.go index 84482e3a..99e4359b 100644 --- a/hrp/parser.go +++ b/hrp/parser.go @@ -7,6 +7,7 @@ import ( "path" "reflect" "regexp" + "strconv" "strings" "github.com/httprunner/funplugin" @@ -66,13 +67,15 @@ func (p *Parser) ParseHeaders(rawHeaders map[string]string, variablesMapping map } func convertString(raw interface{}) string { - if value, ok := raw.(string); ok { - return value - } else { - // raw is not string, e.g. int, float, etc. - // convert to string - return fmt.Sprintf("%v", raw) + if str, ok := raw.(string); ok { + return str } + if float, ok := raw.(float64); ok { + // f: avoid conversion to exponential notation + return strconv.FormatFloat(float, 'f', -1, 64) + } + // convert to string + return fmt.Sprintf("%v", raw) } func (p *Parser) Parse(raw interface{}, variablesMapping map[string]interface{}) (interface{}, error) { @@ -203,7 +206,7 @@ func (p *Parser) ParseString(raw string, variablesMapping map[string]interface{} // raw_string contains one or many functions, e.g. "abc${add_one(3)}def" matchStartPosition += len(funcMatched[0]) - parsedString += fmt.Sprintf("%v", result) + parsedString += convertString(result) remainedString = raw[matchStartPosition:] log.Debug(). Str("parsedString", parsedString). @@ -232,7 +235,7 @@ func (p *Parser) ParseString(raw string, variablesMapping map[string]interface{} } matchStartPosition += len(varMatched[0]) - parsedString += fmt.Sprintf("%v", varValue) + parsedString += convertString(varValue) remainedString = raw[matchStartPosition:] log.Debug(). Str("parsedString", parsedString). diff --git a/hrp/parser_test.go b/hrp/parser_test.go index ee0b6fb3..ced53b69 100644 --- a/hrp/parser_test.go +++ b/hrp/parser_test.go @@ -613,6 +613,8 @@ func TestConvertString(t *testing.T) { {"123", "123"}, {123, "123"}, {1.23, "1.23"}, + {100000000000, "100000000000"}, // avoid exponential notation + {100000000000.23, "100000000000.23"}, {nil, ""}, } diff --git a/hrp/step_request.go b/hrp/step_request.go index f890a93e..8c4de3a4 100644 --- a/hrp/step_request.go +++ b/hrp/step_request.go @@ -125,7 +125,7 @@ func (r *requestBuilder) prepareHeaders(stepVariables map[string]interface{}) er } r.req.AddCookie(&http.Cookie{ Name: cookieName, - Value: fmt.Sprintf("%v", value), + Value: convertString(value), }) } @@ -163,7 +163,7 @@ func (r *requestBuilder) prepareUrlParams(stepVariables map[string]interface{}) if len(parsedParams) > 0 { queryParams = make(url.Values) for k, v := range parsedParams { - queryParams.Add(k, fmt.Sprint(v)) + queryParams.Add(k, convertString(v)) } } } @@ -217,7 +217,7 @@ func (r *requestBuilder) prepareBody(stepVariables map[string]interface{}) error // post form data formData := make(url.Values) for k, v := range vv { - formData.Add(k, fmt.Sprint(v)) + formData.Add(k, convertString(v)) } dataBytes = []byte(formData.Encode()) } else { From bea663f5948c829c191e8c4671085e98a3f878bc Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 14 Jun 2022 13:29:08 +0800 Subject: [PATCH 10/12] change: update changelog --- docs/CHANGELOG.md | 3 +-- hrp/internal/builtin/utils_windows.go | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 9262a3f3..d7ea9f40 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,9 +5,8 @@ **go version** - feat #1342: support specify custom python3 venv, priority is greater than $HOME/.hrp/venv -- feat: assert python3 package is ready with specified version +- feat: assert python3 package installed and version matched - refactor: build plugin mechanism, cancel automatic installation of dependencies -- fix: pip upgrade httprunner when installing hrp - fix #1352: avoid conversion to exponential notation **python version** diff --git a/hrp/internal/builtin/utils_windows.go b/hrp/internal/builtin/utils_windows.go index db1c2b59..1946ba0c 100644 --- a/hrp/internal/builtin/utils_windows.go +++ b/hrp/internal/builtin/utils_windows.go @@ -18,9 +18,7 @@ func getPython3Executable(venvDir string) string { return filepath.Join(venvDir, "Scripts", "python3.exe") } -// EnsurePython3Venv ensures python3 venv for hashicorp python plugin -// venvDir should be directory path of target venv -func EnsurePython3Venv(venvDir string, packages ...string) (python3 string, err error) { +func ensurePython3Venv(venvDir string, packages ...string) (python3 string, err error) { python3 = getPython3Executable(venvDir) log.Info(). Str("python3", python3). From 8731f6800dbcb9da554f0bc7eab6ce19ad5b1c7d Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 14 Jun 2022 14:02:28 +0800 Subject: [PATCH 11/12] change: set pypi index url to https://pypi.org/simple --- hrp/internal/builtin/utils.go | 1 + scripts/install.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hrp/internal/builtin/utils.go b/hrp/internal/builtin/utils.go index 0d05339a..6fd12148 100644 --- a/hrp/internal/builtin/utils.go +++ b/hrp/internal/builtin/utils.go @@ -168,6 +168,7 @@ func InstallPythonPackage(python3 string, pkg string) (err error) { // install package err = ExecCommand(python3, "-m", "pip", "install", "--upgrade", pkg, + "--index-url", "https://pypi.org/simple", "--quiet", "--disable-pip-version-check") if err != nil { return errors.Wrap(err, "pip install package failed") diff --git a/scripts/install.sh b/scripts/install.sh index 23dde6fb..709167ff 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -139,8 +139,8 @@ function main() { if [[ -f $HOME/.hrp/venv/bin/pip3 ]]; then echoInfo "Upgrade httprunner..." - echo "$ $HOME/.hrp/venv/bin/pip3 install --upgrade httprunner==$version" - $HOME/.hrp/venv/bin/pip3 install --upgrade httprunner==$version + echo "$ $HOME/.hrp/venv/bin/pip3 install --upgrade httprunner==$version --index-url https://pypi.org/simple" + $HOME/.hrp/venv/bin/pip3 install --upgrade httprunner==$version --index-url https://pypi.org/simple fi } From 2db8beefcf606f67ecd399afd92d35051ef4e29b Mon Sep 17 00:00:00 2001 From: debugtalk Date: Tue, 14 Jun 2022 14:19:48 +0800 Subject: [PATCH 12/12] fix: add cmd dir path to /Users/debugtalk/MyProjects/HttpRunner-dev/httprunner/examples/demo-with-py-plugin/.venv/bin:/Users/debugtalk/Library/Python/3.9/bin:/Users/debugtalk/.poetry/bin:/usr/local/opt/node@12/bin:/Users/debugtalk/.gvm/bin:/Users/debugtalk/go/go1.16.3/bin:/Users/debugtalk/go/bin:/usr/local/sbin:/Users/debugtalk/.pyenv/bin:/Users/debugtalk/.poetry/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin:/opt/puppetlabs/bin:/Users/debugtalk/Library/Python/3.9/bin:/Users/debugtalk/.poetry/bin:/usr/local/opt/node@12/bin:/Users/debugtalk/.gvm/bin:/Users/debugtalk/go/go1.16.3/bin:/Users/debugtalk/go/bin:/usr/local/sbin:/Users/debugtalk/.pyenv/bin:/Users/debugtalk/.local/bin:/Users/debugtalk/.local/bin:/Users/debugtalk/go/bin:/Users/debugtalk/.local/bin --- .../request_methods/request_with_functions_test.py | 2 +- hrp/internal/builtin/utils_unix.go | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/postman_echo/request_methods/request_with_functions_test.py b/examples/postman_echo/request_methods/request_with_functions_test.py index a6f8ce0b..88be4975 100644 --- a/examples/postman_echo/request_methods/request_with_functions_test.py +++ b/examples/postman_echo/request_methods/request_with_functions_test.py @@ -1,4 +1,4 @@ -# NOTE: Generated By HttpRunner v4.0.0 +# NOTE: Generated By HttpRunner v4.1.3 # FROM: request_methods/request_with_functions.yml diff --git a/hrp/internal/builtin/utils_unix.go b/hrp/internal/builtin/utils_unix.go index 042b2869..f375fc8e 100644 --- a/hrp/internal/builtin/utils_unix.go +++ b/hrp/internal/builtin/utils_unix.go @@ -62,6 +62,15 @@ func ExecCommand(cmdName string, args ...string) error { cmd := exec.Command(cmdName, args...) log.Info().Str("cmd", cmd.String()).Msg("exec command") + // add cmd dir path to $PATH + if cmdDir := filepath.Dir(cmdName); cmdDir != "" { + PATH := fmt.Sprintf("%s:%s", cmdDir, os.Getenv("PATH")) + if err := os.Setenv("PATH", PATH); err != nil { + log.Error().Err(err).Msg("set env $PATH failed") + return err + } + } + // print output with colors cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr