diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f6956a8e..a0c4ae05 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,7 +1,10 @@ # Release History -## v4.1.3 (2022-06-12) +## v4.1.3 (2022-06-13) +**go version** + +- feat #1342: support specify custom python3 venv - fix: pip upgrade httprunner when installing hrp ## v4.1.2 (2022-06-09) diff --git a/docs/cmd/hrp.md b/docs/cmd/hrp.md index 7a58c565..810a22ad 100644 --- a/docs/cmd/hrp.md +++ b/docs/cmd/hrp.md @@ -37,4 +37,4 @@ Copyright 2017 debugtalk * [hrp startproject](hrp_startproject.md) - create a scaffold project * [hrp wiki](hrp_wiki.md) - visit https://httprunner.com -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_boom.md b/docs/cmd/hrp_boom.md index a13c1168..7a9cfbca 100644 --- a/docs/cmd/hrp_boom.md +++ b/docs/cmd/hrp_boom.md @@ -42,4 +42,4 @@ hrp boom [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_build.md b/docs/cmd/hrp_build.md index 8b886d77..73dccf0b 100644 --- a/docs/cmd/hrp_build.md +++ b/docs/cmd/hrp_build.md @@ -28,4 +28,4 @@ hrp build $path ... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_convert.md b/docs/cmd/hrp_convert.md index 8105d41e..4160891c 100644 --- a/docs/cmd/hrp_convert.md +++ b/docs/cmd/hrp_convert.md @@ -22,4 +22,4 @@ hrp convert $path... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_pytest.md b/docs/cmd/hrp_pytest.md index 70be0989..b48bf18e 100644 --- a/docs/cmd/hrp_pytest.md +++ b/docs/cmd/hrp_pytest.md @@ -16,4 +16,4 @@ hrp pytest $path ... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_run.md b/docs/cmd/hrp_run.md index ad37cb40..cc859ed3 100644 --- a/docs/cmd/hrp_run.md +++ b/docs/cmd/hrp_run.md @@ -35,4 +35,4 @@ hrp run $path... [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_startproject.md b/docs/cmd/hrp_startproject.md index 05c015da..78953953 100644 --- a/docs/cmd/hrp_startproject.md +++ b/docs/cmd/hrp_startproject.md @@ -21,4 +21,4 @@ hrp startproject $project_name [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/docs/cmd/hrp_wiki.md b/docs/cmd/hrp_wiki.md index dcfd35ea..7283ba32 100644 --- a/docs/cmd/hrp_wiki.md +++ b/docs/cmd/hrp_wiki.md @@ -16,4 +16,4 @@ hrp wiki [flags] * [hrp](hrp.md) - Next-Generation API Testing Solution. -###### Auto generated by spf13/cobra on 9-Jun-2022 +###### Auto generated by spf13/cobra on 13-Jun-2022 diff --git a/go.mod b/go.mod index 633c73b7..6d70a855 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/go-openapi/spec v0.20.6 github.com/google/uuid v1.3.0 github.com/gorilla/websocket v1.4.1 - github.com/httprunner/funplugin v0.4.9 + github.com/httprunner/funplugin v0.5.0 github.com/jinzhu/copier v0.3.2 github.com/jmespath/go-jmespath v0.4.0 github.com/json-iterator/go v1.1.12 diff --git a/go.sum b/go.sum index 466fe49c..3f819360 100644 --- a/go.sum +++ b/go.sum @@ -253,8 +253,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/httprunner/funplugin v0.4.9 h1:gmF1sP5D4/nvdocqgOAyT3GpVDz3fL4ErZ17WHo8x9U= -github.com/httprunner/funplugin v0.4.9/go.mod h1:vPyeJIfbpGe0epZZtAV0wCn16gLY9+imSw/zfxq0Lcc= +github.com/httprunner/funplugin v0.5.0 h1:Laoe8URu71qeyST9wvRtGSkDWc8Y3T1IrnvFSTHmO84= +github.com/httprunner/funplugin v0.5.0/go.mod h1:vPyeJIfbpGe0epZZtAV0wCn16gLY9+imSw/zfxq0Lcc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= diff --git a/hrp/boomer.go b/hrp/boomer.go index caef351c..839ebd15 100644 --- a/hrp/boomer.go +++ b/hrp/boomer.go @@ -29,9 +29,16 @@ type HRPBoomer struct { pluginsMutex *sync.RWMutex // avoid data race } -func (b *HRPBoomer) SetClientTransport() { +func (b *HRPBoomer) SetClientTransport() *HRPBoomer { // set client transport for high concurrency load testing b.hrpRunner.SetClientTransport(b.GetSpawnCount(), b.GetDisableKeepAlive(), b.GetDisableCompression()) + return b +} + +// SetPython3Venv specifies python3 venv. +func (b *HRPBoomer) SetPython3Venv(venv string) *HRPBoomer { + b.hrpRunner.SetPython3Venv(venv) + return b } // Run starts to run load test for one or multiple testcases. diff --git a/hrp/build.go b/hrp/build.go index 3184fa3c..94692317 100644 --- a/hrp/build.go +++ b/hrp/build.go @@ -12,16 +12,15 @@ import ( "strings" "text/template" + "github.com/httprunner/funplugin/shared" "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/httprunner/funplugin/shared" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/version" ) const ( - funppy = `import funppy` fungo = `"github.com/httprunner/funplugin/fungo"` regexPythonFunctionName = `def ([a-zA-Z_]\w*)\(.*\)` regexGoImports = `import \(([\s\S]*?)\)` @@ -38,7 +37,6 @@ var goTemplate string type TemplateContent struct { Version string // hrp version - Fun string // funplugin package Regexps *Regexps // match import/function Imports []string // python/go import FromImports []string // python from...import... @@ -81,7 +79,7 @@ func (t *TemplateContent) parseGoContent(path string) error { } // import fungo package if !builtin.Contains(t.Imports, fungo) { - t.Imports = append(t.Imports, t.Fun) + t.Imports = append(t.Imports, fungo) } // parse function name @@ -155,15 +153,11 @@ func (t *TemplateContent) parsePyContent(path string) error { // function content t.Functions = append(t.Functions, strings.Trim(content, "\n")) - // import funppy - if !builtin.Contains(t.Imports, t.Fun) { - t.Imports = append(t.Imports, t.Fun) - } return nil } -func (t *TemplateContent) genDebugTalk(path string, templ string) error { - file, err := os.Create(path) +func (t *TemplateContent) genDebugTalk(output string, templ string) error { + file, err := os.Create(output) if err != nil { log.Error().Err(err).Msg("open file failed") return err @@ -178,9 +172,9 @@ func (t *TemplateContent) genDebugTalk(path string, templ string) error { } err = writer.Flush() if err == nil { - log.Info().Str("path", path).Msg("generate debugtalk success") + log.Info().Str("output", output).Msg("generate debugtalk success") } else { - log.Error().Str("path", path).Msg("generate debugtalk failed") + log.Error().Str("output", output).Msg("generate debugtalk failed") } return err } @@ -189,7 +183,6 @@ func (t *TemplateContent) genDebugTalk(path string, templ string) error { func buildGo(path string, output string) error { templateContent := &TemplateContent{ Version: version.VERSION, - Fun: fungo, Regexps: &Regexps{ Import: regexp.MustCompile(regexGoImport), Imports: regexp.MustCompile(regexGoImports), @@ -260,7 +253,6 @@ func buildGo(path string, output string) error { func buildPy(path string, output string) error { templateContent := &TemplateContent{ Version: version.VERSION, - Fun: funppy, Regexps: &Regexps{ FunctionName: regexp.MustCompile(regexPythonFunctionName), }, @@ -271,12 +263,6 @@ func buildPy(path string, output string) error { return err } - // ensure installation of packages - _, err = builtin.EnsurePython3Venv(templateContent.Packages...) - if err != nil { - return err - } - // check the syntax of debugtalk.py err = builtin.CheckPythonScriptSyntax(path) if err != nil { diff --git a/hrp/cmd/boom.go b/hrp/cmd/boom.go index 5ae038e1..ca48d13b 100644 --- a/hrp/cmd/boom.go +++ b/hrp/cmd/boom.go @@ -54,6 +54,9 @@ var boomCmd = &cobra.Command{ hrpBoomer.SetDisableKeepAlive(boomArgs.DisableKeepalive) hrpBoomer.SetDisableCompression(boomArgs.DisableCompression) hrpBoomer.SetClientTransport() + if venv != "" { + hrpBoomer.SetPython3Venv(venv) + } hrpBoomer.EnableCPUProfile(boomArgs.CPUProfile, boomArgs.CPUProfileDuration) hrpBoomer.EnableMemoryProfile(boomArgs.MemoryProfile, boomArgs.MemoryProfileDuration) hrpBoomer.EnableGracefulQuit() diff --git a/hrp/cmd/build.go b/hrp/cmd/build.go index 3c8848e3..9a8c2bb8 100644 --- a/hrp/cmd/build.go +++ b/hrp/cmd/build.go @@ -1,9 +1,11 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/httprunner/httprunner/v4/hrp" + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" ) var buildCmd = &cobra.Command{ @@ -17,6 +19,11 @@ var buildCmd = &cobra.Command{ setLogLevel(logLevel) }, RunE: func(cmd *cobra.Command, args []string) error { + err := builtin.PrepareVenv(venv) + if err != nil { + log.Error().Err(err).Msg("prepare python3 venv failed") + return err + } return hrp.BuildPlugin(args[0], output) }, } diff --git a/hrp/cmd/convert.go b/hrp/cmd/convert.go index 83ba88bb..ee43c2de 100644 --- a/hrp/cmd/convert.go +++ b/hrp/cmd/convert.go @@ -3,8 +3,10 @@ package cmd import ( "errors" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/convert" ) @@ -32,6 +34,12 @@ var convertCmd = &cobra.Command{ if toPyTestFlag { flagCount++ outputType = convert.OutputTypePyTest + + err := builtin.PrepareVenv(venv) + if err != nil { + log.Error().Err(err).Msg("prepare python3 venv failed") + return err + } } if flagCount > 1 { return errors.New("please specify at most one conversion flag") diff --git a/hrp/cmd/pytest.go b/hrp/cmd/pytest.go index fd7732fb..4d49b2be 100644 --- a/hrp/cmd/pytest.go +++ b/hrp/cmd/pytest.go @@ -1,8 +1,10 @@ package cmd import ( + "github.com/rs/zerolog/log" "github.com/spf13/cobra" + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/pytest" ) @@ -15,6 +17,11 @@ var pytestCmd = &cobra.Command{ }, DisableFlagParsing: true, // allow to pass any args to pytest RunE: func(cmd *cobra.Command, args []string) error { + err := builtin.PrepareVenv(venv) + if err != nil { + log.Error().Err(err).Msg("prepare python3 venv failed") + return err + } return pytest.RunPytest(args) }, } diff --git a/hrp/cmd/root.go b/hrp/cmd/root.go index 2909ea15..698d0598 100644 --- a/hrp/cmd/root.go +++ b/hrp/cmd/root.go @@ -48,6 +48,7 @@ Copyright 2017 debugtalk`, var ( logLevel string logJSON bool + venv string ) // Execute adds all child commands to the root command and sets flags appropriately. @@ -55,6 +56,7 @@ var ( func Execute() { rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", "INFO", "set log level") rootCmd.PersistentFlags().BoolVar(&logJSON, "log-json", false, "set log to json format") + rootCmd.PersistentFlags().StringVar(&venv, "venv", "", "specify python3 venv path") if err := rootCmd.Execute(); err != nil { os.Exit(1) diff --git a/hrp/cmd/run.go b/hrp/cmd/run.go index 500e595b..5c7bcd90 100644 --- a/hrp/cmd/run.go +++ b/hrp/cmd/run.go @@ -41,6 +41,9 @@ var runCmd = &cobra.Command{ if pluginLogOn { runner.SetPluginLogOn() } + if venv != "" { + runner.SetPython3Venv(venv) + } if proxyUrl != "" { runner.SetProxyUrl(proxyUrl) } diff --git a/hrp/cmd/scaffold.go b/hrp/cmd/scaffold.go index c51281ed..86830725 100644 --- a/hrp/cmd/scaffold.go +++ b/hrp/cmd/scaffold.go @@ -34,7 +34,7 @@ var scaffoldCmd = &cobra.Command{ pluginType = scaffold.Py // default } - err := scaffold.CreateScaffold(args[0], pluginType, force) + err := scaffold.CreateScaffold(args[0], pluginType, venv, force) if err != nil { log.Error().Err(err).Msg("create scaffold project failed") os.Exit(1) diff --git a/hrp/internal/builtin/utils.go b/hrp/internal/builtin/utils.go index 93a4b64b..0fd52db5 100644 --- a/hrp/internal/builtin/utils.go +++ b/hrp/internal/builtin/utils.go @@ -13,12 +13,13 @@ import ( "strconv" "strings" - "github.com/httprunner/funplugin/shared" + "github.com/httprunner/funplugin/fungo" "github.com/pkg/errors" "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" "github.com/httprunner/httprunner/v4/hrp/internal/json" + "github.com/httprunner/httprunner/v4/hrp/internal/version" ) func Dump2JSON(data interface{}, path string) error { @@ -88,24 +89,104 @@ func FormatResponse(raw interface{}) interface{} { return formattedResponse } -func EnsurePython3Venv(packages ...string) (string, error) { - // create python venv +var Python3Executable string = "python3" // system default python3 + +func PrepareVenv(venv string) error { + defer func() { + log.Info().Str("Python3Executable", Python3Executable).Msg("set python3 executable path") + }() + + // specify python3 venv + if venv != "" { + Python3Executable = getPython3Executable(venv) + return nil + } + + // not specify python3 venv, create default venv + python3, err := createDefaultPython3Venv() + if err != nil { + return errors.Wrap(err, "create default python3 venv failed") + } + Python3Executable = python3 + + return nil +} + +// create default python3 venv located in $HOME/.hrp/venv +func createDefaultPython3Venv() (string, error) { + log.Info().Msg("create default python3 venv at $HOME/.hrp/venv") home, err := os.UserHomeDir() if err != nil { return "", errors.Wrap(err, "get user home dir failed") } - venvDir := filepath.Join(home, ".hrp", "venv") - python3, err := shared.EnsurePython3Venv(venvDir, packages...) - if err != nil { - return "", errors.Wrap(err, "ensure python3 venv failed") + venv := filepath.Join(home, ".hrp", "venv") + packages := []string{ + fmt.Sprintf("funppy==%s", fungo.Version), + fmt.Sprintf("httprunner==%s", version.VERSION), + } + return EnsurePython3Venv(venv, packages...) +} + +func ExecPython3Command(cmdName string, args ...string) error { + args = append([]string{"-m", cmdName}, args...) + return ExecCommand(Python3Executable, args...) +} + +func InstallPythonPackage(python3 string, pkg string) (err error) { + var pkgName string + if strings.Contains(pkg, "==") { + // funppy==0.4.2 + pkgInfo := strings.Split(pkg, "==") + pkgName = pkgInfo[0] + } else if strings.Contains(pkg, ">=") { + // httprunner>=4.0.0-beta + pkgInfo := strings.Split(pkg, ">=") + pkgName = pkgInfo[0] + } else { + pkgName = pkg } - return python3, nil + defer func() { + if err == nil { + // check package version + if out, err := exec.Command( + python3, "-c", fmt.Sprintf("import %s; print(%s.__version__)", pkgName, pkgName), + ).Output(); err == nil { + log.Info(). + Str("name", pkgName). + Str("version", strings.TrimSpace(string(out))). + Msg("python package is ready") + } + } + }() + + // check if package installed + err = exec.Command(python3, "-c", fmt.Sprintf("import %s", pkgName)).Run() + if err == nil { + return nil + } + + // check if pip available + err = ExecCommand(python3, "-m", "pip", "--version") + if err != nil { + log.Warn().Msg("pip is not available") + return errors.Wrap(err, "pip is not available") + } + + log.Info().Str("package", pkg).Msg("installing python package") + + // install package + err = ExecCommand(python3, "-m", "pip", "install", "--upgrade", pkg, + "--quiet", "--disable-pip-version-check") + if err != nil { + return errors.Wrap(err, "pip install package failed") + } + + return nil } func CheckPythonScriptSyntax(path string) error { - err := ExecCommand("python3", "-m", "py_compile", path) - return err + return ExecCommand("python3", "-m", "py_compile", path) } func ExecCommandInDir(cmd *exec.Cmd, dir string) error { @@ -125,30 +206,6 @@ func ExecCommandInDir(cmd *exec.Cmd, dir string) error { return nil } -func ExecCommand(cmdName string, args ...string) error { - cmd := exec.Command(cmdName, args...) - log.Info().Str("cmd", cmd.String()).Msg("exec command") - - // print output with colors - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - // add cmd dir path to PATH - PATH := fmt.Sprintf("%s:%s", filepath.Dir(cmdName), os.Getenv("PATH")) - if err := os.Setenv("PATH", PATH); err != nil { - log.Error().Err(err).Msg("failed to add cmd dir path to $PATH") - return err - } - - err := cmd.Run() - if err != nil { - log.Error().Err(err).Msg("exec command failed") - return err - } - - return err -} - func CreateFolder(folderPath string) error { log.Info().Str("path", folderPath).Msg("create folder") err := os.MkdirAll(folderPath, os.ModePerm) diff --git a/hrp/internal/builtin/utils_unix.go b/hrp/internal/builtin/utils_unix.go new file mode 100644 index 00000000..510ac0c0 --- /dev/null +++ b/hrp/internal/builtin/utils_unix.go @@ -0,0 +1,77 @@ +// +build darwin linux + +package builtin + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + +func getPython3Executable(venvDir string) string { + return filepath.Join(venvDir, "bin", "python3") +} + +// EnsurePython3Venv ensures python3 venv for hashicorp python plugin +// venvDir should be directory path of target venv +func EnsurePython3Venv(venvDir string, packages ...string) (python3 string, err error) { + python3 = getPython3Executable(venvDir) + + log.Info(). + Str("python3", python3). + Interface("packages", packages). + Msg("ensure python3 venv") + + // check if python3 venv is available + if err := exec.Command(python3, "--version").Run(); err != nil { + // python3 venv not available, create one + // check if system python3 is available + if err := ExecCommand("python3", "--version"); err != nil { + return "", errors.Wrap(err, "python3 not found") + } + + // check if .venv exists + if _, err := os.Stat(venvDir); err == nil { + // .venv exists, remove first + if err := ExecCommand("rm", "-rf", venvDir); err != nil { + return "", errors.Wrap(err, "remove existed venv failed") + } + } + + // create python3 .venv + if err := ExecCommand("python3", "-m", "venv", venvDir); err != nil { + return "", errors.Wrap(err, "create python3 venv failed") + } + } + + // install default python packages + for _, pkg := range packages { + err := InstallPythonPackage(python3, pkg) + if err != nil { + return "", errors.Wrap(err, fmt.Sprintf("pip install %s failed", pkg)) + } + } + + return python3, nil +} + +func ExecCommand(cmdName string, args ...string) error { + cmd := exec.Command(cmdName, args...) + log.Info().Str("cmd", cmd.String()).Msg("exec command") + + // print output with colors + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + log.Error().Err(err).Msg("exec command failed") + return err + } + + return nil +} diff --git a/hrp/internal/builtin/utils_windows.go b/hrp/internal/builtin/utils_windows.go new file mode 100644 index 00000000..8501f082 --- /dev/null +++ b/hrp/internal/builtin/utils_windows.go @@ -0,0 +1,100 @@ +// +build windows + +package builtin + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + +func getPython3Executable(venvDir string) string { + return filepath.Join(venvDir, "Scripts", "python3.exe") +} + +// EnsurePython3Venv ensures python3 venv for hashicorp python plugin +// venvDir should be directory path of target venv +func EnsurePython3Venv(venvDir string, packages ...string) (python3 string, err error) { + python3 = getPython3Executable(venvDir) + log.Info(). + Str("python3", python3). + Interface("packages", packages). + Msg("ensure python3 venv") + + systemPython := "python3" + + // check if python3 venv is available + if err := exec.Command("cmd", "/c", python3, "--version").Run(); err != nil { + // python3 venv not available, create one + // check if system python3 is available + if err := ExecCommand(systemPython, "--version"); err != nil { + if err := ExecCommand("python", "--version"); err != nil { + return "", errors.Wrap(err, "python3 not found") + } + systemPython = "python" + } + + // check if .venv exists + if _, err := os.Stat(venvDir); err == nil { + // .venv exists, remove first + if err := ExecCommand("del", "/q", venvDir); err != nil { + return "", errors.Wrap(err, "remove existed venv failed") + } + } + + // create python3 .venv + // notice: --symlinks should be specified for windows + // https://github.com/actions/virtual-environments/issues/2690 + if err := ExecCommand(systemPython, "-m", "venv", "--symlinks", venvDir); err != nil { + // fix: failed to symlink on Windows + log.Warn().Msg("failed to create python3 .venv by using --symlinks, try to use --copies") + if err := ExecCommand(systemPython, "-m", "venv", "--copies", venvDir); err != nil { + return "", errors.Wrap(err, "create python3 venv failed") + } + } + + // fix: python3 doesn't exist in .venv on Windows + if _, err := os.Stat(python3); err != nil { + log.Warn().Msg("python3 doesn't exist, try to link python") + err := os.Link(filepath.Join(venvDir, "Scripts", "python.exe"), python3) + if err != nil { + return "", errors.Wrap(err, "python3 doesn't exist in .venv") + } + } + } + + // install default python packages + for _, pkg := range packages { + err := InstallPythonPackage(python3, pkg) + if err != nil { + return "", errors.Wrap(err, fmt.Sprintf("pip install %s failed", pkg)) + } + } + + return python3, nil +} + +func ExecCommand(cmdName string, args ...string) error { + // "cmd /c" carries out the command specified by string and then stops + // refer: https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/cmd + cmdStr := fmt.Sprintf("%s %s", cmdName, strings.Join(args, " ")) + cmd := exec.Command("cmd", "/c", cmdStr) + log.Info().Str("cmd", cmd.String()).Msg("exec command") + + // print output with colors + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + log.Error().Err(err).Msg("exec command failed") + return err + } + + return nil +} diff --git a/hrp/internal/convert/converter_gotest.go b/hrp/internal/convert/converter_gotest.go index 863da231..04897c76 100644 --- a/hrp/internal/convert/converter_gotest.go +++ b/hrp/internal/convert/converter_gotest.go @@ -48,12 +48,7 @@ func convert2GoTestScripts(paths ...string) error { } // format pytest scripts with black - python3, err := builtin.EnsurePython3Venv("black") - if err != nil { - return err - } - args := append([]string{"-m", "black"}, pytestPaths...) - return builtin.ExecCommand(python3, args...) + return builtin.ExecPython3Command("black", pytestPaths...) } //go:embed testcase.tmpl diff --git a/hrp/internal/convert/converter_json.go b/hrp/internal/convert/converter_json.go index fc380142..f55a7f5c 100644 --- a/hrp/internal/convert/converter_json.go +++ b/hrp/internal/convert/converter_json.go @@ -1,13 +1,10 @@ package convert import ( - "fmt" - "github.com/pkg/errors" "github.com/httprunner/httprunner/v4/hrp" "github.com/httprunner/httprunner/v4/hrp/internal/builtin" - "github.com/httprunner/httprunner/v4/hrp/internal/version" ) func NewConverterJSON(converter *TCaseConverter) *ConverterJSON { @@ -51,7 +48,7 @@ func (c *ConverterJSON) ToYAML() (string, error) { } func (c *ConverterJSON) ToGoTest() (string, error) { - //TODO implement me + // TODO implement me return "", errors.New("convert from json testcase to gotest scripts is not supported yet") } @@ -60,13 +57,8 @@ func (c *ConverterJSON) ToPyTest() (string, error) { } func (c *ConverterJSON) MakePyTestScript() (string, error) { - httprunner := fmt.Sprintf("httprunner>=%s", version.HttpRunnerMinVersion) - python3, err := builtin.EnsurePython3Venv(httprunner) - if err != nil { - return "", err - } - args := append([]string{"-m", "httprunner", "make"}, c.converter.InputPath) - err = builtin.ExecCommand(python3, args...) + args := append([]string{"make"}, c.converter.InputPath) + err := builtin.ExecPython3Command("httprunner", args...) if err != nil { return "", err } diff --git a/hrp/internal/convert/converter_pytest.go b/hrp/internal/convert/converter_pytest.go index 8c094900..75c1490f 100644 --- a/hrp/internal/convert/converter_pytest.go +++ b/hrp/internal/convert/converter_pytest.go @@ -1,19 +1,10 @@ package convert import ( - "fmt" - "github.com/httprunner/httprunner/v4/hrp/internal/builtin" - "github.com/httprunner/httprunner/v4/hrp/internal/version" ) func convert2PyTestScripts(paths ...string) error { - httprunner := fmt.Sprintf("httprunner>=%s", version.HttpRunnerMinVersion) - python3, err := builtin.EnsurePython3Venv(httprunner) - if err != nil { - return err - } - - args := append([]string{"-m", "httprunner", "make"}, paths...) - return builtin.ExecCommand(python3, args...) + args := append([]string{"make"}, paths...) + return builtin.ExecPython3Command("httprunner", args...) } diff --git a/hrp/internal/pytest/main.go b/hrp/internal/pytest/main.go index a1c0c88c..dde5f0ae 100644 --- a/hrp/internal/pytest/main.go +++ b/hrp/internal/pytest/main.go @@ -1,11 +1,8 @@ package pytest import ( - "fmt" - "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/sdk" - "github.com/httprunner/httprunner/v4/hrp/internal/version" ) func RunPytest(args []string) error { @@ -14,12 +11,6 @@ func RunPytest(args []string) error { Action: "hrp pytest", }) - httprunner := fmt.Sprintf("httprunner>=%s", version.HttpRunnerMinVersion) - python3, err := builtin.EnsurePython3Venv(httprunner) - if err != nil { - return err - } - - args = append([]string{"-m", "httprunner", "run"}, args...) - return builtin.ExecCommand(python3, args...) + args = append([]string{"run"}, args...) + return builtin.ExecPython3Command("httprunner", args...) } diff --git a/hrp/internal/scaffold/examples_test.go b/hrp/internal/scaffold/examples_test.go index d480ffb6..0590994d 100644 --- a/hrp/internal/scaffold/examples_test.go +++ b/hrp/internal/scaffold/examples_test.go @@ -6,25 +6,25 @@ import ( func TestGenDemoExamples(t *testing.T) { dir := "../../../examples/demo-with-go-plugin" - err := CreateScaffold(dir, Go, true) + err := CreateScaffold(dir, Go, "", true) if err != nil { t.Fatal() } dir = "../../../examples/demo-with-py-plugin" - err = CreateScaffold(dir, Py, true) + err = CreateScaffold(dir, Py, "", true) if err != nil { t.Fatal() } dir = "../../../examples/demo-without-plugin" - err = CreateScaffold(dir, Ignore, true) + err = CreateScaffold(dir, Ignore, "", true) if err != nil { t.Fatal() } dir = "../../../examples/demo-empty-project" - err = CreateScaffold(dir, Empty, true) + err = CreateScaffold(dir, Empty, "", true) if err != nil { t.Fatal() } diff --git a/hrp/internal/scaffold/main.go b/hrp/internal/scaffold/main.go index 04ad8ac5..ed9405a6 100644 --- a/hrp/internal/scaffold/main.go +++ b/hrp/internal/scaffold/main.go @@ -8,6 +8,7 @@ import ( "path/filepath" "time" + "github.com/httprunner/funplugin/fungo" "github.com/pkg/errors" "github.com/rs/zerolog/log" @@ -51,7 +52,7 @@ func CopyFile(templateFile, targetFile string) error { return nil } -func CreateScaffold(projectName string, pluginType PluginType, force bool) error { +func CreateScaffold(projectName string, pluginType PluginType, venv string, force bool) error { // report event sdk.SendEvent(sdk.EventTracking{ Category: "Scaffold", @@ -165,7 +166,7 @@ func CreateScaffold(projectName string, pluginType PluginType, force bool) error // create debugtalk function plugin switch pluginType { case Py: - return createPythonPlugin(projectName) + return createPythonPlugin(projectName, venv) case Go: return createGoPlugin(projectName) } @@ -194,7 +195,7 @@ func createGoPlugin(projectName string) error { return nil } -func createPythonPlugin(projectName string) error { +func createPythonPlugin(projectName, venv string) error { log.Info().Msg("start to create hashicorp python plugin") // create debugtalk.py @@ -204,7 +205,14 @@ func createPythonPlugin(projectName string) error { return errors.Wrap(err, "copy file failed") } - _, err = builtin.EnsurePython3Venv("funppy") + if venv == "" { + venv = filepath.Join(projectName, ".venv") + } + packages := []string{ + fmt.Sprintf("funppy==%s", fungo.Version), + fmt.Sprintf("httprunner==%s", version.VERSION), + } + _, err = builtin.EnsurePython3Venv(venv, packages...) if err != nil { return err } diff --git a/hrp/internal/scaffold/templates/plugin/debugtalkGoTemplate b/hrp/internal/scaffold/templates/plugin/debugtalkGoTemplate index d5b096bc..be50fa2c 100644 --- a/hrp/internal/scaffold/templates/plugin/debugtalkGoTemplate +++ b/hrp/internal/scaffold/templates/plugin/debugtalkGoTemplate @@ -2,12 +2,12 @@ package main import ( - "github.com/httprunner/funplugin/fungo" + "github.com/httprunner/funplugin/fungo" ) func main() { {{- range $functionName := .FunctionNames }} - fungo.Register("{{ $functionName }}", {{ $functionName }}) + fungo.Register("{{ $functionName }}", {{ $functionName }}) {{- end }} fungo.Serve() } diff --git a/hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate b/hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate index dc16657a..b892dd30 100644 --- a/hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate +++ b/hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate @@ -12,6 +12,7 @@ {{ end }} if __name__ == "__main__": + import funppy {{- range $functionName := .FunctionNames }} funppy.register("{{ $functionName }}", {{ $functionName }}) {{- end }} diff --git a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go index 1e6ceac9..df7727c3 100644 --- a/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go +++ b/hrp/internal/scaffold/templates/plugin/debugtalk_gen.go @@ -1,4 +1,4 @@ -// NOTE: Generated By hrp v4.1.2, DO NOT EDIT! +// NOTE: Generated By hrp v4.1.3, DO NOT EDIT! package main import ( diff --git a/hrp/internal/version/init.go b/hrp/internal/version/init.go index 836ffca7..4887463b 100644 --- a/hrp/internal/version/init.go +++ b/hrp/internal/version/init.go @@ -6,5 +6,3 @@ import ( //go:embed VERSION var VERSION string - -const HttpRunnerMinVersion = "v4.1.0" diff --git a/hrp/plugin.go b/hrp/plugin.go index 5053b2e5..91bdedd8 100644 --- a/hrp/plugin.go +++ b/hrp/plugin.go @@ -11,6 +11,7 @@ import ( "github.com/httprunner/funplugin" "github.com/rs/zerolog/log" + "github.com/httprunner/httprunner/v4/hrp/internal/builtin" "github.com/httprunner/httprunner/v4/hrp/internal/sdk" ) @@ -25,7 +26,7 @@ const ( const projectInfoFile = "proj.json" // used for ensuring root project -func initPlugin(path string, logOn bool) (plugin funplugin.IPlugin, err error) { +func initPlugin(path, venv string, logOn bool) (plugin funplugin.IPlugin, err error) { // plugin file not found if path == "" { return nil, nil @@ -35,6 +36,8 @@ func initPlugin(path string, logOn bool) (plugin funplugin.IPlugin, err error) { return nil, nil } + pluginOptions := []funplugin.Option{funplugin.WithLogOn(logOn)} + if strings.HasSuffix(pluginPath, ".py") { // register funppy plugin genPyPluginPath := filepath.Join(filepath.Dir(pluginPath), PluginPySourceGenFile) @@ -44,10 +47,21 @@ func initPlugin(path string, logOn bool) (plugin funplugin.IPlugin, err error) { return nil, nil } pluginPath = genPyPluginPath + + // priority: specified > projectDir/.venv > $HOME/.hrp/venv + if venv == "" && builtin.IsFolderPathExists(filepath.Join(filepath.Dir(pluginPath), ".venv")) { + venv = filepath.Join(filepath.Dir(pluginPath), ".venv") + } + err = builtin.PrepareVenv(venv) + if err != nil { + log.Error().Err(err).Msg("prepare python3 venv failed") + return + } + pluginOptions = append(pluginOptions, funplugin.WithPython3(builtin.Python3Executable)) } // found plugin file - plugin, err = funplugin.Init(pluginPath, funplugin.WithLogOn(logOn)) + plugin, err = funplugin.Init(pluginPath, pluginOptions...) if err != nil { log.Error().Err(err).Msgf("init plugin failed: %s", pluginPath) return diff --git a/hrp/runner.go b/hrp/runner.go index 8290b1bf..46eb2af8 100644 --- a/hrp/runner.go +++ b/hrp/runner.go @@ -58,6 +58,7 @@ type HRPRunner struct { httpStatOn bool requestsLogOn bool pluginLogOn bool + venv string saveTests bool genHTMLReport bool httpClient *http.Client @@ -116,6 +117,13 @@ func (r *HRPRunner) SetPluginLogOn() *HRPRunner { return r } +// SetPython3Venv specifies python3 venv. +func (r *HRPRunner) SetPython3Venv(venv string) *HRPRunner { + log.Info().Str("venv", venv).Msg("[init] SetPython3Venv") + r.venv = venv + return r +} + // SetProxyUrl configures the proxy URL, which is usually used to capture HTTP packets for debugging. func (r *HRPRunner) SetProxyUrl(proxyUrl string) *HRPRunner { log.Info().Str("proxyUrl", proxyUrl).Msg("[init] SetProxyUrl") @@ -235,7 +243,7 @@ func (r *HRPRunner) newCaseRunner(testcase *TestCase) (*testCaseRunner, error) { } // init parser plugin - plugin, err := initPlugin(testcase.Config.Path, r.pluginLogOn) + plugin, err := initPlugin(testcase.Config.Path, r.venv, r.pluginLogOn) if err != nil { return nil, errors.Wrap(err, "init plugin failed") }