refactor: build plugin

docs/CHANGELOG.md

@@ -5,6 +5,8 @@
 **go version**
 
 - feat #1342: support specify custom python3 venv
+- feat: support python3 venv priority, specified > projectDir/.venv > $HOME/.hrp/venv
+- refactor: build plugin mechanism
 - fix: pip upgrade httprunner when installing hrp
 
 ## v4.1.2 (2022-06-09)

examples/demo-with-go-plugin/proj.json

@@ -1,5 +1,5 @@
 {
     "project_name": "demo-with-go-plugin",
-    "create_time": "2022-06-09T23:39:14.781583+08:00",
+    "create_time": "2022-06-13T23:28:24.920066+08:00",
     "hrp_version": "v4.1.2"
 }

examples/demo-with-go-plugin/testcases/demo.json

@@ -173,4 +173,4 @@
             ]
         }
     ]
-}
+}

examples/demo-with-py-plugin/.debugtalk_gen.py

@@ -1,67 +1,15 @@
 # NOTE: Generated By hrp v4.1.2, DO NOT EDIT!
 
-import logging
+import sys
-import time
+import os
-import funppy
 
-from typing import List
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-
+from debugtalk import *
-def get_user_agent():
-    return "hrp/funppy"
-
-
-def sleep(n_secs):
-    time.sleep(n_secs)
-
-
-def sum(*args):
-    result = 0
-    for arg in args:
-        result += arg
-    return result
-
-
-def sum_ints(*args: List[int]) -> int:
-    result = 0
-    for arg in args:
-        result += arg
-    return result
-
-
-def sum_two_int(a: int, b: int) -> int:
-    return a + b
-
-
-def sum_two_string(a: str, b: str) -> str:
-    return a + b
-
-
-def sum_strings(*args: List[str]) -> str:
-    result = ""
-    for arg in args:
-        result += arg
-    return result
-
-
-def concatenate(*args: List[str]) -> str:
-    result = ""
-    for arg in args:
-        result += str(arg)
-    return result
-
-
-def setup_hook_example(name):
-    logging.warning("setup_hook_example")
-    return f"setup_hook_example: {name}"
-
-
-def teardown_hook_example(name):
-    logging.warning("teardown_hook_example")
-    return f"teardown_hook_example: {name}"
 
 
 if __name__ == "__main__":
+    import funppy
     funppy.register("get_user_agent", get_user_agent)
     funppy.register("sleep", sleep)
     funppy.register("sum", sum)

examples/demo-with-py-plugin/proj.json

@@ -1,5 +1,5 @@
 {
     "project_name": "demo-with-py-plugin",
-    "create_time": "2022-06-09T23:39:14.922843+08:00",
+    "create_time": "2022-06-13T23:28:25.254595+08:00",
     "hrp_version": "v4.1.2"
 }

examples/demo-with-py-plugin/testcases/demo.json

@@ -173,4 +173,4 @@
             ]
         }
     ]
-}
+}

hrp/build.go

@@ -4,13 +4,12 @@ import (
 	"bufio"
 	_ "embed"
 	"fmt"
-	"io"
+	"html/template"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strings"
-	"text/template"
 
 	"github.com/httprunner/funplugin/shared"
 	"github.com/pkg/errors"
@@ -20,156 +19,57 @@ import (
 	"github.com/httprunner/httprunner/v4/hrp/internal/version"
 )
 
-const (
-	fungo                   = `"github.com/httprunner/funplugin/fungo"`
-	regexPythonFunctionName = `def ([a-zA-Z_]\w*)\(.*\)`
-	regexGoImports          = `import \(([\s\S]*?)\)`
-	regexGoImport           = `import (\"[\s\S]*\")`
-	regexGoFunctionName     = `func ([A-Z][a-zA-Z_]\w*)\(.*\)`
-	regexGoFunctionContent  = `func [\s\S]*?\n}`
-)
-
 //go:embed internal/scaffold/templates/plugin/debugtalkPythonTemplate
 var pyTemplate string
 
 //go:embed internal/scaffold/templates/plugin/debugtalkGoTemplate
 var goTemplate string
 
-type TemplateContent struct {
+// regex for finding all function names
+var (
+	regexPyFunctionName = regexp.MustCompile(`def ([a-zA-Z_]\w*)\(.*\)`)
+	regexGoFunctionName = regexp.MustCompile(`func ([A-Z][a-zA-Z_]\w*)\(.*\)`)
+)
+
+type pluginTemplateContent struct {
 	Version       string   // hrp version
-	Regexps       *Regexps // match import/function
+	FunctionNames []string // function names
-	Imports       []string // python/go import
-	FromImports   []string // python from...import...
-	Functions     []string // python/go function
-	FunctionNames []string // function name set by user
-	Packages      []string // python packages
 }
 
-type Regexps struct {
+func findAllFunctionNames(t *regexp.Regexp, path string) (functionNames []string, err error) {
-	Import          *regexp.Regexp
+	log.Info().Str("path", path).Msg("find all function names from plugin file")
-	Imports         *regexp.Regexp
-	FunctionName    *regexp.Regexp
-	FunctionContent *regexp.Regexp // including function define and body
-}
-
-func (t *TemplateContent) parseGoContent(path string) error {
-	log.Info().Str("path", path).Msg("start to parse debugtalk.go")
 
 	content, err := os.ReadFile(path)
 	if err != nil {
 		log.Error().Err(err).Msg("failed to read file")
-		return err
+		return nil, errors.Wrap(err, "read file failed")
-	}
-	originalContent := string(content)
-
-	// parse imports
-	importSlice := t.Regexps.Imports.FindAllStringSubmatch(originalContent, -1)
-	if len(importSlice) != 0 {
-		imports := strings.Replace(importSlice[0][1], "\t", "", -1)
-		for _, elem := range strings.Split(imports, "\n") {
-			t.Imports = append(t.Imports, strings.TrimSpace(elem))
-		}
-	}
-	// parse import
-	importSlice = t.Regexps.Import.FindAllStringSubmatch(originalContent, -1)
-	if len(importSlice) != 0 {
-		for _, elem := range importSlice {
-			t.Imports = append(t.Imports, strings.TrimSpace(elem[1]))
-		}
-	}
-	// import fungo package
-	if !builtin.Contains(t.Imports, fungo) {
-		t.Imports = append(t.Imports, fungo)
 	}
 
-	// parse function name
+	// find all function names
-	functionNameSlice := t.Regexps.FunctionName.FindAllStringSubmatch(originalContent, -1)
+	functionNameSlice := t.FindAllStringSubmatch(string(content), -1)
 	for _, elem := range functionNameSlice {
 		name := strings.Trim(elem[1], " ")
-		if name == "main" {
+		functionNames = append(functionNames, name)
-			continue
-		}
-		t.FunctionNames = append(t.FunctionNames, name)
 	}
 
-	// parse function content
+	return
-	functionContentSlice := t.Regexps.FunctionContent.FindAllStringSubmatch(originalContent, -1)
-	for _, f := range functionContentSlice {
-		if strings.Contains(f[0], "func main") {
-			continue
-		}
-		t.Functions = append(t.Functions, strings.Trim(f[0], "\n"))
-	}
-	return nil
 }
 
-func (t *TemplateContent) parsePyContent(path string) error {
+func generate(data *pluginTemplateContent, tmpl, output string) error {
-	file, err := os.Open(path)
-	if err != nil {
-		log.Error().Err(err).Str("path", path).Msg("failed to open file")
-		return err
-	}
-	defer file.Close()
-
-	r := bufio.NewReader(file)
-
-	// record content excluding import and main
-	content := ""
-
-	// parse python content line by line
-	for {
-		l, _, err := r.ReadLine()
-		if err == io.EOF {
-			break
-		}
-		line := string(l)
-
-		if strings.HasPrefix(line, "import") {
-			t.Imports = append(t.Imports, strings.Trim(line, " "))
-			// e.g. import module as md
-			// import package.module
-			t.Packages = append(t.Packages, strings.Split(strings.Split(line, " ")[1], ".")[0])
-		} else if strings.HasPrefix(line, "from") {
-			t.FromImports = append(t.FromImports, strings.Trim(line, " "))
-			// e.g. from package.module import function
-			// from module import function
-			// from package import module
-			t.Packages = append(t.Packages, strings.Split(strings.Split(line, " ")[1], ".")[0])
-		} else {
-			// no parse content at under of `if __name__ == "__main__"`
-			if strings.HasPrefix(line, "if __name__") {
-				break
-			}
-			if strings.HasPrefix(line, "def") {
-				functionNameSlice := t.Regexps.FunctionName.FindAllStringSubmatch(line, -1)
-				if len(functionNameSlice) == 0 {
-					continue
-				}
-				t.FunctionNames = append(t.FunctionNames, functionNameSlice[0][1])
-			}
-			content += line + "\n"
-		}
-	}
-	// function content
-	t.Functions = append(t.Functions, strings.Trim(content, "\n"))
-
-	return nil
-}
-
-func (t *TemplateContent) genDebugTalk(output string, templ string) error {
 	file, err := os.Create(output)
 	if err != nil {
-		log.Error().Err(err).Msg("open file failed")
+		log.Error().Err(err).Msg("open output file failed")
 		return err
 	}
 	defer file.Close()
+
 	writer := bufio.NewWriter(file)
-	tmpl := template.Must(template.New("debugtalk").Parse(templ))
+	err = template.Must(template.New("debugtalk").Parse(tmpl)).Execute(writer, data)
-	err = tmpl.Execute(writer, t)
 	if err != nil {
-		log.Error().Err(err).Msg("execute applies a parsed template to the specified data object failed")
+		log.Error().Err(err).Msg("execute template parsing failed")
 		return err
 	}
+
 	err = writer.Flush()
 	if err == nil {
 		log.Info().Str("output", output).Msg("generate debugtalk success")
@@ -181,35 +81,35 @@ func (t *TemplateContent) genDebugTalk(output string, templ string) error {
 
 // buildGo builds debugtalk.go to debugtalk.bin
 func buildGo(path string, output string) error {
-	templateContent := &TemplateContent{
+	functionNames, err := findAllFunctionNames(regexGoFunctionName, path)
-		Version: version.VERSION,
+	if err != nil {
-		Regexps: &Regexps{
+		return errors.Wrap(err, "find all function names failed")
-			Import:          regexp.MustCompile(regexGoImport),
+	}
-			Imports:         regexp.MustCompile(regexGoImports),
+	// filter main and init function
-			FunctionName:    regexp.MustCompile(regexGoFunctionName),
+	var filteredFunctionNames []string
-			FunctionContent: regexp.MustCompile(regexGoFunctionContent),
+	for _, name := range functionNames {
-		},
+		if name == "main" || name == "init" {
+			continue
+		}
+		filteredFunctionNames = append(filteredFunctionNames, name)
+	}
+
+	templateContent := &pluginTemplateContent{
+		Version:       version.VERSION,
+		FunctionNames: filteredFunctionNames,
 	}
 
 	pluginDir := filepath.Dir(path)
+	err = generate(templateContent, goTemplate, filepath.Join(pluginDir, PluginGoSourceGenFile))
+	if err != nil {
+		return errors.Wrap(err, "generate hashicorp plugin failed")
+	}
 
 	// check go sdk in tempDir
 	if err := builtin.ExecCommandInDir(exec.Command("go", "version"), pluginDir); err != nil {
 		return errors.Wrap(err, "go sdk not installed")
 	}
 
-	// parse debugtalk.go in pluginDir
-	err := templateContent.parseGoContent(path)
-	if err != nil {
-		return err
-	}
-
-	// generate debugtalk.go in pluginDir
-	err = templateContent.genDebugTalk(filepath.Join(pluginDir, PluginGoSourceGenFile), goTemplate)
-	if err != nil {
-		return err
-	}
-
 	if !builtin.IsFilePathExists(filepath.Join(pluginDir, "go.mod")) {
 		// create go mod
 		if err := builtin.ExecCommandInDir(exec.Command("go", "mod", "init", "main"), pluginDir); err != nil {
@@ -220,64 +120,66 @@ func buildGo(path string, output string) error {
 		// funplugin version should be locked
 		funplugin := fmt.Sprintf("github.com/httprunner/funplugin@%s", shared.Version)
 		if err := builtin.ExecCommandInDir(exec.Command("go", "get", funplugin), pluginDir); err != nil {
-			return err
+			return errors.Wrap(err, "go get funplugin failed")
 		}
 	}
 
 	// add missing and remove unused modules
 	if err := builtin.ExecCommandInDir(exec.Command("go", "mod", "tidy"), pluginDir); err != nil {
-		return err
+		return errors.Wrap(err, "go mod tidy failed")
 	}
 
+	// specify output file path
 	if output == "" {
 		dir, _ := os.Getwd()
 		output = filepath.Join(dir, PluginHashicorpGoBuiltFile)
 	} else if builtin.IsFolderPathExists(output) {
 		output = filepath.Join(output, PluginHashicorpGoBuiltFile)
 	}
-	outputPath, err := filepath.Abs(output)
+	outputPath, _ := filepath.Abs(output)
-	if err != nil {
-		return err
-	}
 
-	// build plugin debugtalk.bin
+	// build go plugin to debugtalk.bin
 	cmd := exec.Command("go", "build", "-o", outputPath, PluginGoSourceGenFile, filepath.Base(path))
 	if err := builtin.ExecCommandInDir(cmd, pluginDir); err != nil {
-		return err
+		return errors.Wrap(err, "go build plugin failed")
 	}
-	log.Info().Str("output", outputPath).Str("plugin", path).Msg("build plugin successfully")
+	log.Info().Str("output", outputPath).Str("plugin", path).Msg("build go plugin successfully")
 	return nil
 }
 
 // buildPy completes funppy information in debugtalk.py
 func buildPy(path string, output string) error {
-	templateContent := &TemplateContent{
-		Version: version.VERSION,
-		Regexps: &Regexps{
-			FunctionName: regexp.MustCompile(regexPythonFunctionName),
-		},
-	}
-
-	err := templateContent.parsePyContent(path)
-	if err != nil {
-		return err
-	}
-
 	// check the syntax of debugtalk.py
-	err = builtin.CheckPythonScriptSyntax(path)
+	err := builtin.ExecCommand("python3", "-m", "py_compile", path)
+	if err != nil {
+		return errors.Wrap(err, "python plugin syntax invalid")
+	}
+
+	functionNames, err := findAllFunctionNames(regexPyFunctionName, path)
 	if err != nil {
 		return err
 	}
+	templateContent := &pluginTemplateContent{
+		Version:       version.VERSION,
+		FunctionNames: functionNames,
+	}
 
-	// generate .debugtalk_gen.py
+	// specify output file path
 	if output == "" {
 		dir, _ := os.Getwd()
 		output = filepath.Join(dir, PluginPySourceGenFile)
 	} else if builtin.IsFolderPathExists(output) {
 		output = filepath.Join(output, PluginPySourceGenFile)
 	}
-	err = templateContent.genDebugTalk(output, pyTemplate)
+
-	return err
+	// generate .debugtalk_gen.py
+	err = generate(templateContent, pyTemplate, output)
+	if err != nil {
+		return err
+	}
+
+	log.Info().Str("output", output).Str("plugin", path).Msg("build python plugin successfully")
+	return nil
 }
 
 func BuildPlugin(path string, output string) (err error) {
@@ -291,7 +193,7 @@ func BuildPlugin(path string, output string) (err error) {
 		return errors.New("type error, expected .py or .go")
 	}
 	if err != nil {
-		log.Error().Err(err).Str("arg", path).Msg("build plugin failed")
+		log.Error().Err(err).Str("path", path).Msg("build plugin failed")
 		os.Exit(1)
 	}
 	return nil

hrp/internal/builtin/utils.go

@@ -185,10 +185,6 @@ func InstallPythonPackage(python3 string, pkg string) (err error) {
 	return nil
 }
 
-func CheckPythonScriptSyntax(path string) error {
-	return ExecCommand("python3", "-m", "py_compile", path)
-}
-
 func ExecCommandInDir(cmd *exec.Cmd, dir string) error {
 	log.Info().Str("cmd", cmd.String()).Str("dir", dir).Msg("exec command")
 	cmd.Dir = dir

hrp/internal/scaffold/templates/plugin/.debugtalk_gen.py

@@ -1,67 +1,15 @@
 # NOTE: Generated By hrp v4.1.2, DO NOT EDIT!
 
-import logging
+import sys
-import time
+import os
-import funppy
 
-from typing import List
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-
+from debugtalk import *
-def get_user_agent():
-    return "hrp/funppy"
-
-
-def sleep(n_secs):
-    time.sleep(n_secs)
-
-
-def sum(*args):
-    result = 0
-    for arg in args:
-        result += arg
-    return result
-
-
-def sum_ints(*args: List[int]) -> int:
-    result = 0
-    for arg in args:
-        result += arg
-    return result
-
-
-def sum_two_int(a: int, b: int) -> int:
-    return a + b
-
-
-def sum_two_string(a: str, b: str) -> str:
-    return a + b
-
-
-def sum_strings(*args: List[str]) -> str:
-    result = ""
-    for arg in args:
-        result += arg
-    return result
-
-
-def concatenate(*args: List[str]) -> str:
-    result = ""
-    for arg in args:
-        result += str(arg)
-    return result
-
-
-def setup_hook_example(name):
-    logging.warning("setup_hook_example")
-    return f"setup_hook_example: {name}"
-
-
-def teardown_hook_example(name):
-    logging.warning("teardown_hook_example")
-    return f"teardown_hook_example: {name}"
 
 
 if __name__ == "__main__":
+    import funppy
     funppy.register("get_user_agent", get_user_agent)
     funppy.register("sleep", sleep)
     funppy.register("sum", sum)

hrp/internal/scaffold/templates/plugin/debugtalkPythonTemplate

@@ -1,15 +1,12 @@
 # NOTE: Generated By hrp {{ .Version }}, DO NOT EDIT!
 
-{{ range $import := .Imports }}
+import sys
-{{- $import}}
+import os
-{{ end }}
+
-{{ range $fromImport := .FromImports }}
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-{{- $fromImport}}
+
-{{ end }}
+from debugtalk import *
 
-{{ range $function := .Functions }}
-{{- $function }}
-{{ end }}
 
 if __name__ == "__main__":
     import funppy

hrp/internal/scaffold/templates/plugin/debugtalk_gen.go

@@ -1,4 +1,4 @@
-// NOTE: Generated By hrp v4.1.3, DO NOT EDIT!
+// NOTE: Generated By hrp v4.1.2, DO NOT EDIT!
 package main
 
 import (