GithubBackup/httprunner
1
0
Fork 0
mirror of https://github.com/httprunner/httprunner.git synced 2026-09-05 07:26:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: support dispatch profile to worker

Browse Source
This commit is contained in:
xucong053
2022-07-25 21:15:03 +08:00
committed by 徐聪
parent c5f482dbfa
commit 4044c15974
11 changed files with 366 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hrp/internal/boomer/boomer.go
+110 -56
View File
@@ -1,16 +1,13 @@
package boomer
import (
"github.com/httprunner/httprunner/v4/hrp/internal/json"
"math"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/httprunner/httprunner/v4/hrp/internal/builtin"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
@@ -49,6 +46,58 @@ type Boomer struct {
disableCompression bool
}
type Profile struct {
SpawnCount int64 `json:"spawn-count,omitempty" yaml:"spawn-count,omitempty" mapstructure:"spawn-count,omitempty"`
SpawnRate float64 `json:"spawn-rate,omitempty" yaml:"spawn-rate,omitempty" mapstructure:"spawn-rate,omitempty"`
MaxRPS int64 `json:"max-rps,omitempty" yaml:"max-rps,omitempty" mapstructure:"max-rps,omitempty"`
LoopCount int64 `json:"loop-count,omitempty" yaml:"loop-count,omitempty" mapstructure:"loop-count,omitempty"`
RequestIncreaseRate string `json:"request-increase-rate,omitempty" yaml:"request-increase-rate,omitempty" mapstructure:"request-increase-rate,omitempty"`
MemoryProfile string `json:"memory-profile,omitempty" yaml:"memory-profile,omitempty" mapstructure:"memory-profile,omitempty"`
MemoryProfileDuration time.Duration `json:"memory-profile-duration,omitempty" yaml:"memory-profile-duration,omitempty" mapstructure:"memory-profile-duration,omitempty"`
CPUProfile string `json:"cpu-profile,omitempty" yaml:"cpu-profile,omitempty" mapstructure:"cpu-profile,omitempty"`
CPUProfileDuration time.Duration `json:"cpu-profile-duration,omitempty" yaml:"cpu-profile-duration,omitempty" mapstructure:"cpu-profile-duration,omitempty"`
PrometheusPushgatewayURL string `json:"prometheus-gateway,omitempty" yaml:"prometheus-gateway,omitempty" mapstructure:"prometheus-gateway,omitempty"`
DisableConsoleOutput bool `json:"disable-console-output,omitempty" yaml:"disable-console-output,omitempty" mapstructure:"disable-console-output,omitempty"`
DisableCompression bool `json:"disable-compression,omitempty" yaml:"disable-compression,omitempty" mapstructure:"disable-compression,omitempty"`
DisableKeepalive bool `json:"disable-keepalive,omitempty" yaml:"disable-keepalive,omitempty" mapstructure:"disable-keepalive,omitempty"`
}
func (b *Boomer) GetProfile() *Profile {
switch b.mode {
case DistributedMasterMode:
return b.masterRunner.profile
case DistributedWorkerMode:
return b.workerRunner.profile
default:
return b.localRunner.profile
}
}
func (b *Boomer) SetProfile(profile *Profile) {
switch b.mode {
case DistributedMasterMode:
b.masterRunner.profile = profile
case DistributedWorkerMode:
b.workerRunner.profile = profile
default:
b.localRunner.profile = profile
}
}
func (p *Profile) dispatch(workers int64) *Profile {
workerProfile := *p
if p.SpawnCount > 0 {
workerProfile.SpawnCount = p.SpawnCount / workers
}
if p.SpawnRate > 0 {
workerProfile.SpawnRate = p.SpawnRate / float64(workers)
}
if p.MaxRPS > 0 {
workerProfile.MaxRPS = p.MaxRPS / workers
}
return &workerProfile
}
// SetMode only accepts boomer.DistributedMasterMode、boomer.DistributedWorkerMode and boomer.StandaloneMode.
func (b *Boomer) SetMode(mode Mode) {
switch mode {
@@ -79,7 +128,7 @@ func (b *Boomer) GetMode() string {
}
// NewStandaloneBoomer returns a new Boomer, which can run without master.
func NewStandaloneBoomer(spawnCount int, spawnRate float64) *Boomer {
func NewStandaloneBoomer(spawnCount int64, spawnRate float64) *Boomer {
return &Boomer{
mode: StandaloneMode,
localRunner: newLocalRunner(spawnCount, spawnRate),
@@ -125,10 +174,56 @@ func (b *Boomer) GetTestCaseBytesChan() chan []byte {
switch b.mode {
case DistributedMasterMode:
return b.masterRunner.testCaseBytes
default:
return nil
}
}
func ProfileToBytes(profile *Profile) []byte {
profileBytes, err := json.Marshal(profile)
if err != nil {
log.Error().Err(err).Msg("failed to marshal testcases")
return nil
}
return profileBytes
}
func BytesToProfile(profileBytes []byte) *Profile {
var profile *Profile
err := json.Unmarshal(profileBytes, &profile)
if err != nil {
log.Error().Err(err).Msg("failed to unmarshal testcases")
}
return profile
}
// GetProfileBytesChan gets profile bytes chan
func (b *Boomer) GetProfileBytesChan() chan []byte {
switch b.mode {
case DistributedMasterMode:
return b.masterRunner.profileBytes
default:
return nil
}
}
// GetTasksChan gets profile bytes chan
func (b *Boomer) GetTasksChan() chan *profileMessage {
switch b.mode {
case DistributedWorkerMode:
return b.workerRunner.testCaseBytes
return b.workerRunner.tasksChan
default:
return nil
}
}
func (b *Boomer) GetRebalanceChan() chan bool {
switch b.mode {
case DistributedWorkerMode:
return b.workerRunner.rebalance
default:
return nil
}
return nil
}
func (b *Boomer) SetTestCasesPath(paths []string) {
@@ -390,66 +485,25 @@ func (b *Boomer) RecordFailure(requestType, name string, responseTime int64, exc
}
// Start starts to run
func (b *Boomer) Start(Args map[string]interface{}) error {
func (b *Boomer) Start(Args *Profile) error {
if b.masterRunner.isStarted() {
return errors.New("already started")
}
spawnCount, ok := Args["spawn_count"]
if ok {
v, err := strconv.Atoi(spawnCount.(string))
if err != nil {
log.Error().Err(err).Msg("spawn_count sets error")
return err
}
b.SetSpawnCount(int64(v))
} else {
return errors.New("spawn count error")
}
spawnRate, ok := Args["spawn_rate"]
if ok {
v, err := builtin.Interface2Float64(spawnRate)
if err != nil {
log.Error().Err(err).Msg("spawn_count sets error")
return err
}
b.SetSpawnRate(v)
} else {
b.SetSpawnRate(float64(b.GetSpawnCount()))
}
path, ok := Args["path"].(string)
if ok {
paths := strings.Split(path, ",")
b.SetTestCasesPath(paths)
} else {
return errors.New("testcase path error")
}
b.SetSpawnCount(Args.SpawnCount)
b.SetSpawnRate(Args.SpawnRate)
b.SetProfile(Args)
err := b.masterRunner.start()
return err
}
// ReBalance starts to rebalance load test
func (b *Boomer) ReBalance(Args map[string]interface{}) error {
func (b *Boomer) ReBalance(Args *Profile) error {
if !b.masterRunner.isStarted() {
return errors.New("no start")
}
spawnCount, ok := Args["spawn_count"]
if ok {
v, err := strconv.Atoi(spawnCount.(string))
if err != nil {
log.Error().Err(err).Msg("spawn_count sets error")
return err
}
b.SetSpawnCount(int64(v))
}
spawnRate, ok := Args["spawn_rate"]
if ok {
v, err := builtin.Interface2Float64(spawnRate)
if err != nil {
log.Error().Err(err).Msg("spawn_count sets error")
return err
}
b.SetSpawnRate(v)
}
b.SetSpawnCount(Args.SpawnCount)
b.SetSpawnRate(Args.SpawnRate)
b.SetProfile(Args)
err := b.masterRunner.rebalance()
if err != nil {
log.Error().Err(err).Msg("failed to rebalance")
hrp/internal/boomer/client_grpc.go
+11 -13
View File
@@ -80,6 +80,9 @@ func (c *grpcClient) connect() (err error) {
return err
}
go c.recv()
go c.send()
biStream, err := messager.NewMessageClient(c.config.conn).BidirectionalStreamingMessage(c.config.ctx)
if err != nil {
log.Error().Err(err).Msg("call bidirectional streaming message err")
@@ -87,19 +90,11 @@ func (c *grpcClient) connect() (err error) {
}
c.config.setBiStreamClient(biStream)
log.Info().Msg(fmt.Sprintf("Boomer is connected to master(%s) press Ctrl+c to quit.\n", addr))
go c.recv()
go c.send()
return nil
}
func (c *grpcClient) reConnect() (err error) {
addr := fmt.Sprintf("%v:%v", c.masterHost, c.masterPort)
c.config.conn, err = grpc.Dial(addr, grpc.WithInsecure())
if err != nil {
return
}
biStream, err := messager.NewMessageClient(c.config.conn).BidirectionalStreamingMessage(c.config.ctx)
if err != nil {
return
@@ -111,7 +106,7 @@ func (c *grpcClient) reConnect() (err error) {
//// tell master, I'm ready
//log.Info().Msg("send client ready signal")
//c.sendChannel() <- newClientReadyMessageToMaster(c.identity)
log.Info().Msg(fmt.Sprintf("Boomer is reConnected to master(%s) press Ctrl+c to quit.\n", addr))
log.Info().Msg(fmt.Sprintf("Boomer is reConnected to master press Ctrl+c to quit.\n"))
return
}
@@ -136,6 +131,7 @@ func (c *grpcClient) recv() {
return
default:
if c.config.getBiStreamClient() == nil {
time.Sleep(1 * time.Second)
continue
}
msg, err := c.config.getBiStreamClient().Recv()
@@ -158,10 +154,11 @@ func (c *grpcClient) recv() {
}
c.fromMaster <- &genericMessage{
Type: msg.Type,
Data: msg.Data,
NodeID: msg.NodeID,
Tasks: msg.Tasks,
Type: msg.Type,
Profile: msg.Profile,
Data: msg.Data,
NodeID: msg.NodeID,
Tasks: msg.Tasks,
}
log.Info().
@@ -204,6 +201,7 @@ func (c *grpcClient) sendMessage(msg *genericMessage) {
Interface("data", msg.Data).
Msg("send data to server")
if c.config.getBiStreamClient() == nil {
atomic.AddInt32(&c.failCount, 1)
return
}
err := c.config.getBiStreamClient().Send(&messager.StreamRequest{Type: msg.Type, Data: msg.Data, NodeID: msg.NodeID})
hrp/internal/boomer/message.go
+14 -10
View File
@@ -10,14 +10,17 @@ const (
typeException = "exception"
)
type message interface {
type genericMessage struct {
Type string `json:"type,omitempty"`
Profile []byte `json:"profile,omitempty"`
Data map[string]int64 `json:"data,omitempty"`
NodeID string `json:"node_id,omitempty"`
Tasks []byte `json:"tasks,omitempty"`
}
type genericMessage struct {
Type string `json:"type,omitempty"`
Data map[string]int64 `json:"data,omitempty"`
NodeID string `json:"node_id,omitempty"`
Tasks []byte `json:"tasks,omitempty"`
type profileMessage struct {
Profile []byte `json:"profile,omitempty"`
Tasks []byte `json:"tasks,omitempty"`
}
func newGenericMessage(t string, data map[string]int64, nodeID string) (msg *genericMessage) {
@@ -35,11 +38,12 @@ func newQuitMessage(nodeID string) (msg *genericMessage) {
}
}
func newSpawnMessageToWorker(t string, data map[string]int64, tasks []byte) (msg *genericMessage) {
func newMessageToWorker(t string, profile []byte, data map[string]int64, tasks []byte) (msg *genericMessage) {
return &genericMessage{
Type: t,
Data: data,
Tasks: tasks,
Type: t,
Profile: profile,
Data: data,
Tasks: tasks,
}
}
hrp/internal/boomer/runner.go
+42 -47
View File
@@ -270,6 +270,9 @@ func (r *runner) outputOnEvent(data map[string]interface{}) {
}
func (r *runner) outputOnStop() {
defer func() {
r.outputs = make([]Output, 0)
}()
size := len(r.outputs)
if size == 0 {
return
@@ -332,6 +335,7 @@ func (r *runner) reset() {
}
func (r *runner) spawnWorkers(spawnCount int64, spawnRate float64, quit chan bool, spawnCompleteFunc func()) {
r.updateState(StateSpawning)
log.Info().
Int64("spawnCount", spawnCount).
Float64("spawnRate", spawnRate).
@@ -339,7 +343,6 @@ func (r *runner) spawnWorkers(spawnCount int64, spawnRate float64, quit chan boo
r.controller.setSpawn(spawnCount, spawnRate)
r.updateState(StateSpawning)
for {
select {
case <-quit:
@@ -510,14 +513,16 @@ func (r *runner) isStarted() bool {
type localRunner struct {
runner
profile *Profile
}
func newLocalRunner(spawnCount int, spawnRate float64) *localRunner {
func newLocalRunner(spawnCount int64, spawnRate float64) *localRunner {
return &localRunner{
runner: runner{
state: StateInit,
stats: newRequestStats(),
spawnCount: int64(spawnCount),
spawnCount: spawnCount,
spawnRate: spawnRate,
controller: &Controller{},
outputs: make([]Output, 0),
@@ -535,14 +540,13 @@ func (r *localRunner) start() {
if r.rateLimitEnabled {
r.rateLimiter.Start()
}
r.spawnWorkers(r.getSpawnCount(), r.getSpawnRate(), r.stopChan, nil)
// output setup
r.outputOnStart()
go r.spawnWorkers(r.getSpawnCount(), r.getSpawnRate(), r.stopChan, nil)
// start stats report
go r.runner.statsStart()
go r.statsStart()
// stop
<-r.stopChan
@@ -582,10 +586,9 @@ type workerRunner struct {
masterPort int
client *grpcClient
// this channel will start worker for spawning.
spawnStartChan chan bool
// get testcase from master
testCaseBytes chan []byte
profile *Profile
tasksChan chan *profileMessage
ignoreQuit bool
}
@@ -594,15 +597,15 @@ func newWorkerRunner(masterHost string, masterPort int) (r *workerRunner) {
r = &workerRunner{
runner: runner{
stats: newRequestStats(),
outputs: make([]Output, 0),
controller: &Controller{},
closeChan: make(chan bool),
once: &sync.Once{},
},
masterHost: masterHost,
masterPort: masterPort,
nodeID: getNodeID(),
spawnStartChan: make(chan bool),
testCaseBytes: make(chan []byte, 10),
masterHost: masterHost,
masterPort: masterPort,
nodeID: getNodeID(),
tasksChan: make(chan *profileMessage, 10),
}
return r
}
@@ -615,30 +618,26 @@ func (r *workerRunner) spawnComplete() {
func (r *workerRunner) onSpawnMessage(msg *genericMessage) {
r.client.sendChannel() <- newGenericMessage("spawning", nil, r.nodeID)
spawnCount, ok := msg.Data["spawn_count"]
if ok {
r.setSpawnCount(spawnCount)
if msg.Profile == nil {
log.Error().Msg("miss profile")
}
spawnRate, ok := msg.Data["spawn_rate"]
if ok {
r.setSpawnRate(float64(spawnRate))
if msg.Tasks == nil {
log.Error().Msg("miss tasks")
}
if msg.Tasks != nil {
r.testCaseBytes <- msg.Tasks
r.tasksChan <- &profileMessage{
Profile: msg.Profile,
Tasks: msg.Tasks,
}
log.Info().Msg("on spawn message successful")
}
func (r *workerRunner) onRebalanceMessage(msg *genericMessage) {
spawnCount, ok := msg.Data["spawn_count"]
if ok {
r.setSpawnCount(spawnCount)
if msg.Profile == nil {
log.Error().Msg("miss profile")
}
spawnRate, ok := msg.Data["spawn_rate"]
if ok {
r.setSpawnRate(float64(spawnRate))
r.tasksChan <- &profileMessage{
Profile: msg.Profile,
}
r.rebalance <- true
log.Info().Msg("on rebalance message successful")
}
@@ -705,7 +704,6 @@ func (r *workerRunner) run() {
err := r.client.connect()
if err != nil {
log.Printf("Failed to connect to master(%s:%d) with error %v\n", r.masterHost, r.masterPort, err)
return
}
// listen to master
@@ -758,7 +756,7 @@ func (r *workerRunner) start() {
r.once.Do(r.outputOnStart)
r.spawnWorkers(r.getSpawnCount(), r.getSpawnRate(), r.stopChan, r.spawnComplete)
go r.spawnWorkers(r.getSpawnCount(), r.getSpawnRate(), r.stopChan, r.spawnComplete)
// start stats report
go r.statsStart()
@@ -783,7 +781,7 @@ func (r *workerRunner) close() {
return
}
// waiting report finished
time.Sleep(3 * time.Second)
time.Sleep(1 * time.Second)
close(r.closeChan)
var ticker = time.NewTicker(1 * time.Second)
if r.client != nil {
@@ -811,8 +809,12 @@ type masterRunner struct {
expectWorkers int
expectWorkersMaxWait int
profile *Profile
parseTestCasesChan chan bool
testCaseBytes chan []byte
// set profile to worker
profileBytes chan []byte
}
func newMasterRunner(masterBindHost string, masterBindPort int) *masterRunner {
@@ -990,20 +992,17 @@ func (r *masterRunner) start() error {
if numWorkers == 0 {
return errors.New("current workers: 0")
}
workerSpawnRate := r.getSpawnRate() / float64(numWorkers)
workerSpawnCount := r.getSpawnCount() / int64(numWorkers)
log.Info().Msg("send spawn data to worker")
r.updateState(StateSpawning)
// waitting to fetch testcase
// fetching testcase
testcase, err := r.fetchTestCase()
if err != nil {
return err
}
r.server.sendChannel() <- newSpawnMessageToWorker("spawn", map[string]int64{
"spawn_count": workerSpawnCount,
"spawn_rate": int64(workerSpawnRate),
}, testcase)
profile := r.profile.dispatch(int64(numWorkers))
r.server.sendChannel() <- newMessageToWorker("spawn", ProfileToBytes(profile), nil, testcase)
println("send spawn data to worker successful")
log.Info().Msg("send spawn data to worker successful")
return nil
@@ -1014,13 +1013,9 @@ func (r *masterRunner) rebalance() error {
if numWorkers == 0 {
return errors.New("current workers: 0")
}
workerSpawnRate := r.getSpawnRate() / float64(numWorkers)
workerSpawnCount := r.getSpawnCount() / int64(numWorkers)
profile := r.profile.dispatch(int64(numWorkers))
r.server.sendChannel() <- newSpawnMessageToWorker("rebalance", map[string]int64{
"spawn_count": workerSpawnCount,
"spawn_rate": int64(workerSpawnRate),
}, nil)
r.server.sendChannel() <- newMessageToWorker("rebalance", ProfileToBytes(profile), nil, nil)
println("send rebalance data to worker successful")
return nil
}
hrp/internal/boomer/server_grpc.go
+5 -4
View File
@@ -312,10 +312,11 @@ func (s *grpcServer) sendMessage(msg *genericMessage) {
}
err := workerInfo.messenger.Send(
&messager.StreamResponse{
Type: msg.Type,
Data: msg.Data,
NodeID: workerInfo.ID,
Tasks: msg.Tasks},
Type: msg.Type,
Profile: msg.Profile,
Data: msg.Data,
NodeID: workerInfo.ID,
Tasks: msg.Tasks},
)
switch err {
case nil: