Merge pull request #1585 from httprunner/fix-android-screencap

refactor android screencap

- feat: add adb `screencap` sub command
- feat: add `IsAppInForeground` to check if the given package is in foreground
- feat: check if app is in foreground when step failed
- fix: take screenshot after each step
- fix: screencap compatibility for shell v1 and v2
This commit is contained in:
debugtalk
2023-04-15 00:12:51 +08:00
committed by GitHub
19 changed files with 278 additions and 110 deletions

View File

@@ -1,14 +1,19 @@
# Release History
## v4.3.3 (2023-04-11)
## v4.3.3 (2023-04-14)
**go version**
- feat: add `sleep_random` to sleep random seconds, with weight for multiple time ranges
- feat: input text with adb
- feat: add adb `screencap` sub command
- feat: add `IsAppInForeground` to check if the given package is in foreground
- feat: check if app is in foreground when step failed
- fix: adb driver for TapFloat
- fix: stop logcat only when enabled
- fix: do not fail case when kill logcat error
- fix: take screenshot after each step
- fix: screencap compatibility for shell v1 and v2
## v4.3.2 (2022-12-26)

View File

@@ -5,10 +5,8 @@ import (
"fmt"
"os"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/httprunner/httprunner/v4/hrp/pkg/gadb"
"github.com/httprunner/httprunner/v4/hrp/pkg/uixt"
)
@@ -21,22 +19,10 @@ var listAndroidDevicesCmd = &cobra.Command{
Use: "devices",
Short: "List all Android devices",
RunE: func(cmd *cobra.Command, args []string) error {
devices, err := uixt.DeviceList()
deviceList, err := uixt.GetAndroidDevices(serial)
if err != nil {
return errors.Wrap(err, "list android devices failed")
}
var deviceList []*gadb.Device
// filter by serial
for _, d := range devices {
if serial != "" && serial != d.Serial() {
continue
}
deviceList = append(deviceList, d)
}
if serial != "" && len(deviceList) == 0 {
fmt.Printf("no android device found for serial: %s\n", serial)
os.Exit(1)
fmt.Println(err)
os.Exit(0)
}
for _, d := range deviceList {

View File

@@ -1,6 +1,13 @@
package adb
import "github.com/spf13/cobra"
import (
"fmt"
"github.com/spf13/cobra"
"github.com/httprunner/httprunner/v4/hrp/pkg/gadb"
"github.com/httprunner/httprunner/v4/hrp/pkg/uixt"
)
var androidRootCmd = &cobra.Command{
Use: "adb",
@@ -8,6 +15,17 @@ var androidRootCmd = &cobra.Command{
PersistentPreRun: func(cmd *cobra.Command, args []string) {},
}
func getDevice(serial string) (*gadb.Device, error) {
devices, err := uixt.GetAndroidDevices(serial)
if err != nil {
return nil, err
}
if len(devices) > 1 {
return nil, fmt.Errorf("found multiple attached devices, please specify android serial")
}
return devices[0], nil
}
func Init(rootCmd *cobra.Command) {
rootCmd.AddCommand(androidRootCmd)
}

37
hrp/cmd/adb/screencap.go Normal file
View File

@@ -0,0 +1,37 @@
package adb
import (
"fmt"
"io/ioutil"
"time"
"github.com/spf13/cobra"
)
var screencapAndroidDevicesCmd = &cobra.Command{
Use: "screencap",
Short: "Start android screen capture",
RunE: func(cmd *cobra.Command, args []string) error {
device, err := getDevice(serial)
if err != nil {
return err
}
res, err := device.ScreenCap()
if err != nil {
return err
}
filepath := fmt.Sprintf("screencap_%d.png", time.Now().Unix())
if err = ioutil.WriteFile(filepath, res, 0o644); err != nil {
return err
}
fmt.Println("screencap saved to", filepath)
return nil
},
}
func init() {
screencapAndroidDevicesCmd.Flags().StringVarP(&serial, "serial", "s", "", "filter by device's serial")
androidRootCmd.AddCommand(screencapAndroidDevicesCmd)
}

View File

@@ -70,18 +70,10 @@ var listDevicesCmd = &cobra.Command{
Short: "List all iOS devices",
PersistentPreRun: func(cmd *cobra.Command, args []string) {},
RunE: func(cmd *cobra.Command, args []string) error {
devices, err := uixt.IOSDevices(udid)
devices, err := uixt.GetIOSDevices(udid)
if err != nil {
return err
}
if len(devices) == 0 {
if udid != "" {
fmt.Printf("no ios device found for udid: %s\n", udid)
os.Exit(1)
} else {
fmt.Println("no ios device found")
os.Exit(0)
}
fmt.Println(err)
os.Exit(0)
}
for _, d := range devices {

View File

@@ -2,7 +2,6 @@ package ios
import (
"fmt"
"os"
"github.com/spf13/cobra"
@@ -16,16 +15,12 @@ var iosRootCmd = &cobra.Command{
}
func getDevice(udid string) (gidevice.Device, error) {
devices, err := uixt.IOSDevices(udid)
devices, err := uixt.GetIOSDevices(udid)
if err != nil {
return nil, err
}
if len(devices) == 0 {
fmt.Println("no ios device found")
os.Exit(1)
}
if len(devices) > 1 {
return nil, fmt.Errorf("multiple devices found, please specify udid")
return nil, fmt.Errorf("found multiple attached devices, please specify ios udid")
}
return devices[0], nil
}

View File

@@ -67,8 +67,9 @@ var (
// UI automation related: [70, 80)
var (
MobileUIDriverError = errors.New("mobile UI driver error") // 70
MobileUIValidationError = errors.New("mobile UI validation error") // 75
MobileUIDriverError = errors.New("mobile UI driver error") // 70
MobileUIValidationError = errors.New("mobile UI validation error") // 75
MobileUIAppNotInForegroundError = errors.New("mobile UI app not in foreground error") // 76
)
// OCR related: [80, 90)

View File

@@ -1 +1 @@
v4.3.3
v4.3.3.2304142356

View File

@@ -572,5 +572,20 @@ func (d *Device) Uninstall(packageName string, keepData ...bool) (string, error)
}
func (d *Device) ScreenCap() ([]byte, error) {
return d.RunShellCommandV2WithBytes("screencap", "-p")
if d.HasFeature(FeatShellV2) {
return d.RunShellCommandV2WithBytes("screencap", "-p")
}
// for shell v1, screenshot buffer maybe truncated
// thus we firstly save it to local file and then pull it
tempPath := fmt.Sprintf("/data/local/tmp/screenshot_%d.png",
time.Now().Unix())
_, err := d.RunShellCommandWithBytes("screencap", "-p", tempPath)
if err != nil {
return nil, err
}
buffer := bytes.NewBuffer(nil)
err = d.Pull(tempPath, buffer)
return buffer.Bytes(), err
}

View File

@@ -153,11 +153,11 @@ func (ad *adbDriver) PressKeyCode(keyCode KeyCode, metaState KeyMeta) (err error
return
}
func (ad *adbDriver) AppLaunch(bundleId string) (err error) {
func (ad *adbDriver) AppLaunch(packageName string) (err error) {
// 不指定 Activity 名称启动(启动主 Activity
// adb shell monkey -p <packagename> -c android.intent.category.LAUNCHER 1
sOutput, err := ad.adbClient.RunShellCommand(
"monkey", "-p", bundleId, "-c", "android.intent.category.LAUNCHER", "1",
"monkey", "-p", packageName, "-c", "android.intent.category.LAUNCHER", "1",
)
if err != nil {
return err
@@ -165,14 +165,22 @@ func (ad *adbDriver) AppLaunch(bundleId string) (err error) {
if strings.Contains(sOutput, "monkey aborted") {
return fmt.Errorf("app launch: %s", strings.TrimSpace(sOutput))
}
ad.lastLaunchedPackageName = packageName
return nil
}
func (ad *adbDriver) AppTerminate(bundleId string) (successful bool, err error) {
func (ad *adbDriver) AppTerminate(packageName string) (successful bool, err error) {
// 强制停止应用,停止 <packagename> 相关的进程
// adb shell am force-stop <packagename>
_, err = ad.adbClient.RunShellCommand("am", "force-stop", bundleId)
return err == nil, err
_, err = ad.adbClient.RunShellCommand("am", "force-stop", packageName)
if err != nil {
return false, err
}
if ad.lastLaunchedPackageName == packageName {
ad.lastLaunchedPackageName = "" // reset last launched package name
}
return true, nil
}
func (ad *adbDriver) Tap(x, y int, options ...DataOption) error {
@@ -321,7 +329,7 @@ func (ad *adbDriver) StartCaptureLog(identifier ...string) (err error) {
log.Info().Msg("start adb log recording")
// clear logcat
if _, err = ad.adbClient.RunShellCommand("logcat", "--clear"); err != nil {
if _, err = ad.adbClient.RunShellCommand("logcat", "-c"); err != nil {
return err
}
@@ -347,3 +355,34 @@ func (ad *adbDriver) StopCaptureLog() (result interface{}, err error) {
content := ad.logcat.logBuffer.String()
return ConvertPoints(content), nil
}
func (ad *adbDriver) GetLastLaunchedApp() (packageName string) {
return ad.lastLaunchedPackageName
}
func (ad *adbDriver) IsAppInForeground(packageName string) (bool, error) {
if packageName == "" {
return false, errors.New("package name is not given")
}
// adb shell dumpsys activity activities | grep mResumedActivity
output, err := ad.adbClient.RunShellCommand("dumpsys", "activity", "activities")
if err != nil {
return false, err
}
lines := strings.Split(string(output), "\n")
isInForeground := false
for _, line := range lines {
trimmedLine := strings.TrimSpace(line)
if strings.HasPrefix(trimmedLine, "mResumedActivity:") {
if strings.Contains(trimmedLine, packageName) {
isInForeground = true
}
break
}
}
return isInForeground, nil
}

View File

@@ -81,15 +81,6 @@ func GetAndroidDeviceOptions(dev *AndroidDevice) (deviceOptions []AndroidDeviceO
// uiautomator2 server must be started before
// adb shell am instrument -w io.appium.uiautomator2.server.test/androidx.test.runner.AndroidJUnitRunner
func NewAndroidDevice(options ...AndroidDeviceOption) (device *AndroidDevice, err error) {
deviceList, err := DeviceList()
if err != nil {
return nil, errors.Wrap(code.AndroidDeviceConnectionError,
fmt.Sprintf("get attached devices failed: %v", err))
} else if len(deviceList) == 0 {
return nil, errors.Wrap(code.AndroidDeviceConnectionError,
"not attached device found")
}
device = &AndroidDevice{
UIA2IP: UIA2ServerHost,
UIA2Port: UIA2ServerPort,
@@ -98,30 +89,52 @@ func NewAndroidDevice(options ...AndroidDeviceOption) (device *AndroidDevice, er
option(device)
}
serialNumber := device.SerialNumber
for _, dev := range deviceList {
// find device by serial number if specified
if serialNumber != "" && dev.Serial() != serialNumber {
continue
}
device.SerialNumber = dev.Serial()
device.d = dev
device.logcat = NewAdbLogcat(device.SerialNumber)
return device, nil
deviceList, err := GetAndroidDevices(device.SerialNumber)
if err != nil {
return nil, errors.Wrap(code.AndroidDeviceConnectionError, err.Error())
}
return nil, errors.Wrap(code.AndroidDeviceConnectionError,
fmt.Sprintf("device %s not found", device.SerialNumber))
dev := deviceList[0]
device.SerialNumber = dev.Serial()
device.d = dev
device.logcat = NewAdbLogcat(device.SerialNumber)
log.Info().Str("serial", device.SerialNumber).Msg("select android device")
return device, nil
}
func DeviceList() (devices []*gadb.Device, err error) {
func GetAndroidDevices(serial ...string) (devices []*gadb.Device, err error) {
var adbClient gadb.Client
if adbClient, err = gadb.NewClientWith(AdbServerHost, AdbServerPort); err != nil {
return nil, errors.Wrap(code.AndroidDeviceConnectionError, err.Error())
}
return adbClient.DeviceList()
if devices, err = adbClient.DeviceList(); err != nil {
return nil, errors.Wrap(code.AndroidDeviceConnectionError,
fmt.Sprintf("list android devices failed: %v", err))
}
var deviceList []*gadb.Device
// filter by serial
for _, d := range devices {
for _, s := range serial {
if s != "" && s != d.Serial() {
continue
}
deviceList = append(deviceList, d)
}
}
if len(deviceList) == 0 {
var err error
if serial == nil || (len(serial) == 1 && serial[0] == "") {
err = fmt.Errorf("no android device found")
} else {
err = fmt.Errorf("no android device found for serial %v", serial)
}
return nil, err
}
return deviceList, nil
}
type AndroidDevice struct {
@@ -315,7 +328,7 @@ func (l *AdbLogcat) CatchLogcat() (err error) {
}
// clear logcat
if err = myexec.RunCommand("adb", "-s", l.serial, "logcat", "--clear"); err != nil {
if err = myexec.RunCommand("adb", "-s", l.serial, "shell", "logcat", "-c"); err != nil {
return
}

View File

@@ -324,7 +324,7 @@ func Test_getFreePort(t *testing.T) {
}
func TestDeviceList(t *testing.T) {
devices, err := DeviceList()
devices, err := GetAndroidDevices()
if err != nil {
t.Fatal(err)
}
@@ -353,6 +353,34 @@ func TestDriver_AppLaunch(t *testing.T) {
t.Log(ioutil.WriteFile("s1.png", raw.Bytes(), 0o600))
}
func TestDriver_IsAppInForeground(t *testing.T) {
device, _ := NewAndroidDevice()
driver, err := device.NewDriver(nil)
if err != nil {
t.Fatal(err)
}
err = driver.Driver.AppLaunch("com.android.settings")
if err != nil {
t.Fatal(err)
}
yes, err := driver.Driver.IsAppInForeground(driver.Driver.GetLastLaunchedApp())
if err != nil || !yes {
t.Fatal(err)
}
_, err = driver.Driver.AppTerminate("com.android.settings")
if err != nil {
t.Fatal(err)
}
yes, err = driver.Driver.IsAppInForeground("com.android.settings")
if err != nil || yes {
t.Fatal(err)
}
}
func TestDriver_KeepAlive(t *testing.T) {
device, _ := NewAndroidDevice()
driver, err := device.NewDriver(nil)

View File

@@ -20,6 +20,8 @@ type Driver struct {
urlPrefix *url.URL
sessionId string
client *http.Client
// cache the last launched package name
lastLaunchedPackageName string
}
func (wd *Driver) concatURL(u *url.URL, elem ...string) string {

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"image"
"image/gif"
"image/jpeg"
"image/png"
"math/rand"
@@ -284,6 +285,8 @@ func saveScreenShot(raw *bytes.Buffer, fileName string) (string, error) {
err = png.Encode(file, img)
case "jpeg":
err = jpeg.Encode(file, img, nil)
case "gif":
err = gif.Encode(file, img, nil)
default:
return "", fmt.Errorf("unsupported image format: %s", format)
}

View File

@@ -627,10 +627,14 @@ type WebDriver interface {
// AppLaunch Launch an application with given bundle identifier in scope of current session.
// !This method is only available since Xcode9 SDK
AppLaunch(bundleId string) error
// AppTerminate Terminate an application with the given bundle id.
AppLaunch(packageName string) error
// AppTerminate Terminate an application with the given pacakge name.
// Either `true` if the app has been successfully terminated or `false` if it was not running
AppTerminate(bundleId string) (bool, error)
AppTerminate(packageName string) (bool, error)
// GetLastLaunchedApp returns the package name of the last launched app
GetLastLaunchedApp() string
// IsAppInForeground returns true if the given package is in foreground
IsAppInForeground(packageName string) (bool, error)
// StartCamera Starts a new camera for recording
StartCamera() error

View File

@@ -141,7 +141,7 @@ func WithIOSPcapOptions(options ...gidevice.PcapOption) IOSDeviceOption {
}
}
func IOSDevices(udid ...string) (devices []gidevice.Device, err error) {
func GetIOSDevices(udid ...string) (devices []gidevice.Device, err error) {
var usbmux gidevice.Usbmux
if usbmux, err = gidevice.NewUsbmux(); err != nil {
return nil, errors.Wrap(code.IOSDeviceConnectionError,
@@ -168,6 +168,15 @@ func IOSDevices(udid ...string) (devices []gidevice.Device, err error) {
}
}
if len(deviceList) == 0 {
var err error
if udid == nil || (len(udid) == 1 && udid[0] == "") {
err = fmt.Errorf("no ios device found")
} else {
err = fmt.Errorf("no ios device found for udid %v", udid)
}
return nil, err
}
return deviceList, nil
}
@@ -223,31 +232,27 @@ func NewIOSDevice(options ...IOSDeviceOption) (device *IOSDevice, err error) {
option(device)
}
deviceList, err := IOSDevices(device.UDID)
deviceList, err := GetIOSDevices(device.UDID)
if err != nil {
return nil, err
return nil, errors.Wrap(code.IOSDeviceConnectionError, err.Error())
}
for _, dev := range deviceList {
udid := dev.Properties().SerialNumber
device.UDID = udid
device.d = dev
dev := deviceList[0]
udid := dev.Properties().SerialNumber
device.UDID = udid
device.d = dev
// run xctest if XCTestBundleID is set
if device.XCTestBundleID != "" {
_, err = device.RunXCTest(device.XCTestBundleID)
if err != nil {
log.Error().Err(err).Str("udid", udid).Msg("failed to init XCTest")
continue
}
// run xctest if XCTestBundleID is set
if device.XCTestBundleID != "" {
_, err = device.RunXCTest(device.XCTestBundleID)
if err != nil {
log.Error().Err(err).Str("udid", udid).Msg("failed to init XCTest")
return
}
log.Info().Str("udid", device.UDID).Msg("select device")
return device, nil
}
return nil, errors.Wrap(code.IOSDeviceConnectionError,
fmt.Sprintf("device %s not found", device.UDID))
log.Info().Str("udid", device.UDID).Msg("select ios device")
return device, nil
}
type IOSDevice struct {

View File

@@ -308,6 +308,9 @@ func (wd *wdaDriver) AppLaunch(bundleId string) (err error) {
data := make(map[string]interface{})
data["bundleId"] = bundleId
_, err = wd.httpPOST(data, "/session", wd.sessionId, "/wda/apps/launch")
if err == nil {
wd.lastLaunchedPackageName = bundleId
}
return
}
@@ -328,6 +331,9 @@ func (wd *wdaDriver) AppTerminate(bundleId string) (successful bool, err error)
if successful, err = rawResp.valueConvertToBool(); err != nil {
return false, err
}
if wd.lastLaunchedPackageName == bundleId {
wd.lastLaunchedPackageName = "" // reset last launched package name
}
return
}
@@ -348,6 +354,14 @@ func (wd *wdaDriver) AppDeactivate(second float64) (err error) {
return
}
func (wd *wdaDriver) GetLastLaunchedApp() (packageName string) {
return wd.lastLaunchedPackageName
}
func (wd *wdaDriver) IsAppInForeground(packageName string) (bool, error) {
return false, errors.New("not implemented")
}
func (wd *wdaDriver) Tap(x, y int, options ...DataOption) error {
return wd.TapFloat(float64(x), float64(y), options...)
}

View File

@@ -93,16 +93,19 @@ func (s *veDEMOCRService) getOCRResult(imageBuf *bytes.Buffer) ([]OCRResult, err
// retry 3 times
for i := 1; i <= 3; i++ {
resp, err = client.Do(req)
if err == nil {
break
}
var logID string
if resp != nil {
logID = getLogID(resp.Header)
}
if err == nil && resp.StatusCode == http.StatusOK {
log.Debug().
Str("X-TT-LOGID", logID).
Int("imageBufSize", size).
Msg("request OCR service success")
break
}
log.Error().Err(err).
Str("logID", logID).
Str("X-TT-LOGID", logID).
Int("imageBufSize", size).
Msgf("request OCR service failed, retry %d", i)
time.Sleep(1 * time.Second)

View File

@@ -561,6 +561,24 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err
attachments := make(map[string]interface{})
if err != nil {
attachments["error"] = err.Error()
// check if app is in foreground
packageName := uiDriver.Driver.GetLastLaunchedApp()
yes, err2 := uiDriver.Driver.IsAppInForeground(packageName)
if packageName != "" && (!yes || err2 != nil) {
log.Error().Err(err2).Str("packageName", packageName).Msg("app is not in foreground")
err = errors.Wrap(code.MobileUIAppNotInForegroundError, err.Error())
}
}
// take screenshot after each step
screenshotPath, err := uiDriver.ScreenShot(
fmt.Sprintf("step_%d", time.Now().Unix()))
if err != nil {
log.Error().Err(err).Str("step", step.Name).Msg("take screenshot failed")
} else {
log.Info().Str("path", screenshotPath).Msg("take screenshot on step finished")
screenshots = append(screenshots, screenshotPath)
}
// save attachments
@@ -599,16 +617,6 @@ func runStepMobileUI(s *SessionRunner, step *TStep) (stepResult *StepResult, err
}
}
// take snapshot
screenshotPath, err := uiDriver.ScreenShot(
fmt.Sprintf("validate_%d", time.Now().Unix()))
if err != nil {
log.Warn().Err(err).Str("step", step.Name).Msg("take screenshot failed")
} else {
log.Info().Str("path", screenshotPath).Msg("take screenshot before validation")
screenshots = append(screenshots, screenshotPath)
}
// validate
validateResults, err := validateUI(uiDriver, step.Validators)
if err != nil {