feat: add AIAction, do actions with LLM

internal/version/VERSION

@@ -1 +1 @@
-v5.0.0-beta-2503221208
+v5.0.0-beta-2503221320

step_ui.go

@@ -164,6 +164,18 @@ func (s *StepMobile) TapByUITypes(opts ...option.ActionOption) *StepMobile {
 	return s
 }
 
+// AIAction do actions with VLM
+func (s *StepMobile) AIAction(prompt string, opts ...option.ActionOption) *StepMobile {
+	action := uixt.MobileAction{
+		Method:  uixt.ACTION_AIAction,
+		Params:  prompt,
+		Options: option.NewActionOptions(opts...),
+	}
+
+	s.obj().Actions = append(s.obj().Actions, action)
+	return s
+}
+
 // DoubleTapXY double taps the point {X,Y}, X & Y is percentage of coordinates
 func (s *StepMobile) DoubleTapXY(x, y float64, opts ...option.ActionOption) *StepMobile {
 	s.obj().Actions = append(s.obj().Actions, uixt.MobileAction{
@@ -527,6 +539,21 @@ func (s *StepMobileUIValidation) AssertImageNotExists(expectedImagePath string,
 	return s
 }
 
+func (s *StepMobileUIValidation) AssertByAI(prompt string, msg ...string) *StepMobileUIValidation {
+	v := Validator{
+		Check:  uixt.SelectorAI,
+		Assert: uixt.AssertionAI,
+		Expect: prompt,
+	}
+	if len(msg) > 0 {
+		v.Message = msg[0]
+	} else {
+		v.Message = fmt.Sprintf("assert ai prompt [%s] failed", prompt)
+	}
+	s.Validators = append(s.Validators, v)
+	return s
+}
+
 func (s *StepMobileUIValidation) AssertAppInForeground(packageName string, msg ...string) *StepMobileUIValidation {
 	v := Validator{
 		Check:  uixt.SelectorForegroundApp,

uixt/driver_action.go

@@ -45,6 +45,7 @@ const (
 	ACTION_Input       ActionMethod = "input"
 	ACTION_Back        ActionMethod = "back"
 	ACTION_KeyCode     ActionMethod = "keycode"
+	ACTION_AIAction    ActionMethod = "ai_action" // action with ai
 
 	// custom actions
 	ACTION_SwipeToTapApp   ActionMethod = "swipe_to_tap_app"   // swipe left & right to find app and tap
@@ -64,12 +65,14 @@ const (
 	SelectorLabel         string = "ui_label"
 	SelectorOCR           string = "ui_ocr"
 	SelectorImage         string = "ui_image"
+	SelectorAI            string = "ui_ai" // ui query with ai
 	SelectorForegroundApp string = "ui_foreground_app"
 	// assertions
 	AssertionEqual     string = "equal"
 	AssertionNotEqual  string = "not_equal"
 	AssertionExists    string = "exists"
 	AssertionNotExists string = "not_exists"
+	AssertionAI        string = "ai_assert" // assert with ai
 )
 
 type MobileAction struct {
@@ -281,6 +284,11 @@ func (dExt *XTDriver) DoAction(action MobileAction) (err error) {
 		fn := action.Fn
 		fn()
 		return nil
+	case ACTION_AIAction:
+		if prompt, ok := action.Params.(string); ok {
+			return dExt.AIAction(prompt, action.GetOptions()...)
+		}
+		return fmt.Errorf("invalid %s params: %v", ACTION_AIAction, action.Params)
 	default:
 		log.Warn().Str("action", string(action.Method)).Msg("action not implemented")
 		return errors.Wrapf(code.InvalidCaseError,