Merge pull request #132 from sam80180/main

fix: support W3C touch actions (for WDA version >= 7.0.0)

pom.xml

@@ -101,6 +101,22 @@
             <version>4.6.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+		    <groupId>com.github.zafarkhaja</groupId>
+		    <artifactId>java-semver</artifactId>
+		    <version>0.10.2</version>
+		</dependency>
+		<dependency>
+		    <groupId>org.seleniumhq.selenium</groupId>
+		    <artifactId>selenium-api</artifactId>
+		    <version>4.22.0</version>
+		</dependency>
+		<dependency>
+		    <groupId>org.apache.httpcomponents</groupId>
+		    <artifactId>httpcore</artifactId>
+		    <version>4.4.16</version>
+		</dependency>
+
 
         <!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
         <dependency>

src/main/java/org/cloud/sonic/driver/common/models/WDAStatus.java

@@ -0,0 +1,24 @@
+package org.cloud.sonic.driver.common.models;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.ToString;
+
+@Getter
+@ToString
+@AllArgsConstructor
+public class WDAStatus {
+    private WDABuild build;
+
+    @Getter
+    @ToString
+    @AllArgsConstructor
+    public static class WDABuild {
+    	private String version;
+    } // end class
+} // end class
+
+/*
+References:
+https://github.com/SonicCloudOrg/sonic-driver-core/blob/faa0948e11e02be04db3ec9754fb66d613cf8bfa/src/main/java/org/cloud/sonic/driver/ios/service/impl/WdaClientImpl.java#L128
+*/

src/main/java/org/cloud/sonic/driver/common/tool/SemanticVersionTools.java

@@ -0,0 +1,25 @@
+package org.cloud.sonic.driver.common.tool;
+
+import com.github.zafarkhaja.semver.Version;
+
+public class SemanticVersionTools {
+	public static Version parseSemVer(final String s) {
+		return Version.parse(s);
+	} // end parseSemVer()
+
+	public static Version getVersionCore(final Version v) {
+		return Version.of(v.majorVersion(), v.minorVersion(), v.patchVersion());
+	} // end getVersionCore()
+
+	public static String pad4SemVer(final String s) {
+		if (Version.isValid(s)) { return s; } // end if
+		final String p[] = s.split("\\.");
+		final String vv[] = new String[] {"0", "0", "0"};
+		for (int i=0; i<3; i++) {
+			try {
+				vv[i] = String.format("%d", Integer.parseInt(p[i], 10));
+			} catch (Exception e) {} // end try
+		} // end for
+		return String.join(".", vv);
+	} // end pad4SemVer()
+} // end class

src/main/java/org/cloud/sonic/driver/common/tool/StringTool.java

@@ -0,0 +1,22 @@
+package org.cloud.sonic.driver.common.tool;
+
+import java.util.UUID;
+
+public class StringTool {
+	private static String[] CHARS = new String[] {
+		"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
+		"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
+		"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"
+	};
+
+	public static String generateShortUuid() { // http://www.java2s.com/example/java-utility-method/uuid-create/generateshortuuid-db743.html
+        StringBuilder shortBuilder = new StringBuilder();
+        String uuid = UUID.randomUUID().toString().replace("-", "");
+        for (int i=0; i<8; i++) {
+            String str = uuid.substring(i*4, i*4+4);
+            int x = Integer.parseInt(str, 16);
+            shortBuilder.append(CHARS[x%0x3E]);
+        } // end for
+        return shortBuilder.toString();
+    } // end generateShortUuid()
+} // end class

src/main/java/org/cloud/sonic/driver/ios/IOSDriver.java

@@ -16,10 +16,20 @@
  */
 package org.cloud.sonic.driver.ios;
 
+import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
+import com.github.zafarkhaja.semver.Version;
+
+import cn.hutool.http.HttpUtil;
+import cn.hutool.http.Method;
+
 import org.cloud.sonic.driver.common.enums.PasteboardType;
+import org.cloud.sonic.driver.common.models.BaseResp;
+import org.cloud.sonic.driver.common.models.ErrorMsg;
+import org.cloud.sonic.driver.common.models.WDAStatus;
 import org.cloud.sonic.driver.common.models.WindowSize;
 import org.cloud.sonic.driver.common.tool.RespHandler;
+import org.cloud.sonic.driver.common.tool.SemanticVersionTools;
 import org.cloud.sonic.driver.common.tool.SonicRespException;
 import org.cloud.sonic.driver.ios.enums.*;
 import org.cloud.sonic.driver.ios.models.TouchActions;
@@ -36,6 +46,7 @@
  */
 public class IOSDriver {
     private WdaClient wdaClient;
+    private Version wdaVersion;
 
     /**
      * Init ios driver.
@@ -82,6 +93,7 @@ public IOSDriver(String url, int timeOut, JSONObject cap) throws SonicRespExcept
         wdaClient.setRemoteUrl(url);
         wdaClient.setGlobalTimeOut(timeOut);
         wdaClient.newSession(cap);
+        this.wdaVersion = this.getWDAVersion();
     }
 
     /**
@@ -230,7 +242,11 @@ public void swipe(double fromX, double fromY, double toX, double toY, double dur
      * @throws SonicRespException
      */
     public void performTouchAction(TouchActions touchActions) throws SonicRespException {
-        wdaClient.performTouchAction(touchActions);
+    	if (this.wdaVersion.satisfies("<7.0.0")) { 
+    		this.wdaClient.performTouchAction(touchActions);
+    	} else {
+    		this.wdaClient.performW3CTouchAction(touchActions);
+    	} // end if
     }
 
     /**
@@ -699,4 +715,16 @@ public void rotate(Orientation orientation) throws SonicRespException {
     public Orientation getRotate() throws SonicRespException {
         return wdaClient.getRotate();
     }
+
+    public Version getWDAVersion() throws SonicRespException {
+		this.wdaClient.checkSessionId();
+		final String url = this.wdaClient.getRemoteUrl()+"/status";
+		final BaseResp<?> b = this.wdaClient.getRespHandler().getResp(HttpUtil.createRequest(Method.GET, url));
+		if (b==null) { throw new SonicRespException("null response body"); } // end if
+		final ErrorMsg err = b.getErr();
+		if (err!=null) { throw new SonicRespException(err.getMessage()); } // end if
+		final String result = b.getValue().toString();
+		final WDAStatus sessionStatus = JSON.parseObject(result, WDAStatus.class);
+		return SemanticVersionTools.parseSemVer(SemanticVersionTools.pad4SemVer(sessionStatus.getBuild().getVersion()));
+	} // end getWDAVersion()
 }

src/main/java/org/cloud/sonic/driver/ios/models/W3CActions.java

@@ -0,0 +1,49 @@
+package org.cloud.sonic.driver.ios.models;
+
+import java.time.Duration;
+import org.cloud.sonic.driver.common.tool.StringTool;
+import org.cloud.sonic.driver.ios.enums.ActionType;
+import org.cloud.sonic.driver.ios.models.TouchActions.TouchAction;
+import org.cloud.sonic.driver.ios.models.TouchActions.TouchAction.Options;
+import org.openqa.selenium.interactions.Pause;
+import org.openqa.selenium.interactions.PointerInput;
+import org.openqa.selenium.interactions.Sequence;
+import org.openqa.selenium.interactions.PointerInput.Kind;
+import org.openqa.selenium.interactions.PointerInput.MouseButton;
+import org.openqa.selenium.interactions.PointerInput.Origin;
+
+public class W3CActions {
+    public static Sequence convert(final TouchActions touchActions) {
+    	final PointerInput FINGER = new PointerInput(Kind.TOUCH, "finger-"+StringTool.generateShortUuid());
+    	final Sequence seq = new Sequence(FINGER, 0 /* https://github.com/appium/appium/issues/11273#issuecomment-416636734 */);
+    	for (TouchAction action: touchActions.getActions()) {
+    		final String strAction = action.getAction();
+    		final Options actionData = action.getOptions();
+    		if (strAction.equals(ActionType.MOVE.getType())) {
+    			final Integer optMs = actionData.getMs();
+    			final long ms = (optMs==null ? 0 : Math.max(optMs, 0));
+    			seq.addAction(FINGER.createPointerMove(Duration.ofMillis(ms), Origin.viewport(), actionData.getX(), actionData.getY()));
+    		} else if (strAction.equals(ActionType.PRESS.getType())) {
+    			final Integer optMs = actionData.getMs();
+    			final long ms = (optMs==null ? 0 : Math.max(optMs, 0));
+    			seq.addAction(FINGER.createPointerMove(Duration.ofMillis(ms), Origin.viewport(), actionData.getX(), actionData.getY()));
+    			seq.addAction(FINGER.createPointerDown(MouseButton.LEFT.asArg()));
+    		} else if (strAction.equals(ActionType.RELEASE.getType())) {
+    			seq.addAction(FINGER.createPointerUp(MouseButton.LEFT.asArg()));
+    		} else if (strAction.equals(ActionType.WAIT.getType())) {
+    			final Integer optMs = actionData.getMs();
+    			final long ms = (optMs==null ? 0 : Math.max(optMs, 0));
+    			if (ms<=0) { continue; } // end if
+    			seq.addAction(new Pause(FINGER, Duration.ofMillis(ms)));
+    		} // end if
+    	} // end for
+    	return seq;
+    } // end convert()
+} // end class
+
+/*
+References:
+https://stackoverflow.com/a/71038411/12857692
+https://github.com/appium/appium-espresso-driver/issues/244
+https://github.com/appium/appium/issues/11273
+*/

src/main/java/org/cloud/sonic/driver/ios/service/WdaClient.java

@@ -71,6 +71,8 @@ public interface WdaClient {
 
     //perform handler.
     void performTouchAction(TouchActions touchActions) throws SonicRespException;
+
+    void performW3CTouchAction(final TouchActions touchActions) throws SonicRespException;
 
     //button handler.
     void pressButton(String buttonName) throws SonicRespException;

src/main/java/org/cloud/sonic/driver/ios/service/impl/WdaClientImpl.java

@@ -20,17 +20,20 @@
 import cn.hutool.http.Method;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
+import org.apache.http.entity.ContentType;
 import org.cloud.sonic.driver.common.models.BaseResp;
+import org.cloud.sonic.driver.common.models.ErrorMsg;
 import org.cloud.sonic.driver.common.models.SessionInfo;
 import org.cloud.sonic.driver.common.models.WindowSize;
 import org.cloud.sonic.driver.common.tool.Logger;
 import org.cloud.sonic.driver.common.tool.RespHandler;
 import org.cloud.sonic.driver.common.tool.SonicRespException;
 import org.cloud.sonic.driver.ios.enums.Orientation;
 import org.cloud.sonic.driver.ios.models.TouchActions;
+import org.cloud.sonic.driver.ios.models.W3CActions;
 import org.cloud.sonic.driver.ios.service.IOSElement;
 import org.cloud.sonic.driver.ios.service.WdaClient;
-import org.jsoup.select.CombiningEvaluator;
+import org.openqa.selenium.interactions.Sequence;
 
 import java.nio.charset.StandardCharsets;
 import java.util.*;
@@ -214,6 +217,18 @@ public void performTouchAction(TouchActions touchActions) throws SonicRespExcept
             throw new SonicRespException(b.getErr().getMessage());
         }
     }
+
+    @SuppressWarnings("serial")
+	@Override
+    public void performW3CTouchAction(final TouchActions touchActions) throws SonicRespException {
+    	final Sequence seq = W3CActions.convert(touchActions);
+		final int timeoutUpdateSettings = 1500;
+		final String strAppiumSettingsURL = "%s/session/%s/actions".formatted(this.remoteUrl, this.sessionId);
+		final JSONObject payload = new JSONObject();
+		payload.put("actions", new LinkedList<Map<String, Object>>() {{ add(seq.toJson()); }});
+		final byte[] rawPayload = payload.toJSONString().getBytes(StandardCharsets.UTF_8);
+		this.sendAppiumRequest(Method.POST, strAppiumSettingsURL, ContentType.APPLICATION_JSON.toString(), rawPayload, timeoutUpdateSettings);
+    } // end performW3CTouchAction()
 
     @Override
     public void pressButton(String buttonName) throws SonicRespException {
@@ -607,4 +622,17 @@ public Orientation getRotate() throws SonicRespException {
             throw new SonicRespException(b.getErr().getMessage());
         }
     }
+
+    private void sendAppiumRequest(final Method method, final String url, final String contentType, final byte[] bodyBytes, final int timeout) throws SonicRespException {
+		this.checkSessionId();
+		final BaseResp<?> b = this.getRespHandler().getResp(HttpUtil.createRequest(method, url).body(bodyBytes).contentType(contentType), timeout);
+		if (b!=null) {
+			final ErrorMsg err = b.getErr();
+			if (err!=null) {
+				throw new SonicRespException(err.getMessage());
+	        } // end if
+		} else {
+			throw new SonicRespException("null response body");
+		} // end if
+	} // end sendAppiumRequest()
 }