From 16f55e7516e2218c0d9d6a609ce6e35ab5c9b12e Mon Sep 17 00:00:00 2001 From: ZhouYixun <291028775@qq.com> Date: Wed, 19 Jan 2022 20:38:17 +0800 Subject: [PATCH] removecv --- pom.xml | 280 ++++++++++++++- .../agent/automation/AndroidStepHandler.java | 119 ++++--- .../agent/automation/IOSStepHandler.java | 118 ++++--- .../org/cloud/sonic/agent/cv/AKAZEFinder.java | 318 ++++++++++++++++++ .../org/cloud/sonic/agent/cv/SIFTFinder.java | 103 ++++++ .../sonic/agent/cv/SimilarityChecker.java | 70 ++++ .../org/cloud/sonic/agent/cv/TemMatcher.java | 61 ++++ src/main/resources/application.yml | 2 +- 8 files changed, 939 insertions(+), 132 deletions(-) create mode 100644 src/main/java/org/cloud/sonic/agent/cv/AKAZEFinder.java create mode 100644 src/main/java/org/cloud/sonic/agent/cv/SIFTFinder.java create mode 100644 src/main/java/org/cloud/sonic/agent/cv/SimilarityChecker.java create mode 100644 src/main/java/org/cloud/sonic/agent/cv/TemMatcher.java diff --git a/pom.xml b/pom.xml index 21f338b9..f1dd853f 100644 --- a/pom.xml +++ b/pom.xml @@ -102,28 +102,292 @@ org.bytedeco - ffmpeg-platform - 4.2.2-1.5.3 + openblas + 0.3.9-1.5.3 org.bytedeco - javacpp-platform - 1.5.3 + opencv + 4.3.0-1.5.3 + - dev + windows-x86 + + windows-x86 + + + + org.bytedeco + ffmpeg + 4.2.2-1.5.3 + ${profileActive} + + + org.bytedeco + javacpp + 1.5.3 + ${profileActive} + + + org.bytedeco + opencv + 4.3.0-1.5.3 + ${profileActive} + + + org.bytedeco + openblas + 0.3.9-1.5.3 + ${profileActive} + + + org.bytedeco + tesseract + 4.1.1-1.5.3 + ${profileActive} + + + org.bytedeco + leptonica + 1.79.0-1.5.3 + ${profileActive} + + + + + + windows-x86_64 + + windows-x86_64 + + + + org.bytedeco + ffmpeg + 4.2.2-1.5.3 + ${profileActive} + + + org.bytedeco + javacpp + 1.5.3 + ${profileActive} + + + org.bytedeco + opencv + 4.3.0-1.5.3 + ${profileActive} + + + org.bytedeco + openblas + 0.3.9-1.5.3 + ${profileActive} + + + + + + linux-arm64 + + linux-arm64 + + + + org.bytedeco + ffmpeg + 4.2.2-1.5.3 + ${profileActive} + + + org.bytedeco + javacpp + 1.5.3 + ${profileActive} + + + org.bytedeco + opencv + 4.3.0-1.5.3 + ${profileActive} + + + org.bytedeco + openblas + 0.3.9-1.5.3 + ${profileActive} + + + + + + linux-armhf - dev + linux-armhf + + + org.bytedeco + ffmpeg + 4.2.2-1.5.3 + ${profileActive} + + + org.bytedeco + javacpp + 1.5.3 + ${profileActive} + + + org.bytedeco + opencv + 4.3.0-1.5.3 + ${profileActive} + + + org.bytedeco + openblas + 0.3.9-1.5.3 + ${profileActive} + + + + + linux-ppc64le + + linux-ppc64le + + + + org.bytedeco + ffmpeg + 4.2.2-1.5.3 + ${profileActive} + + + org.bytedeco + javacpp + 1.5.3 + ${profileActive} + + + org.bytedeco + opencv + 4.3.0-1.5.3 + ${profileActive} + + + org.bytedeco + openblas + 0.3.9-1.5.3 + ${profileActive} + + + + + + linux-x86 + + linux-x86 + + + + org.bytedeco + ffmpeg + 4.2.2-1.5.3 + ${profileActive} + + + org.bytedeco + javacpp + 1.5.3 + ${profileActive} + + + org.bytedeco + opencv + 4.3.0-1.5.3 + ${profileActive} + + + org.bytedeco + openblas + 0.3.9-1.5.3 + ${profileActive} + + + + + + linux-x86_64 + + linux-x86_64 + + + + org.bytedeco + ffmpeg + 4.2.2-1.5.3 + ${profileActive} + + + org.bytedeco + javacpp + 1.5.3 + ${profileActive} + + + org.bytedeco + opencv + 4.3.0-1.5.3 + ${profileActive} + + + org.bytedeco + openblas + 0.3.9-1.5.3 + ${profileActive} + + + + - prod + macosx-x86_64 - prod + macosx-x86_64 + + + org.bytedeco + ffmpeg + 4.2.2-1.5.3 + ${profileActive} + + + org.bytedeco + javacpp + 1.5.3 + ${profileActive} + + + org.bytedeco + opencv + 4.3.0-1.5.3 + ${profileActive} + + + org.bytedeco + openblas + 0.3.9-1.5.3 + ${profileActive} + + diff --git a/src/main/java/org/cloud/sonic/agent/automation/AndroidStepHandler.java b/src/main/java/org/cloud/sonic/agent/automation/AndroidStepHandler.java index 0f9393bb..e9584f9e 100644 --- a/src/main/java/org/cloud/sonic/agent/automation/AndroidStepHandler.java +++ b/src/main/java/org/cloud/sonic/agent/automation/AndroidStepHandler.java @@ -5,6 +5,10 @@ import com.android.ddmlib.IDevice; import org.cloud.sonic.agent.bridge.android.AndroidDeviceBridgeTool; import org.cloud.sonic.agent.bridge.android.AndroidDeviceThreadPool; +import org.cloud.sonic.agent.cv.AKAZEFinder; +import org.cloud.sonic.agent.cv.SIFTFinder; +import org.cloud.sonic.agent.cv.SimilarityChecker; +import org.cloud.sonic.agent.cv.TemMatcher; import org.cloud.sonic.agent.interfaces.ErrorType; import org.cloud.sonic.agent.interfaces.ResultDetailStatus; import org.cloud.sonic.agent.interfaces.StepType; @@ -54,10 +58,6 @@ */ public class AndroidStepHandler { public LogTool log = new LogTool(); - private RestTemplate restTemplate = SpringTool.getBean(RestTemplate.class); - private Environment environment = SpringTool.getBean(Environment.class); - private String baseUrl = "http://" + environment.getProperty("sonic.server.host") - + ":" + environment.getProperty("sonic.server.folder-port") + "/api/folder"; private AndroidDriver androidDriver; private JSONObject globalParams = new JSONObject(); //包版本 @@ -884,40 +884,56 @@ public void clickByImg(HandleDes handleDes, String des, String pathValue) throws return; } } - File localCap = getScreenToLocal(); - FindResult findResult; - FileSystemResource resource1 = new FileSystemResource(file); - FileSystemResource resource2 = new FileSystemResource(localCap); - MultiValueMap param = new LinkedMultiValueMap<>(); - param.add("file1", resource1); - param.add("file2", resource2); - param.add("type", "finder"); + FindResult findResult = null; try { - ResponseEntity responseEntity = - restTemplate.postForEntity(baseUrl + "/upload/cv", param, JSONObject.class); - if (responseEntity.getBody().getInteger("code") == 2000) { - findResult = responseEntity.getBody().getJSONObject("data").toJavaObject(FindResult.class); + SIFTFinder siftFinder = new SIFTFinder(); + findResult = siftFinder.getSIFTFindResult(file, getScreenToLocal()); + } catch (Exception e) { + log.sendStepLog(StepType.WARN, "SIFT图像算法出错，切换算法中...", + ""); + } + if (findResult != null) { + log.sendStepLog(StepType.INFO, "图片定位到坐标：(" + findResult.getX() + "," + findResult.getY() + ") 耗时：" + findResult.getTime() + " ms", + findResult.getUrl()); + } else { + log.sendStepLog(StepType.INFO, "SIFT算法无法定位图片，切换AKAZE算法中...", + ""); + try { + AKAZEFinder akazeFinder = new AKAZEFinder(); + findResult = akazeFinder.getAKAZEFindResult(file, getScreenToLocal()); + } catch (Exception e) { + log.sendStepLog(StepType.WARN, "AKAZE图像算法出错，切换模版匹配算法中...", + ""); + } + if (findResult != null) { + log.sendStepLog(StepType.INFO, "图片定位到坐标：(" + findResult.getX() + "," + findResult.getY() + ") 耗时：" + findResult.getTime() + " ms", + findResult.getUrl()); + } else { + log.sendStepLog(StepType.INFO, "AKAZE算法无法定位图片，切换模版匹配算法中...", + ""); + try { + TemMatcher temMatcher = new TemMatcher(); + findResult = temMatcher.getTemMatchResult(file, getScreenToLocal()); + } catch (Exception e) { + log.sendStepLog(StepType.WARN, "模版匹配算法出错", + ""); + } if (findResult != null) { - try { - log.sendStepLog(StepType.INFO, "图片定位到坐标：(" + findResult.getX() + "," + findResult.getY() + ") 耗时：" + findResult.getTime() + " ms", - findResult.getUrl()); - TouchAction ta = new TouchAction(androidDriver); - ta.tap(PointOption.point(findResult.getX(), findResult.getY())).perform(); - } catch (Exception e) { - log.sendStepLog(StepType.ERROR, "点击" + des + "失败！", ""); - handleDes.setE(e); - } + log.sendStepLog(StepType.INFO, "图片定位到坐标：(" + findResult.getX() + "," + findResult.getY() + ") 耗时：" + findResult.getTime() + " ms", + findResult.getUrl()); + } else { + handleDes.setE(new Exception("图片定位失败！")); } - } else if (responseEntity.getBody().getInteger("code") == 4003) { - handleDes.setE(new Exception("图像匹配失败！")); - } else { - handleDes.setE(new Exception("点击失败！cv服务出错！")); } - } catch (Exception e) { - handleDes.setE(new Exception("点击失败！cv服务访问出错！")); - } finally { - file.delete(); - localCap.delete(); + } + if (findResult != null) { + try { + TouchAction ta = new TouchAction(androidDriver); + ta.tap(PointOption.point(findResult.getX(), findResult.getY())).perform(); + } catch (Exception e) { + log.sendStepLog(StepType.ERROR, "点击" + des + "失败！", ""); + handleDes.setE(e); + } } } @@ -986,34 +1002,13 @@ public void checkImage(HandleDes handleDes, String des, String pathValue, double if (pathValue.startsWith("http")) { file = DownImageTool.download(pathValue); } - File localCap = getScreenToLocal(); - FileSystemResource resource1 = new FileSystemResource(file); - FileSystemResource resource2 = new FileSystemResource(localCap); - MultiValueMap param = new LinkedMultiValueMap<>(); - param.add("file1", resource1); - param.add("file2", resource2); - param.add("type", "checker"); - try { - ResponseEntity responseEntity = - restTemplate.postForEntity(baseUrl + "/upload/cv", param, JSONObject.class); - if (responseEntity.getBody().getInteger("code") == 2000) { - double score = responseEntity.getBody().getDouble("data"); - handleDes.setStepDes("检测" + des + "图片相似度"); - handleDes.setDetail("相似度为" + score * 100 + "%"); - if (score == 0) { - handleDes.setE(new Exception("图片相似度检测不通过！比对图片分辨率不一致！")); - } else if (score < (matchThreshold / 100)) { - handleDes.setE(new Exception("图片相似度检测不通过！expect " + matchThreshold + " but " + score * 100)); - } - } else { - handleDes.setE(new Exception("图片相似度检测出错！cv服务出错！")); - } - } catch (Exception e) { - e.printStackTrace(); - handleDes.setE(new Exception("图片相似度检测出错！cv服务访问出错！")); - } finally { - file.delete(); - localCap.delete(); + double score = SimilarityChecker.getSimilarMSSIMScore(file, getScreenToLocal(), true); + handleDes.setStepDes("检测" + des + "图片相似度"); + handleDes.setDetail("相似度为" + score * 100 + "%"); + if (score == 0) { + handleDes.setE(new Exception("图片相似度检测不通过！比对图片分辨率不一致！")); + } else if (score < (matchThreshold / 100)) { + handleDes.setE(new Exception("图片相似度检测不通过！expect " + matchThreshold + " but " + score * 100)); } } diff --git a/src/main/java/org/cloud/sonic/agent/automation/IOSStepHandler.java b/src/main/java/org/cloud/sonic/agent/automation/IOSStepHandler.java index e03831aa..5851752f 100644 --- a/src/main/java/org/cloud/sonic/agent/automation/IOSStepHandler.java +++ b/src/main/java/org/cloud/sonic/agent/automation/IOSStepHandler.java @@ -4,6 +4,10 @@ import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.cloud.sonic.agent.bridge.ios.TIDeviceTool; +import org.cloud.sonic.agent.cv.AKAZEFinder; +import org.cloud.sonic.agent.cv.SIFTFinder; +import org.cloud.sonic.agent.cv.SimilarityChecker; +import org.cloud.sonic.agent.cv.TemMatcher; import org.cloud.sonic.agent.interfaces.ErrorType; import org.cloud.sonic.agent.interfaces.ResultDetailStatus; import org.cloud.sonic.agent.interfaces.StepType; @@ -62,10 +66,6 @@ */ public class IOSStepHandler { public LogTool log = new LogTool(); - private RestTemplate restTemplate = SpringTool.getBean(RestTemplate.class); - private Environment environment = SpringTool.getBean(Environment.class); - private String baseUrl = "http://" + environment.getProperty("sonic.server.host") - + ":" + environment.getProperty("sonic.server.folder-port") + "/api/folder"; private IOSDriver iosDriver; private JSONObject globalParams = new JSONObject(); private String testPackage = ""; @@ -580,40 +580,56 @@ public void clickByImg(HandleDes handleDes, String des, String pathValue) throws return; } } - File localCap = getScreenToLocal(); - FindResult findResult; - FileSystemResource resource1 = new FileSystemResource(file); - FileSystemResource resource2 = new FileSystemResource(localCap); - MultiValueMap param = new LinkedMultiValueMap<>(); - param.add("file1", resource1); - param.add("file2", resource2); - param.add("type", "finder"); + FindResult findResult = null; try { - ResponseEntity responseEntity = - restTemplate.postForEntity(baseUrl + "/upload/cv", param, JSONObject.class); - if (responseEntity.getBody().getInteger("code") == 2000) { - findResult = responseEntity.getBody().getJSONObject("data").toJavaObject(FindResult.class); + SIFTFinder siftFinder = new SIFTFinder(); + findResult = siftFinder.getSIFTFindResult(file, getScreenToLocal()); + } catch (Exception e) { + log.sendStepLog(StepType.WARN, "SIFT图像算法出错，切换算法中...", + ""); + } + if (findResult != null) { + log.sendStepLog(StepType.INFO, "图片定位到坐标：(" + findResult.getX() + "," + findResult.getY() + ") 耗时：" + findResult.getTime() + " ms", + findResult.getUrl()); + } else { + log.sendStepLog(StepType.INFO, "SIFT算法无法定位图片，切换AKAZE算法中...", + ""); + try { + AKAZEFinder akazeFinder = new AKAZEFinder(); + findResult = akazeFinder.getAKAZEFindResult(file, getScreenToLocal()); + } catch (Exception e) { + log.sendStepLog(StepType.WARN, "AKAZE图像算法出错，切换模版匹配算法中...", + ""); + } + if (findResult != null) { + log.sendStepLog(StepType.INFO, "图片定位到坐标：(" + findResult.getX() + "," + findResult.getY() + ") 耗时：" + findResult.getTime() + " ms", + findResult.getUrl()); + } else { + log.sendStepLog(StepType.INFO, "AKAZE算法无法定位图片，切换模版匹配算法中...", + ""); + try { + TemMatcher temMatcher = new TemMatcher(); + findResult = temMatcher.getTemMatchResult(file, getScreenToLocal()); + } catch (Exception e) { + log.sendStepLog(StepType.WARN, "模版匹配算法出错", + ""); + } if (findResult != null) { - try { - log.sendStepLog(StepType.INFO, "图片定位到坐标：(" + findResult.getX() + "," + findResult.getY() + ") 耗时：" + findResult.getTime() + " ms", - findResult.getUrl()); - TouchAction ta = new TouchAction(iosDriver); - ta.tap(PointOption.point(findResult.getX(), findResult.getY())).perform(); - } catch (Exception e) { - log.sendStepLog(StepType.ERROR, "点击" + des + "失败！", ""); - handleDes.setE(e); - } + log.sendStepLog(StepType.INFO, "图片定位到坐标：(" + findResult.getX() + "," + findResult.getY() + ") 耗时：" + findResult.getTime() + " ms", + findResult.getUrl()); + } else { + handleDes.setE(new Exception("图片定位失败！")); } - } else if (responseEntity.getBody().getInteger("code") == 4003) { - handleDes.setE(new Exception("图像匹配失败！")); - } else { - handleDes.setE(new Exception("点击失败！cv服务出错！")); } - } catch (Exception e) { - handleDes.setE(new Exception("点击失败！cv服务访问出错！")); - } finally { - file.delete(); - localCap.delete(); + } + if (findResult != null) { + try { + TouchAction ta = new TouchAction(iosDriver); + ta.tap(PointOption.point(findResult.getX(), findResult.getY())).perform(); + } catch (Exception e) { + log.sendStepLog(StepType.ERROR, "点击" + des + "失败！", ""); + handleDes.setE(e); + } } } @@ -664,33 +680,13 @@ public void checkImage(HandleDes handleDes, String des, String pathValue, double if (pathValue.startsWith("http")) { file = DownImageTool.download(pathValue); } - File localCap = getScreenToLocal(); - FileSystemResource resource1 = new FileSystemResource(file); - FileSystemResource resource2 = new FileSystemResource(localCap); - MultiValueMap param = new LinkedMultiValueMap<>(); - param.add("file1", resource1); - param.add("file2", resource2); - param.add("type", "checker"); - try { - ResponseEntity responseEntity = - restTemplate.postForEntity(baseUrl + "/upload/cv", param, JSONObject.class); - if (responseEntity.getBody().getInteger("code") == 2000) { - double score = responseEntity.getBody().getDouble("data"); - handleDes.setStepDes("检测" + des + "图片相似度"); - handleDes.setDetail("相似度为" + score * 100 + "%"); - if (score == 0) { - handleDes.setE(new Exception("图片相似度检测不通过！比对图片分辨率不一致！")); - } else if (score < (matchThreshold / 100)) { - handleDes.setE(new Exception("图片相似度检测不通过！expect " + matchThreshold + " but " + score * 100)); - } - } else { - handleDes.setE(new Exception("图片相似度检测出错！cv服务出错！")); - } - } catch (Exception e) { - handleDes.setE(new Exception("图片相似度检测出错！cv服务访问出错！")); - } finally { - file.delete(); - localCap.delete(); + double score = SimilarityChecker.getSimilarMSSIMScore(file, getScreenToLocal(), true); + handleDes.setStepDes("检测" + des + "图片相似度"); + handleDes.setDetail("相似度为" + score * 100 + "%"); + if (score == 0) { + handleDes.setE(new Exception("图片相似度检测不通过！比对图片分辨率不一致！")); + } else if (score < (matchThreshold / 100)) { + handleDes.setE(new Exception("图片相似度检测不通过！expect " + matchThreshold + " but " + score * 100)); } } diff --git a/src/main/java/org/cloud/sonic/agent/cv/AKAZEFinder.java b/src/main/java/org/cloud/sonic/agent/cv/AKAZEFinder.java new file mode 100644 index 00000000..5c344611 --- /dev/null +++ b/src/main/java/org/cloud/sonic/agent/cv/AKAZEFinder.java @@ -0,0 +1,318 @@ +package org.cloud.sonic.agent.cv; + +import org.cloud.sonic.agent.automation.FindResult; +import org.cloud.sonic.agent.tools.UploadTools; +import org.bytedeco.opencv.opencv_core.*; +import org.bytedeco.opencv.opencv_features2d.AKAZE; +import org.bytedeco.opencv.opencv_flann.Index; +import org.bytedeco.opencv.opencv_flann.IndexParams; +import org.bytedeco.opencv.opencv_flann.LshIndexParams; +import org.bytedeco.opencv.opencv_flann.SearchParams; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.concurrent.ThreadLocalRandom; + +import static org.bytedeco.opencv.global.opencv_calib3d.CV_RANSAC; +import static org.bytedeco.opencv.global.opencv_calib3d.findHomography; +import static org.bytedeco.opencv.global.opencv_core.*; +import static org.bytedeco.opencv.global.opencv_flann.FLANN_DIST_HAMMING; +import static org.bytedeco.opencv.global.opencv_imgcodecs.*; +import static org.bytedeco.opencv.global.opencv_imgproc.*; + +public class AKAZEFinder { + private final Logger logger = LoggerFactory.getLogger(AKAZEFinder.class); + IplImage objectImage = null; + AKAZE detector = AKAZE.create(); + double distanceThreshold = 0.6; + int matchesMin = 4; + double ransacReprojThreshold = 1.0; + boolean useFLANN = false; + KeyPointVector objectKeypoints = null, imageKeypoints = null; + Mat objectDescriptors = null, imageDescriptors = null; + Mat indicesMat, distsMat; + Index flannIndex = null; + IndexParams indexParams = null; + SearchParams searchParams = null; + Mat pt1 = null, pt2 = null, mask = null, H = null; + ArrayList ptpairs = null; + + public void init() { + objectKeypoints = new KeyPointVector(); + objectDescriptors = new Mat(); + detector.detectAndCompute(cvarrToMat(objectImage), + new Mat(), objectKeypoints, objectDescriptors, false); + + int total = (int) objectKeypoints.size(); + if (useFLANN) { + indicesMat = new Mat(total, 2, CV_32SC1); + distsMat = new Mat(total, 2, CV_32FC1); + flannIndex = new Index(); + indexParams = new LshIndexParams(12, 20, 2); // using LSH Hamming distance + searchParams = new SearchParams(64, 0, true); // maximum number of leafs checked + searchParams.deallocate(false); // for some reason FLANN seems to do it for us + } + pt1 = new Mat(total, 1, CV_32FC2); + pt2 = new Mat(total, 1, CV_32FC2); + mask = new Mat(total, 1, CV_8UC1); + H = new Mat(3, 3, CV_64FC1); + ptpairs = new ArrayList(2 * objectDescriptors.rows()); + logger.info("模版图一共有" + total + "个特征点"); + } + + public double[] find(IplImage image) { + if (objectDescriptors.rows() < matchesMin) { + return null; + } + imageKeypoints = new KeyPointVector(); + imageDescriptors = new Mat(); + detector.detectAndCompute(cvarrToMat(image), + new Mat(), imageKeypoints, imageDescriptors, false); + if (imageDescriptors.rows() < matchesMin) { + return null; + } + + int total = (int) imageKeypoints.size(); + logger.info("原图一共有" + total + "个特征点"); + + int w = objectImage.width(); + int h = objectImage.height(); + double[] srcCorners = {0, 0, w, 0, w, h, 0, h}; + double[] dstCorners = locatePlanarObject(objectKeypoints, objectDescriptors, + imageKeypoints, imageDescriptors, srcCorners); + return dstCorners; + } + + private static final int[] bits = new int[256]; + + static { + for (int i = 0; i < bits.length; i++) { + for (int j = i; j != 0; j >>= 1) { + bits[i] += j & 0x1; + } + } + } + + private int compareDescriptors(ByteBuffer d1, ByteBuffer d2, int best) { + int totalCost = 0; + assert d1.limit() - d1.position() == d2.limit() - d2.position(); + while (d1.position() < d1.limit()) { + totalCost += bits[(d1.get() ^ d2.get()) & 0xFF]; + if (totalCost > best) + break; + } + return totalCost; + } + + private int naiveNearestNeighbor(ByteBuffer vec, ByteBuffer modelDescriptors) { + int neighbor = -1; + int d, dist1 = Integer.MAX_VALUE, dist2 = Integer.MAX_VALUE; + int size = vec.limit() - vec.position(); + + for (int i = 0; i * size < modelDescriptors.capacity(); i++) { + ByteBuffer mvec = (ByteBuffer) modelDescriptors.position(i * size).limit((i + 1) * size); + d = compareDescriptors((ByteBuffer) vec.reset(), mvec, dist2); + if (d < dist1) { + dist2 = dist1; + dist1 = d; + neighbor = i; + } else if (d < dist2) { + dist2 = d; + } + } + if (dist1 < distanceThreshold * dist2) + return neighbor; + return -1; + } + + private void findPairs(Mat objectDescriptors, Mat imageDescriptors) { + int size = imageDescriptors.cols(); + ByteBuffer objectBuf = objectDescriptors.createBuffer(); + ByteBuffer imageBuf = imageDescriptors.createBuffer(); + + for (int i = 0; i * size < objectBuf.capacity(); i++) { + ByteBuffer descriptor = (ByteBuffer) objectBuf.position(i * size).limit((i + 1) * size).mark(); + int nearestNeighbor = naiveNearestNeighbor(descriptor, imageBuf); + if (nearestNeighbor >= 0) { + ptpairs.add(i); + ptpairs.add(nearestNeighbor); + } + } + } + + private void flannFindPairs(Mat objectDescriptors, Mat imageDescriptors) { + int length = objectDescriptors.rows(); + flannIndex.build(imageDescriptors, indexParams, FLANN_DIST_HAMMING); + flannIndex.knnSearch(objectDescriptors, indicesMat, distsMat, 2, searchParams); + IntBuffer indicesBuf = indicesMat.createBuffer(); + IntBuffer distsBuf = distsMat.createBuffer(); + for (int i = 0; i < length; i++) { + if (distsBuf.get(2 * i) < distanceThreshold * distsBuf.get(2 * i + 1)) { + ptpairs.add(i); + ptpairs.add(indicesBuf.get(2 * i)); + } + } + } + + private double[] locatePlanarObject(KeyPointVector objectKeypoints, Mat objectDescriptors, + KeyPointVector imageKeypoints, Mat imageDescriptors, double[] srcCorners) { + ptpairs.clear(); + if (useFLANN) { + flannFindPairs(objectDescriptors, imageDescriptors); + } else { + findPairs(objectDescriptors, imageDescriptors); + } + int n = ptpairs.size() / 2; + logger.info("筛选后共有" + n + "个吻合点"); + if (n < matchesMin) { + return null; + } + + pt1.resize(n); + pt2.resize(n); + mask.resize(n); + FloatBuffer pt1Idx = pt1.createBuffer(); + FloatBuffer pt2Idx = pt2.createBuffer(); + for (int i = 0; i < n; i++) { + Point2f p1 = objectKeypoints.get(ptpairs.get(2 * i)).pt(); + pt1Idx.put(2 * i, p1.x()); + pt1Idx.put(2 * i + 1, p1.y()); + Point2f p2 = imageKeypoints.get(ptpairs.get(2 * i + 1)).pt(); + pt2Idx.put(2 * i, p2.x()); + pt2Idx.put(2 * i + 1, p2.y()); + } + + H = findHomography(pt1, pt2, CV_RANSAC, ransacReprojThreshold, mask, 2000, 0.995); + if (H.empty() || countNonZero(mask) < matchesMin) { + return null; + } + + double[] h = (double[]) H.createIndexer(false).array(); + double[] dstCorners = new double[srcCorners.length]; + for (int i = 0; i < srcCorners.length / 2; i++) { + double x = srcCorners[2 * i], y = srcCorners[2 * i + 1]; + double Z = 1 / (h[6] * x + h[7] * y + h[8]); + double X = (h[0] * x + h[1] * y + h[2]) * Z; + double Y = (h[3] * x + h[4] * y + h[5]) * Z; + dstCorners[2 * i] = X; + dstCorners[2 * i + 1] = Y; + } + return dstCorners; + } + + public static Scalar randColor() { + int b, g, r; + b = ThreadLocalRandom.current().nextInt(0, 255); + g = ThreadLocalRandom.current().nextInt(0, 255); + r = ThreadLocalRandom.current().nextInt(0, 255); + return new Scalar(b, g, r, 0); + } + + public FindResult getAKAZEFindResult(File temFile, File beforeFile) throws IOException { + IplImage object = cvLoadImage(temFile.getAbsolutePath(), IMREAD_GRAYSCALE); + IplImage image = cvLoadImage(beforeFile.getAbsolutePath(), IMREAD_GRAYSCALE); + logger.info("原图宽：" + image.width()); + logger.info("原图高：" + image.height()); + if (object == null || image == null) { + logger.error("读取图片失败！"); + temFile.delete(); + beforeFile.delete(); + return null; + } + + IplImage correspond = IplImage.create(image.width() + object.width(), image.height(), 8, 1); + cvSetImageROI(correspond, cvRect(0, 0, image.width(), correspond.height())); + cvCopy(image, correspond); + cvSetImageROI(correspond, cvRect(image.width(), 0, object.width(), object.height())); + cvCopy(object, correspond); + cvResetImageROI(correspond); + + objectImage = object; + useFLANN = true; + ransacReprojThreshold = 3; + init(); + + long start = System.currentTimeMillis(); + double[] dst_corners = find(image); + FindResult findResult = new FindResult(); + findResult.setTime((int) (System.currentTimeMillis() - start)); + logger.info("特征匹配时间: " + (System.currentTimeMillis() - start) + " ms"); + + IplImage correspondColor = IplImage.create(correspond.width(), correspond.height(), 8, 3); + cvCvtColor(correspond, correspondColor, CV_GRAY2BGR); + cvSetImageROI(correspondColor, cvRect(0, 0, image.width(), correspondColor.height())); + cvCopy(cvLoadImage(beforeFile.getAbsolutePath(), IMREAD_COLOR), correspondColor); + cvSetImageROI(correspondColor, cvRect(image.width(), 0, object.width(), object.height())); + cvCopy(cvLoadImage(temFile.getAbsolutePath(), IMREAD_COLOR), correspondColor); + cvResetImageROI(correspondColor); + + if (dst_corners != null) { + int resultX = 0; + int resultY = 0; + for (int i = 0; i < 4; i++) { + int j = (i + 1) % 4; + int x1 = (int) Math.round(dst_corners[2 * i]); + int y1 = (int) Math.round(dst_corners[2 * i + 1]); + int x2 = (int) Math.round(dst_corners[2 * j]); + int y2 = (int) Math.round(dst_corners[2 * j + 1]); + line(cvarrToMat(correspondColor), new Point(x1, y1), + new Point(x2, y2), + Scalar.RED, 2, CV_AA, 0); + if (i == 0) { + resultX = (x1 + x2) / 2; + } + if (i == 1) { + resultY = (y1 + y2) / 2; + } + } + if (resultX == 0 && resultY == 0) { + temFile.delete(); + beforeFile.delete(); + return null; + } + findResult.setX(resultX); + findResult.setY(resultY); + logger.info("结果坐标为（" + resultX + "," + resultY + ")"); + } + + if (objectKeypoints != null) { + for (int i = 0; i < objectKeypoints.size(); i++) { + KeyPoint r = objectKeypoints.get(i); + Point center = new Point(Math.round(r.pt().x()) + image.width(), Math.round(r.pt().y())); + int radius = Math.round(r.size() / 2); + circle(cvarrToMat(correspondColor), center, radius, randColor(), 1, CV_AA, 0); + } + } + + if (imageKeypoints != null) { + for (int i = 0; i < imageKeypoints.size(); i++) { + KeyPoint r = imageKeypoints.get(i); + Point center = new Point(Math.round(r.pt().x()), Math.round(r.pt().y())); + int radius = Math.round(r.size() / 2); + circle(cvarrToMat(correspondColor), center, radius, randColor(), 1, CV_AA, 0); + } + } + + for (int i = 0; i < ptpairs.size(); i += 2) { + Point2f pt1 = objectKeypoints.get(ptpairs.get(i)).pt(); + Point2f pt2 = imageKeypoints.get(ptpairs.get(i + 1)).pt(); + line(cvarrToMat(correspondColor), new Point(Math.round(pt1.x()) + image.width(), Math.round(pt1.y())), + new Point(Math.round(pt2.x()), Math.round(pt2.y())), + randColor(), 1, CV_AA, 0); + } + long time = Calendar.getInstance().getTimeInMillis(); + String fileName = "test-output" + File.separator + time + ".jpg"; + cvSaveImage(fileName, correspondColor); + findResult.setUrl(UploadTools.upload(new File(fileName), "imageFiles")); + temFile.delete(); + beforeFile.delete(); + return findResult; + } +} \ No newline at end of file diff --git a/src/main/java/org/cloud/sonic/agent/cv/SIFTFinder.java b/src/main/java/org/cloud/sonic/agent/cv/SIFTFinder.java new file mode 100644 index 00000000..2f058e74 --- /dev/null +++ b/src/main/java/org/cloud/sonic/agent/cv/SIFTFinder.java @@ -0,0 +1,103 @@ +package org.cloud.sonic.agent.cv; + +import org.bytedeco.opencv.opencv_core.*; +import org.bytedeco.opencv.opencv_features2d.FlannBasedMatcher; +import org.bytedeco.opencv.opencv_xfeatures2d.SIFT; +import org.cloud.sonic.agent.automation.FindResult; +import org.cloud.sonic.agent.tools.UploadTools; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.List; + +import static org.bytedeco.opencv.global.opencv_features2d.drawMatchesKnn; +import static org.bytedeco.opencv.global.opencv_imgcodecs.imread; +import static org.bytedeco.opencv.global.opencv_imgcodecs.imwrite; +import static org.bytedeco.opencv.global.opencv_imgproc.*; + +public class SIFTFinder { + private final Logger logger = LoggerFactory.getLogger(SIFTFinder.class); + + public FindResult getSIFTFindResult(File temFile, File beforeFile) throws Exception { + Mat image01 = imread(beforeFile.getAbsolutePath()); + Mat image02 = imread(temFile.getAbsolutePath()); + + Mat image1 = new Mat(); + Mat image2 = new Mat(); + cvtColor(image01, image1, COLOR_BGR2GRAY); + cvtColor(image02, image2, COLOR_BGR2GRAY); + + KeyPointVector keyPointVector1 = new KeyPointVector(); + KeyPointVector keyPointVector2 = new KeyPointVector(); + Mat image11 = new Mat(); + Mat image22 = new Mat(); + + long start = System.currentTimeMillis(); + SIFT sift = SIFT.create(); + sift.detectAndCompute(image1, image1, keyPointVector1, image11); + sift.detectAndCompute(image2, image2, keyPointVector2, image22); + + FlannBasedMatcher flannBasedMatcher = new FlannBasedMatcher(); + DMatchVectorVector matchPoints = new DMatchVectorVector(); + + flannBasedMatcher.knnMatch(image11, image22, matchPoints, 2); + logger.info("处理前共有匹配数：" + matchPoints.size()); + DMatchVectorVector goodMatches = new DMatchVectorVector(); + + List xs = new ArrayList<>(); + List ys = new ArrayList<>(); + for (long i = 0; i < matchPoints.size(); i++) { + if (matchPoints.get(i).size() >= 2) { + DMatch match1 = matchPoints.get(i).get(0); + DMatch match2 = matchPoints.get(i).get(1); + + if (match1.distance() <= 0.6 * match2.distance()) { + xs.add((int) keyPointVector1.get(match1.queryIdx()).pt().x()); + ys.add((int) keyPointVector1.get(match1.queryIdx()).pt().y()); + goodMatches.push_back(matchPoints.get(i)); + } + } + } + logger.info("处理后匹配数：" + goodMatches.size()); + if (goodMatches.size() <= 4) { + temFile.delete(); + beforeFile.delete(); + return null; + } + FindResult findResult = new FindResult(); + findResult.setTime((int) (System.currentTimeMillis() - start)); + Mat result = new Mat(); + + drawMatchesKnn(image01, keyPointVector1, image02, keyPointVector2, goodMatches, result); + + int resultX = majorityElement(xs); + int resultY = majorityElement(ys); + findResult.setX(resultX); + findResult.setY(resultY); + logger.info("结果坐标为（" + resultX + "," + resultY + ")"); + circle(result, new Point(resultX, resultY), 5, Scalar.RED, 10, CV_AA, 0); + long time = Calendar.getInstance().getTimeInMillis(); + String fileName = "test-output" + File.separator + time + ".jpg"; + imwrite(fileName, result); + findResult.setUrl(UploadTools.upload(new File(fileName), "imageFiles")); + temFile.delete(); + beforeFile.delete(); + return findResult; + } + + public static int majorityElement(List nums) { + double j; + Collections.sort(nums); + int size = nums.size(); + if (size % 2 == 1) { + j = nums.get((size - 1) / 2); + } else { + j = (nums.get(size / 2 - 1) + nums.get(size / 2) + 0.0) / 2; + } + return (int) j; + } +} diff --git a/src/main/java/org/cloud/sonic/agent/cv/SimilarityChecker.java b/src/main/java/org/cloud/sonic/agent/cv/SimilarityChecker.java new file mode 100644 index 00000000..f6a2d7f3 --- /dev/null +++ b/src/main/java/org/cloud/sonic/agent/cv/SimilarityChecker.java @@ -0,0 +1,70 @@ +package org.cloud.sonic.agent.cv; + +import org.bytedeco.opencv.global.opencv_core; +import org.bytedeco.opencv.opencv_core.Mat; +import org.bytedeco.opencv.opencv_core.Scalar; +import org.bytedeco.opencv.opencv_core.Size; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.math.BigDecimal; +import java.math.RoundingMode; + +import static org.bytedeco.opencv.global.opencv_core.*; +import static org.bytedeco.opencv.global.opencv_imgcodecs.imread; +import static org.bytedeco.opencv.global.opencv_imgproc.GaussianBlur; + +public class SimilarityChecker { + private final Logger logger = LoggerFactory.getLogger(SimilarityChecker.class); + + public static double getSimilarMSSIMScore(File file1, File file2, Boolean isDelete) { + synchronized (SimilarityChecker.class) { + Mat i1 = imread(file1.getAbsolutePath()); + Mat i2 = imread(file2.getAbsolutePath()); + if (i1.size().get() != i2.size().get()) { + return 0; + } + double C1 = 6.5025, C2 = 58.5225; + int d = opencv_core.CV_32F; + Mat I1 = new Mat(); + Mat I2 = new Mat(); + i1.convertTo(I1, d); + i2.convertTo(I2, d); + Mat I2_2 = I2.mul(I2).asMat(); + Mat I1_2 = I1.mul(I1).asMat(); + Mat I1_I2 = I1.mul(I2).asMat(); + Mat mu1 = new Mat(); + Mat mu2 = new Mat(); + GaussianBlur(I1, mu1, new Size(11, 11), 1.5); + GaussianBlur(I2, mu2, new Size(11, 11), 1.5); + Mat mu1_2 = mu1.mul(mu1).asMat(); + Mat mu2_2 = mu2.mul(mu2).asMat(); + Mat mu1_mu2 = mu1.mul(mu2).asMat(); + Mat sigma1_2 = new Mat(); + Mat sigma2_2 = new Mat(); + Mat sigma12 = new Mat(); + GaussianBlur(I1_2, sigma1_2, new Size(11, 11), 1.5); + sigma1_2 = subtract(sigma1_2, mu1_2).asMat(); + GaussianBlur(I2_2, sigma2_2, new Size(11, 11), 1.5); + sigma2_2 = subtract(sigma2_2, mu2_2).asMat(); + GaussianBlur(I1_I2, sigma12, new Size(11, 11), 1.5); + sigma12 = subtract(sigma12, mu1_mu2).asMat(); + Mat t1, t2, t3; + t1 = add(multiply(2, mu1_mu2), Scalar.all(C1)).asMat(); + t2 = add(multiply(2, sigma12), Scalar.all(C2)).asMat(); + t3 = t1.mul(t2).asMat(); + t1 = add(add(mu1_2, mu2_2), Scalar.all(C1)).asMat(); + t2 = add(add(sigma1_2, sigma2_2), Scalar.all(C2)).asMat(); + t1 = t1.mul(t2).asMat(); + Mat ssim_map = new Mat(); + divide(t3, t1, ssim_map); + Scalar mSsim = mean(ssim_map); + if (isDelete) { + file1.delete(); + file2.delete(); + } + return new BigDecimal((mSsim.get(0) + mSsim.get(1) + mSsim.get(2)) / 3).setScale(2, RoundingMode.DOWN).doubleValue(); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/cloud/sonic/agent/cv/TemMatcher.java b/src/main/java/org/cloud/sonic/agent/cv/TemMatcher.java new file mode 100644 index 00000000..ecaf382d --- /dev/null +++ b/src/main/java/org/cloud/sonic/agent/cv/TemMatcher.java @@ -0,0 +1,61 @@ +package org.cloud.sonic.agent.cv; + +import org.bytedeco.javacpp.DoublePointer; +import org.bytedeco.opencv.opencv_core.*; +import org.cloud.sonic.agent.automation.FindResult; +import org.cloud.sonic.agent.tools.UploadTools; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Calendar; +import java.util.concurrent.ThreadLocalRandom; + +import static org.bytedeco.opencv.global.opencv_core.*; +import static org.bytedeco.opencv.global.opencv_imgcodecs.*; +import static org.bytedeco.opencv.global.opencv_imgproc.*; + +public class TemMatcher { + private final Logger logger = LoggerFactory.getLogger(TemMatcher.class); + + public FindResult getTemMatchResult(File temFile, File beforeFile) throws IOException { + try { + Mat sourceColor = imread(beforeFile.getAbsolutePath()); + Mat sourceGrey = new Mat(sourceColor.size(), CV_8UC1); + cvtColor(sourceColor, sourceGrey, COLOR_BGR2GRAY); + Mat template = imread(temFile.getAbsolutePath(), IMREAD_GRAYSCALE); + Size size = new Size(sourceGrey.cols() - template.cols() + 1, sourceGrey.rows() - template.rows() + 1); + Mat result = new Mat(size, CV_32FC1); + + long start = System.currentTimeMillis(); + matchTemplate(sourceGrey, template, result, TM_CCORR_NORMED); + DoublePointer minVal = new DoublePointer(); + DoublePointer maxVal = new DoublePointer(); + Point min = new Point(); + Point max = new Point(); + minMaxLoc(result, minVal, maxVal, min, max, null); + rectangle(sourceColor, new Rect(max.x(), max.y(), template.cols(), template.rows()), randColor(), 2, 0, 0); + FindResult findResult = new FindResult(); + findResult.setTime((int) (System.currentTimeMillis() - start)); + long time = Calendar.getInstance().getTimeInMillis(); + String fileName = "test-output" + File.separator + time + ".jpg"; + imwrite(fileName, sourceColor); + findResult.setX(max.x() + template.cols() / 2); + findResult.setY(max.y() + template.rows() / 2); + findResult.setUrl(UploadTools.upload(new File(fileName), "imageFiles")); + return findResult; + } finally { + temFile.delete(); + beforeFile.delete(); + } + } + + public static Scalar randColor() { + int b, g, r; + b = ThreadLocalRandom.current().nextInt(0, 255 + 1); + g = ThreadLocalRandom.current().nextInt(0, 255 + 1); + r = ThreadLocalRandom.current().nextInt(0, 255 + 1); + return new Scalar(b, g, r, 0); + } +} \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 97ee7be7..9473ce6c 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,4 +1,4 @@ spring: version: @project.version@ profiles: - active: @profileActive@ \ No newline at end of file + active: dev,@profileActive@ \ No newline at end of file