From d773c5a3792bab944d9dfa21feaab0491296b01d Mon Sep 17 00:00:00 2001
From: TTangNingzhi <tnz241736@gmail.com>
Date: Mon, 25 Dec 2023 09:29:17 -0500
Subject: [PATCH] try jcodec

---
 .idea/workspace.xml                        | 35 +++++++++--
 build.gradle.kts                           |  8 ++-
 src/main/java/component/AlertDialog.java   |  2 -
 src/main/java/component/ConfigDialog.java  |  2 +-
 src/main/java/trackers/EyeTracker.java     |  4 +-
 src/main/java/trackers/ScreenRecorder.java | 68 ++++++++++++----------
 6 files changed, 74 insertions(+), 45 deletions(-)

diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 6000fc4..6207d6f 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -6,9 +6,11 @@
   <component name="ChangeListManager">
     <list default="true" id="14bb91d1-ebd0-4f11-b77e-711543adf329" name="Changes" comment="refresh">
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/API Reference.md" beforeDir="false" afterPath="$PROJECT_DIR$/API Reference.md" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/DOCUMENT.md" beforeDir="false" afterPath="$PROJECT_DIR$/Output Format.md" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/build.gradle.kts" beforeDir="false" afterPath="$PROJECT_DIR$/build.gradle.kts" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/main/java/component/AlertDialog.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/component/AlertDialog.java" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/main/java/component/ConfigDialog.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/component/ConfigDialog.java" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/main/java/trackers/EyeTracker.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/trackers/EyeTracker.java" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/src/main/java/trackers/ScreenRecorder.java" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/java/trackers/ScreenRecorder.java" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -69,17 +71,18 @@
   </component>
   <component name="PropertiesComponent">{
   &quot;keyToString&quot;: {
+    &quot;JavaVariableFindUsagesOptions.isSearchForAccessors&quot;: &quot;true&quot;,
     &quot;RunOnceActivity.OpenProjectViewOnStart&quot;: &quot;true&quot;,
     &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
     &quot;WebServerToolWindowFactoryState&quot;: &quot;false&quot;,
     &quot;git-widget-placeholder&quot;: &quot;main&quot;,
-    &quot;last_opened_file_path&quot;: &quot;C:/Users/Lenovo/IdeaProjects/CodeGRITS&quot;,
+    &quot;last_opened_file_path&quot;: &quot;E:/Tai-e-assignments/A1/tai-e&quot;,
     &quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
     &quot;node.js.detected.package.tslint&quot;: &quot;true&quot;,
     &quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
     &quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
     &quot;nodejs_package_manager_path&quot;: &quot;npm&quot;,
-    &quot;settings.editor.selected.configurable&quot;: &quot;reference.settingsdialog.project.gradle&quot;,
+    &quot;settings.editor.selected.configurable&quot;: &quot;inlay.hints&quot;,
     &quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
   }
 }</component>
@@ -103,6 +106,18 @@
     </recent_temporary>
   </component>
   <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
+  <component name="StructuralSearchPlugin">
+    <searchConfiguration name="$Qualifier$::$Method$" created="1700455312225" uuid="9637ce76-ebdf-3549-a5b2-67a3262c83d2" text="$Qualifier$::$Method$" recursive="true" caseInsensitive="false" type="JAVA" pattern_context="default" scope_type="PROJECT" scope_descriptor="" search_injected="false" history="1">
+      <constraint name="__context__" within="" contains="" />
+      <constraint name="Qualifier" within="" contains="" />
+      <constraint name="Method" target="true" within="" contains="" />
+    </searchConfiguration>
+    <searchConfiguration name="$Expr$ instanceof $Type$" created="1700455267644" uuid="87383e7a-b033-3359-beda-410098c0d97e" text="$Expr$ instanceof $Type$" recursive="true" caseInsensitive="false" type="JAVA" pattern_context="default" scope_type="PROJECT" scope_descriptor="" search_injected="false" history="1">
+      <constraint name="__context__" within="" contains="" />
+      <constraint name="Expr" within="" contains="" />
+      <constraint name="Type" within="" contains="" />
+    </searchConfiguration>
+  </component>
   <component name="TaskManager">
     <task active="true" id="Default" summary="Default task">
       <changelist id="14bb91d1-ebd0-4f11-b77e-711543adf329" name="Changes" comment="" />
@@ -121,6 +136,16 @@
       <workItem from="1698538889005" duration="1115000" />
       <workItem from="1698547984000" duration="82000" />
       <workItem from="1699580673480" duration="3076000" />
+      <workItem from="1700450377387" duration="524000" />
+      <workItem from="1700450925421" duration="8149000" />
+      <workItem from="1701319707843" duration="4935000" />
+      <workItem from="1702446987219" duration="254000" />
+      <workItem from="1702790447857" duration="725000" />
+      <workItem from="1703293633081" duration="735000" />
+      <workItem from="1703485997553" duration="452000" />
+      <workItem from="1703495906239" duration="25000" />
+      <workItem from="1703495947720" duration="14652000" />
+      <workItem from="1703514033102" duration="497000" />
     </task>
     <task id="LOCAL-00001" summary="fix api bug">
       <option name="closed" value="true" />
diff --git a/build.gradle.kts b/build.gradle.kts
index 2396b07..919c8da 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -17,15 +17,17 @@ dependencies {
     // https://mvnrepository.com/artifact/com.opencsv/opencsv
     // https://mvnrepository.com/artifact/com.opencsv/opencsv
     implementation("com.opencsv:opencsv:5.7.1")
+    implementation("org.bytedeco:javacv-platform:1.5.9")
+
 }
 
 // Configure Gradle IntelliJ Plugin
 // Read more: https://plugins.jetbrains.com/docs/intellij/tools-gradle-intellij-plugin.html
 intellij {
-    version.set("2022.2.5")
+    version.set("2023.1.4")
 
-//    type.set("IC")
-//    plugins.set(listOf("com.intellij.java"))
+    type.set("IC")
+    plugins.set(listOf("com.intellij.java"))
 
 //    type.set("PC")
 //    plugins.set(listOf("PythonCore"))
diff --git a/src/main/java/component/AlertDialog.java b/src/main/java/component/AlertDialog.java
index 04cc120..ad70d6b 100644
--- a/src/main/java/component/AlertDialog.java
+++ b/src/main/java/component/AlertDialog.java
@@ -33,8 +33,6 @@ public AlertDialog(String label,Icon icon) {
         JPanel dialogPanel = new JPanel();
         dialogPanel.setLayout(new BoxLayout(dialogPanel, BoxLayout.Y_AXIS));
 
-        //show a warning icon
-
         JLabel label = new JLabel(this.label);
         JLabel icon = new JLabel();
         label.setIcon(this.icon);
diff --git a/src/main/java/component/ConfigDialog.java b/src/main/java/component/ConfigDialog.java
index a954abc..c1a62b3 100644
--- a/src/main/java/component/ConfigDialog.java
+++ b/src/main/java/component/ConfigDialog.java
@@ -79,7 +79,7 @@ public ConfigDialog(Project project) throws IOException, InterruptedException {
                     for (String freq : freqList) {
                         freqCombo.addItem(Double.parseDouble(freq));
                     }
-                }else{ //use mouse and default freq
+                } else { //use mouse and default freq
                     deviceCombo.removeAllItems();
                     deviceCombo.addItem("Mouse");
                     freqCombo.removeAllItems();
diff --git a/src/main/java/trackers/EyeTracker.java b/src/main/java/trackers/EyeTracker.java
index 9cb1a7b..13a7cdd 100644
--- a/src/main/java/trackers/EyeTracker.java
+++ b/src/main/java/trackers/EyeTracker.java
@@ -153,7 +153,7 @@ public void startTracking(Project project) throws IOException {
         if (virtualFiles.length > 0) {
             filePath = virtualFiles[0].getPath();
         }
-        if (deviceIndex == 0 ) {
+        if (deviceIndex == 0) {
             setting.setAttribute("eye_tracker", "Mouse");
         } else {
             setting.setAttribute("eye_tracker", "Tobii Pro Fusion");
@@ -237,7 +237,7 @@ public void processRawData(String message) {
                 Element aSTStructure = getASTStructureElement(psiElement);
                 gaze.appendChild(aSTStructure);
                 lastElement = psiElement;
-                System.out.println(gaze.getAttribute("timestamp") + " " + System.currentTimeMillis());
+//                System.out.println(gaze.getAttribute("timestamp") + " " + System.currentTimeMillis());
                 handleElement(gaze);
             }
         }));
diff --git a/src/main/java/trackers/ScreenRecorder.java b/src/main/java/trackers/ScreenRecorder.java
index def152a..d92685b 100644
--- a/src/main/java/trackers/ScreenRecorder.java
+++ b/src/main/java/trackers/ScreenRecorder.java
@@ -1,10 +1,13 @@
 package trackers;
 
 import com.opencsv.CSVWriter;
+import org.bytedeco.javacv.FFmpegFrameRecorder;
+import org.jcodec.api.FrameGrab;
 import org.jcodec.api.SequenceEncoder;
 import org.jcodec.api.awt.AWTSequenceEncoder;
 import org.jcodec.common.model.ColorSpace;
 import org.jcodec.common.model.Picture;
+import org.jcodec.scale.AWTUtil;
 
 import javax.imageio.ImageIO;
 import java.awt.*;
@@ -13,6 +16,8 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Timer;
+import java.util.TimerTask;
 
 public class ScreenRecorder {
 
@@ -22,18 +27,17 @@ public class ScreenRecorder {
      * 2: started, paused; only resumeAction enabled
      */
     int state = 0;
+    int frameRate = 4;
     private AWTSequenceEncoder awtEncoder;
+//    private FFmpegFrameRecorder recorder;
     private final ArrayList<String[]> timeList = new ArrayList<>();
     private CSVWriter csvWriter;
     boolean isRecording = false;
     private int clipNumber = 1;
+    private int frameNumber = 0;
     private String dataOutputPath = "";
-
     private static ScreenRecorder instance = null;
 
-    private ScreenRecorder() {
-    }
-
     public static ScreenRecorder getInstance() {
         if (instance == null) {
             instance = new ScreenRecorder();
@@ -44,12 +48,13 @@ public static ScreenRecorder getInstance() {
 
     private void createEncoder() throws IOException {
         awtEncoder = AWTSequenceEncoder.createSequenceEncoder(
-                new File(dataOutputPath + "/screen_recording/video_clip_" + clipNumber + ".mp4"), 24);
+                new File(dataOutputPath + "/screen_recording/video_clip_" + clipNumber + ".mp4"), frameRate);
     }
 
 
     public void startRecording() throws IOException {
         state = 1;
+        clipNumber = 1;
         isRecording = true;
         File file = new File(dataOutputPath + "/screen_recording/frames.csv");
         file.getParentFile().mkdirs();
@@ -86,37 +91,36 @@ public void resumeRecording() {
     }
 
     private void recordScreen() throws AWTException, IOException {
+        createEncoder();
+        frameNumber = 0;
         Rectangle bounds = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize());
         Robot robot = new Robot();
-        createEncoder();
-        long frameRate = 24;
-        Thread recordThread = new Thread(() -> {
-            long frameNumber = 0;
-            long lastFrameTime = System.nanoTime();
-            long frameDuration = 1000000000 / frameRate;
-            while (isRecording) {
-                long currentTime = System.nanoTime();
-                if (currentTime - lastFrameTime < frameDuration) {
-                    continue;
-                }
-                lastFrameTime = currentTime;
-
-                BufferedImage screenCapture = robot.createScreenCapture(bounds);
-                frameNumber++;
-                timeList.add(new String[]{String.valueOf(System.currentTimeMillis()), String.valueOf(frameNumber), String.valueOf(clipNumber)});
-                try {
-                    awtEncoder.encodeImage(screenCapture);
-                } catch (IOException e) {
-                    throw new RuntimeException(e);
+        Timer timer = new Timer();
+        timer.scheduleAtFixedRate(new TimerTask() {
+            @Override
+            public void run() {
+                if (!isRecording) {
+                    try {
+                        awtEncoder.finish();
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                    timer.cancel();
+                } else {
+                    BufferedImage screenCapture = robot.createScreenCapture(bounds);
+                    frameNumber++;
+                    timeList.add(new String[]{String.valueOf(System.currentTimeMillis()), String.valueOf(frameNumber), String.valueOf(clipNumber)});
+                    try {
+                        if (frameNumber == 1) {
+                            ImageIO.write(screenCapture, "png", new File(dataOutputPath + "/screen_recording/frame_" + frameNumber + ".png"));
+                        }
+                        awtEncoder.encodeImage(screenCapture);
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
                 }
             }
-            try {
-                awtEncoder.finish();
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        });
-        recordThread.start();
+        }, 0, 1000 / frameRate);
     }
 
     public void setDataOutputPath(String dataOutputPath) {