forked from ddupont808/GPT-4V-Act
-
Notifications
You must be signed in to change notification settings - Fork 0
/
preload-view.js
257 lines (221 loc) · 7.86 KB
/
preload-view.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
const customCSS = `
::-webkit-scrollbar {
width: 10px;
}
::-webkit-scrollbar-track {
background: #27272a;
}
::-webkit-scrollbar-thumb {
background: #888;
border-radius: 0.375rem;
}
::-webkit-scrollbar-thumb:hover {
background: #555;
}
`;
window.addEventListener('DOMContentLoaded', () => {
const styleTag = document.createElement('style');
styleTag.textContent = customCSS;
document.head.append(styleTag);
});
const { ipcRenderer } = require('electron');
// Listen for messages from preload.js to navigate
ipcRenderer.on('navigate-webview', (event, action, payload) => {
switch (action) {
case 'goBack':
if (window.history.length > 1) {
window.history.back();
}
break;
case 'goForward':
if (window.history.length > 1) {
window.history.forward();
}
break;
case 'reload':
window.location.reload();
break;
case 'loadURL':
window.location.href = payload;
break;
}
});
// Send the current URL whenever it changes
window.addEventListener('load', () => {
ipcRenderer.send('current-url', window.location.href);
let oldHref = document.location.href;
const body = document.querySelector("body");
const observer = new MutationObserver(mutations => {
if (oldHref !== document.location.href) {
oldHref = document.location.href;
ipcRenderer.send('current-url', window.location.href);
}
});
observer.observe(body, { childList: true, subtree: true });
});
window.addEventListener('beforeunload', () => {
ipcRenderer.send('current-url', window.location.href);
});
window.addEventListener('popstate', () => {
ipcRenderer.send('current-url', window.location.href);
});
ipcRenderer.on('randomize', (event) => {
// Define random x and y coordinates within the dimensions of the document
var x = Math.floor(Math.random() * (document.body.scrollWidth + 1));
var y = Math.floor(Math.random() * (document.body.scrollHeight + 1));
// Scroll to the random positions
window.scrollTo(x, y);
});
ipcRenderer.on('shuffle', (event) => {
// Helper function to shuffle a string
function shuffleString(str) {
var arr = str.split('');
var len = arr.length;
// While there remain elements to shuffle…
for (var i = len - 1; i > 0; i--) {
// Pick a remaining element…
var j = Math.floor(Math.random() * (i + 1));
// And swap it with the current element.
var temp = arr[i];
arr[i] = arr[j];
arr[j] = temp;
}
return arr.join(''); // Convert Array back to string
}
function shuffleChildren(parent) {
if(parent.children && parent.children.length > 0) {
var children = Array.from(parent.children);
while (children.length) {
// Pick a random index
var randomIndex = Math.floor(Math.random() * children.length);
// Get the child at the random index
var child = children.splice(randomIndex, 1)[0];
// Append the child to the parent
parent.appendChild(child);
// If child node only contains text, then shuffle the text
if(child.children.length === 0) {
child.textContent = shuffleString(child.textContent);
}
// Recursively shuffle children's children
shuffleChildren(child);
}
}
}
// Call the function with the body as the parent node
shuffleChildren(document.body);
// Define random x and y coordinates within the dimensions of the document
var x = Math.floor(Math.random() * (document.body.scrollWidth + 1));
var y = Math.floor(Math.random() * (document.body.scrollHeight + 1));
// Scroll to the random positions
window.scrollTo(x, y);
});
ipcRenderer.on('observer', (event, state, payload) => {
switch (state) {
case 'screenshot-start':
markPage();
break;
case 'screenshot-end':
unmarkPage();
break;
}
});
// DOM Labeler
let labels = [];
function unmarkPage() {
for(const label of labels) {
document.body.removeChild(label);
}
labels = [];
}
function markPage() {
unmarkPage();
var bodyRect = document.body.getBoundingClientRect();
var items = Array.prototype.slice.call(
document.querySelectorAll('*')
).map(function(element) {
var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0);
var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
var rects = [...element.getClientRects()].filter(bb => {
var center_x = bb.left + bb.width / 2;
var center_y = bb.top + bb.height / 2;
var elAtCenter = document.elementFromPoint(center_x, center_y);
return elAtCenter === element || element.contains(elAtCenter)
}).map(bb => {
const rect = {
left: Math.max(0, bb.left),
top: Math.max(0, bb.top),
right: Math.min(vw, bb.right),
bottom: Math.min(vh, bb.bottom)
};
return {
...rect,
width: rect.right - rect.left,
height: rect.bottom - rect.top
}
});
var area = rects.reduce((acc, rect) => acc + rect.width * rect.height, 0);
return {
element: element,
include:
(element.tagName === "INPUT" || element.tagName === "TEXTAREA" || element.tagName === "SELECT") ||
(element.tagName === "BUTTON" || element.tagName === "A" || (element.onclick != null) || window.getComputedStyle(element).cursor == "pointer") ||
(element.tagName === "IFRAME" || element.tagName === "VIDEO")
,
area,
rects,
text: element.textContent.trim().replace(/\s{2,}/g, ' ')
};
}).filter(item =>
item.include && (item.area >= 20)
);
// Only keep inner clickable items
items = items.filter(x => !items.some(y => x.element.contains(y.element) && !(x == y)))
// Function to generate random colors
function getRandomColor() {
var letters = '0123456789ABCDEF';
var color = '#';
for (var i = 0; i < 6; i++) {
color += letters[Math.floor(Math.random() * 16)];
}
return color;
}
// Lets create a floating border on top of these elements that will always be visible
items.forEach(function(item, index) {
item.rects.forEach((bbox) => {
newElement = document.createElement("div");
var borderColor = getRandomColor();
newElement.style.outline = `2px dashed ${borderColor}`;
newElement.style.position = "fixed";
newElement.style.left = bbox.left + "px";
newElement.style.top = bbox.top + "px";
newElement.style.width = bbox.width + "px";
newElement.style.height = bbox.height + "px";
newElement.style.pointerEvents = "none";
newElement.style.boxSizing = "border-box";
newElement.style.zIndex = 2147483647;
// newElement.style.background = `${borderColor}80`;
// Add floating label at the corner
var label = document.createElement("span");
label.textContent = index;
label.style.position = "absolute";
label.style.top = "-19px";
label.style.left = "0px";
label.style.background = borderColor;
label.style.color = "white";
label.style.padding = "2px 4px";
label.style.fontSize = "12px";
label.style.borderRadius = "2px";
newElement.appendChild(label);
document.body.appendChild(newElement);
labels.push(newElement);
// item.element.setAttribute("-ai-label", label.textContent);
});
})
ipcRenderer.send('label-data', JSON.stringify(items.map(item => {
return {
x: (item.rects[0].left + item.rects[0].right) / 2,
y: (item.rects[0].top + item.rects[0].bottom) / 2,
bboxs: item.rects.map(({left, top, width, height}) => [left, top, width, height])
}
})));
}