Skip to content

Commit 07f4221

Browse files
RohitR311RohitR311
committed
feat: resolve merge conflicts for recordings table
2 parents 044001d + 651ab19 commit 07f4221

File tree

96 files changed

+2508
-1541
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

96 files changed

+2508
-1541
lines changed

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web
1515

1616

1717
<p align="center">
18+
<a href="https://docs.maxun.dev/?ref=ghread"><b>Documentation</b></a> |
1819
<a href="https://www.maxun.dev/?ref=ghread"><b>Website</b></a> |
1920
<a href="https://discord.gg/5GbPjBUkws"><b>Discord</b></a> |
2021
<a href="https://x.com/maxun_io?ref=ghread"><b>Twitter</b></a> |

‎docker-compose.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ services:
4343
#build:
4444
#context: .
4545
#dockerfile: server/Dockerfile
46-
image: getmaxun/maxun-backend:v0.0.9
46+
image: getmaxun/maxun-backend:v0.0.10
4747
ports:
4848
- "${BACKEND_PORT:-8080}:${BACKEND_PORT:-8080}"
4949
env_file: .env
@@ -70,7 +70,7 @@ services:
7070
#build:
7171
#context: .
7272
#dockerfile: Dockerfile
73-
image: getmaxun/maxun-frontend:v0.0.5
73+
image: getmaxun/maxun-frontend:v0.0.7
7474
ports:
7575
- "${FRONTEND_PORT:-5173}:${FRONTEND_PORT:-5173}"
7676
env_file: .env

‎maxun-core/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "maxun-core",
3-
"version": "0.0.7",
3+
"version": "0.0.8",
44
"description": "Core package for Maxun, responsible for data extraction",
55
"main": "build/index.js",
66
"typings": "build/index.d.ts",

‎maxun-core/src/browserSide/scraper.js

Lines changed: 68 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -207,69 +207,82 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
207207
function findAllElements(config) {
208208
// Regular DOM query if no special delimiters
209209
if (!config.selector.includes('>>') && !config.selector.includes(':>>')) {
210-
return Array.from(document.querySelectorAll(config.selector));
210+
return Array.from(document.querySelectorAll(config.selector));
211211
}
212212

213-
// Split by both types of delimiters
214-
const parts = config.selector.split(/(?:>>|:>>)/).map(s => s.trim());
215-
const delimiters = config.selector.match(/(?:>>|:>>)/g) || [];
216-
let currentElements = [document];
217-
218-
for (let i = 0; i < parts.length; i++) {
219-
const part = parts[i];
220-
const nextElements = [];
221-
const isLast = i === parts.length - 1;
222-
const delimiter = delimiters[i] || '';
223-
const isIframeTraversal = delimiter === ':>>';
224-
225-
for (const element of currentElements) {
226-
try {
227-
let targets;
228-
229-
if (i === 0) {
230-
// First selector is queried from main document
231-
targets = Array.from(element.querySelectorAll(part))
232-
.filter(el => {
233-
if (isLast) return true;
234-
// For iframe traversal, only include iframes
235-
if (isIframeTraversal) return el.tagName === 'IFRAME';
236-
// For shadow DOM traversal, only include elements with shadow root
237-
return el.shadowRoot && el.shadowRoot.mode === 'open';
238-
});
239-
} else {
240-
if (isIframeTraversal) {
241-
// Handle iframe traversal
242-
const iframeDocument = element.contentDocument || element.contentWindow?.document;
243-
if (!iframeDocument) continue;
244-
245-
targets = Array.from(iframeDocument.querySelectorAll(part));
246-
if (!isLast) {
247-
targets = targets.filter(el => el.tagName === 'IFRAME');
213+
// First handle iframe traversal if present
214+
if (config.selector.includes(':>>')) {
215+
const parts = config.selector.split(':>>').map(s => s.trim());
216+
let currentElements = [document];
217+
218+
// Traverse through each part of the selector
219+
for (let i = 0; i < parts.length; i++) {
220+
const part = parts[i];
221+
const nextElements = [];
222+
const isLast = i === parts.length - 1;
223+
224+
for (const element of currentElements) {
225+
try {
226+
// For document or iframe document
227+
const doc = element.contentDocument || element || element.contentWindow?.document;
228+
if (!doc) continue;
229+
230+
// Query elements in current context
231+
const found = Array.from(doc.querySelectorAll(part));
232+
233+
if (isLast) {
234+
// If it's the last part, keep all matching elements
235+
nextElements.push(...found);
236+
} else {
237+
// If not last, only keep iframes for next iteration
238+
const iframes = found.filter(el => el.tagName === 'IFRAME');
239+
nextElements.push(...iframes);
240+
}
241+
} catch (error) {
242+
console.warn('Cannot access iframe content:', error, {
243+
part,
244+
element,
245+
index: i
246+
});
248247
}
249-
} else {
250-
// Handle shadow DOM traversal
251-
const shadowRoot = element.shadowRoot;
252-
if (!shadowRoot || shadowRoot.mode !== 'open') continue;
253-
254-
targets = Array.from(shadowRoot.querySelectorAll(part));
255-
if (!isLast) {
256-
targets = targets.filter(el => el.shadowRoot && el.shadowRoot.mode === 'open');
257-
}
258-
}
259248
}
260-
261-
nextElements.push(...targets);
262-
} catch (error) {
263-
console.warn('Cannot access content:', error);
264-
continue;
265-
}
249+
250+
if (nextElements.length === 0) {
251+
console.warn('No elements found for part:', part, 'at depth:', i);
252+
return [];
253+
}
254+
currentElements = nextElements;
266255
}
256+
257+
return currentElements;
258+
}
259+
260+
// Handle shadow DOM traversal
261+
if (config.selector.includes('>>')) {
262+
const parts = config.selector.split('>>').map(s => s.trim());
263+
let currentElements = [document];
267264

268-
if (nextElements.length === 0) return [];
269-
currentElements = nextElements;
265+
for (const part of parts) {
266+
const nextElements = [];
267+
for (const element of currentElements) {
268+
// Try regular DOM first
269+
const found = Array.from(element.querySelectorAll(part));
270+
271+
// Then check shadow roots
272+
for (const foundEl of found) {
273+
if (foundEl.shadowRoot) {
274+
nextElements.push(foundEl.shadowRoot);
275+
} else {
276+
nextElements.push(foundEl);
277+
}
278+
}
279+
}
280+
currentElements = nextElements;
281+
}
282+
return currentElements.filter(el => !(el instanceof ShadowRoot));
270283
}
271284

272-
return currentElements;
285+
return [];
273286
}
274287

275288
// Modified to handle iframe context for URL resolution

‎package.json

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "maxun",
3-
"version": "0.0.5",
3+
"version": "0.0.6",
44
"author": "Maxun",
55
"license": "AGPL-3.0-or-later",
66
"dependencies": {
@@ -44,9 +44,10 @@
4444
"joi": "^17.6.0",
4545
"jsonwebtoken": "^9.0.2",
4646
"jwt-decode": "^4.0.0",
47+
"lodash": "^4.17.21",
4748
"loglevel": "^1.8.0",
4849
"loglevel-plugin-remote": "^0.6.8",
49-
"maxun-core": "^0.0.7",
50+
"maxun-core": "^0.0.8",
5051
"minio": "^8.0.1",
5152
"moment-timezone": "^0.5.45",
5253
"node-cron": "^3.0.3",
@@ -66,6 +67,7 @@
6667
"react-transition-group": "^4.4.2",
6768
"sequelize": "^6.37.3",
6869
"sequelize-typescript": "^2.1.6",
70+
"sharp": "^0.33.5",
6971
"socket.io": "^4.4.1",
7072
"socket.io-client": "^4.4.1",
7173
"styled-components": "^5.3.3",
@@ -97,6 +99,7 @@
9799
"@types/cookie-parser": "^1.4.7",
98100
"@types/express": "^4.17.13",
99101
"@types/js-cookie": "^3.0.6",
102+
"@types/lodash": "^4.17.14",
100103
"@types/loglevel": "^1.6.3",
101104
"@types/node": "22.7.9",
102105
"@types/node-cron": "^3.0.11",

‎perf/performance.ts

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
// Frontend Performance Monitoring
2+
export class FrontendPerformanceMonitor {
3+
private metrics: {
4+
fps: number[];
5+
memoryUsage: MemoryInfo[];
6+
renderTime: number[];
7+
eventLatency: number[];
8+
};
9+
private lastFrameTime: number;
10+
private frameCount: number;
11+
12+
constructor() {
13+
this.metrics = {
14+
fps: [],
15+
memoryUsage: [],
16+
renderTime: [],
17+
eventLatency: [],
18+
};
19+
this.lastFrameTime = performance.now();
20+
this.frameCount = 0;
21+
22+
// Start monitoring
23+
this.startMonitoring();
24+
}
25+
26+
private startMonitoring(): void {
27+
// Monitor FPS
28+
const measureFPS = () => {
29+
const currentTime = performance.now();
30+
const elapsed = currentTime - this.lastFrameTime;
31+
this.frameCount++;
32+
33+
if (elapsed >= 1000) { // Calculate FPS every second
34+
const fps = Math.round((this.frameCount * 1000) / elapsed);
35+
this.metrics.fps.push(fps);
36+
this.frameCount = 0;
37+
this.lastFrameTime = currentTime;
38+
}
39+
requestAnimationFrame(measureFPS);
40+
};
41+
requestAnimationFrame(measureFPS);
42+
43+
// Monitor Memory Usage
44+
if (window.performance && (performance as any).memory) {
45+
setInterval(() => {
46+
const memory = (performance as any).memory;
47+
this.metrics.memoryUsage.push({
48+
usedJSHeapSize: memory.usedJSHeapSize,
49+
totalJSHeapSize: memory.totalJSHeapSize,
50+
timestamp: Date.now()
51+
});
52+
}, 1000);
53+
}
54+
}
55+
56+
// Monitor Canvas Render Time
57+
public measureRenderTime(renderFunction: () => void): void {
58+
const startTime = performance.now();
59+
renderFunction();
60+
const endTime = performance.now();
61+
this.metrics.renderTime.push(endTime - startTime);
62+
}
63+
64+
// Monitor Event Latency
65+
public measureEventLatency(event: MouseEvent | KeyboardEvent): void {
66+
const latency = performance.now() - event.timeStamp;
67+
this.metrics.eventLatency.push(latency);
68+
}
69+
70+
// Get Performance Report
71+
public getPerformanceReport(): PerformanceReport {
72+
return {
73+
averageFPS: this.calculateAverage(this.metrics.fps),
74+
averageRenderTime: this.calculateAverage(this.metrics.renderTime),
75+
averageEventLatency: this.calculateAverage(this.metrics.eventLatency),
76+
memoryTrend: this.getMemoryTrend(),
77+
lastMemoryUsage: this.metrics.memoryUsage[this.metrics.memoryUsage.length - 1]
78+
};
79+
}
80+
81+
private calculateAverage(array: number[]): number {
82+
return array.length ? array.reduce((a, b) => a + b) / array.length : 0;
83+
}
84+
85+
private getMemoryTrend(): MemoryTrend {
86+
if (this.metrics.memoryUsage.length < 2) return 'stable';
87+
const latest = this.metrics.memoryUsage[this.metrics.memoryUsage.length - 1];
88+
const previous = this.metrics.memoryUsage[this.metrics.memoryUsage.length - 2];
89+
const change = latest.usedJSHeapSize - previous.usedJSHeapSize;
90+
if (change > 1000000) return 'increasing'; // 1MB threshold
91+
if (change < -1000000) return 'decreasing';
92+
return 'stable';
93+
}
94+
}
95+
96+
// Backend Performance Monitoring
97+
export class BackendPerformanceMonitor {
98+
private metrics: {
99+
screenshotTimes: number[];
100+
emitTimes: number[];
101+
memoryUsage: NodeJS.MemoryUsage[];
102+
};
103+
104+
constructor() {
105+
this.metrics = {
106+
screenshotTimes: [],
107+
emitTimes: [],
108+
memoryUsage: []
109+
};
110+
this.startMonitoring();
111+
}
112+
113+
private startMonitoring(): void {
114+
// Monitor Memory Usage
115+
setInterval(() => {
116+
this.metrics.memoryUsage.push(process.memoryUsage());
117+
}, 1000);
118+
}
119+
120+
public async measureScreenshotPerformance(
121+
makeScreenshot: () => Promise<void>
122+
): Promise<void> {
123+
const startTime = process.hrtime();
124+
await makeScreenshot();
125+
const [seconds, nanoseconds] = process.hrtime(startTime);
126+
this.metrics.screenshotTimes.push(seconds * 1000 + nanoseconds / 1000000);
127+
}
128+
129+
public measureEmitPerformance(emitFunction: () => void): void {
130+
const startTime = process.hrtime();
131+
emitFunction();
132+
const [seconds, nanoseconds] = process.hrtime(startTime);
133+
this.metrics.emitTimes.push(seconds * 1000 + nanoseconds / 1000000);
134+
}
135+
136+
public getPerformanceReport(): BackendPerformanceReport {
137+
return {
138+
averageScreenshotTime: this.calculateAverage(this.metrics.screenshotTimes),
139+
averageEmitTime: this.calculateAverage(this.metrics.emitTimes),
140+
currentMemoryUsage: this.metrics.memoryUsage[this.metrics.memoryUsage.length - 1],
141+
memoryTrend: this.getMemoryTrend()
142+
};
143+
}
144+
145+
private calculateAverage(array: number[]): number {
146+
return array.length ? array.reduce((a, b) => a + b) / array.length : 0;
147+
}
148+
149+
private getMemoryTrend(): MemoryTrend {
150+
if (this.metrics.memoryUsage.length < 2) return 'stable';
151+
const latest = this.metrics.memoryUsage[this.metrics.memoryUsage.length - 1];
152+
const previous = this.metrics.memoryUsage[this.metrics.memoryUsage.length - 2];
153+
const change = latest.heapUsed - previous.heapUsed;
154+
if (change > 1000000) return 'increasing';
155+
if (change < -1000000) return 'decreasing';
156+
return 'stable';
157+
}
158+
}
159+
160+
interface MemoryInfo {
161+
usedJSHeapSize: number;
162+
totalJSHeapSize: number;
163+
timestamp: number;
164+
}
165+
166+
type MemoryTrend = 'increasing' | 'decreasing' | 'stable';
167+
168+
interface PerformanceReport {
169+
averageFPS: number;
170+
averageRenderTime: number;
171+
averageEventLatency: number;
172+
memoryTrend: MemoryTrend;
173+
lastMemoryUsage: MemoryInfo;
174+
}
175+
176+
interface BackendPerformanceReport {
177+
averageScreenshotTime: number;
178+
averageEmitTime: number;
179+
currentMemoryUsage: NodeJS.MemoryUsage;
180+
memoryTrend: MemoryTrend;
181+
}

0 commit comments

Comments
 (0)