-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
9441c47
commit cfe2435
Showing
7 changed files
with
245 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{ | ||
"postman.settings.dotenv-detection-notification-visibility": false | ||
} |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,11 +1,12 @@ | ||
{ | ||
"dependencies": { | ||
"dotenv": "^16.4.7", | ||
"playwright": "^1.49.1" | ||
"playwright": "^1.50.0", | ||
"typescript": "^5.7.3" | ||
}, | ||
"devDependencies": { | ||
"@playwright/test": "^1.50.0", | ||
"@types/node": "^22.10.7" | ||
"@types/node": "^22.12.0" | ||
}, | ||
"scripts": {} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
[ | ||
{ | ||
"price": "$29.99", | ||
"imageAlt": "", | ||
"url": "https://www.saucedemo.com/" | ||
}, | ||
{ | ||
"price": "$9.99", | ||
"imageAlt": "", | ||
"url": "https://www.saucedemo.com/" | ||
}, | ||
{ | ||
"price": "$15.99", | ||
"imageAlt": "", | ||
"url": "https://www.saucedemo.com/" | ||
}, | ||
{ | ||
"price": "$49.99", | ||
"imageAlt": "", | ||
"url": "https://www.saucedemo.com/" | ||
}, | ||
{ | ||
"price": "$7.99", | ||
"imageAlt": "", | ||
"url": "https://www.saucedemo.com/" | ||
}, | ||
{ | ||
"price": "$15.99", | ||
"imageAlt": "", | ||
"url": "https://www.saucedemo.com/" | ||
} | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,115 @@ | ||
// scrapedata.ts | ||
const { chromium } = require('playwright'); | ||
const fs = require('fs').promises; | ||
const path = require('path'); | ||
|
||
interface ProductData { | ||
price: string; | ||
imageAlt: string; | ||
url: string; | ||
} | ||
|
||
async function scrapeProductPage(page: any, url: string): Promise<ProductData> { | ||
console.log(`Scraping: ${url}`); | ||
await page.goto(url, { waitUntil: 'domcontentloaded' }); | ||
|
||
const productData: ProductData = { | ||
price: '', | ||
imageAlt: '', | ||
url: url | ||
}; | ||
|
||
try { | ||
// Wait for and extract price | ||
const priceSelector = '.price'; | ||
await page.waitForSelector(priceSelector, { timeout: 5000 }); | ||
const priceElement = await page.$(priceSelector); | ||
if (priceElement) { | ||
productData.price = await priceElement.textContent() || ''; | ||
productData.price = productData.price.trim(); | ||
} | ||
|
||
// Wait for and extract image alt | ||
const imageSelector = '.inventory_item_img'; | ||
await page.waitForSelector(imageSelector, { timeout: 5000 }); | ||
const imageElement = await page.$(imageSelector); | ||
if (imageElement) { | ||
productData.imageAlt = await imageElement.getAttribute('alt') || ''; | ||
} | ||
|
||
console.log('Scraped data:', productData); | ||
} catch (error) { | ||
console.error(`Error scraping ${url}:`, error); | ||
} | ||
|
||
return productData; | ||
} | ||
|
||
async function scrapeMultipleProducts(): Promise<ProductData[]> { | ||
console.log('Launching browser...'); | ||
const browser = await chromium.launch({ | ||
headless: false // Set to false to see the browser in action | ||
}); | ||
|
||
const context = await browser.newContext({ | ||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' | ||
}); | ||
|
||
const page = await context.newPage(); | ||
const results: ProductData[] = []; | ||
|
||
try { | ||
// Go to the demo site | ||
const url = 'https://meetanshi.com/magento-2-extensions.html'; | ||
console.log('Navigating to product listing page...'); | ||
await page.goto(url); | ||
|
||
// Login | ||
console.log('Logging in...'); | ||
|
||
|
||
// Wait for products page to load | ||
await page.waitForSelector('.inventory_item', { timeout: 5000 }); | ||
|
||
// Get all product elements | ||
const productElements = await page.$$('.inventory_item'); | ||
console.log(`Found ${productElements.length} products`); | ||
|
||
for (const element of productElements) { | ||
const price = await element.$eval('.inventory_item_price', (el: any) => el.textContent || ''); | ||
const imageAlt = await element.$eval('.inventory_item_img', (el: any) => el.getAttribute('alt') || ''); | ||
|
||
results.push({ | ||
price: price.trim(), | ||
imageAlt: imageAlt, | ||
url: url | ||
}); | ||
} | ||
|
||
// Save results to JSON file | ||
await fs.writeFile( | ||
'product_data.json', | ||
JSON.stringify(results, null, 2) | ||
); | ||
|
||
console.log('Results saved to product_data.json'); | ||
|
||
} catch (error) { | ||
console.error('Scraping failed:', error); | ||
} finally { | ||
console.log('Closing browser...'); | ||
await browser.close(); | ||
} | ||
|
||
return results; | ||
} | ||
|
||
// Execute the scraper | ||
scrapeMultipleProducts() | ||
.then(results => { | ||
console.log('Scraping completed!'); | ||
console.log('Total products scraped:', results.length); | ||
}) | ||
.catch(error => { | ||
console.error('Script failed:', error); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
{ | ||
"compilerOptions": { | ||
"target": "ES2018", | ||
"module": "CommonJS", | ||
"moduleResolution": "node", | ||
"esModuleInterop": true, | ||
"strict": true, | ||
"skipLibCheck": true, | ||
"forceConsistentCasingInFileNames": true, | ||
"outDir": "./dist", | ||
"rootDir": "./", | ||
"allowJs": true | ||
}, | ||
"include": ["*.ts"], | ||
"exclude": ["node_modules"] | ||
} |