From 984cd33be39f0181c0006ab5eb25f1e2b352bf48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=BCrker?= <63150613+turkwr@users.noreply.github.com> Date: Mon, 15 Jul 2024 11:50:08 +0300 Subject: [PATCH] chore: Update package.json and add format script --- .github/workflows/node.js.yml | 32 +++--- .gitignore | 1 + main.yaml | 32 +++--- package.json | 5 +- readme.md | 11 ++- webfreeze.js | 178 +++++++++++++++++----------------- 6 files changed, 133 insertions(+), 126 deletions(-) create mode 100644 .gitignore diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 44bf6b3..4aed4eb 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -2,9 +2,9 @@ name: WebFreeze CI on: push: - branches: [ "main" ] + branches: ["main"] pull_request: - branches: [ "main" ] + branches: ["main"] jobs: build: @@ -15,17 +15,17 @@ jobs: node-version: [14.x, 16.x, 18.x] steps: - - uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: 'npm' - - name: Install dependencies - run: npm ci - - name: Install Puppeteer dependencies - run: sudo apt-get install -y libgbm-dev - - name: Run linter - run: npm run lint || echo "Linting failed but continuing..." - - name: Run tests - run: npm test || echo "No tests specified" + - uses: actions/checkout@v4 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: "npm" + - name: Install dependencies + run: npm ci + - name: Install Puppeteer dependencies + run: sudo apt-get install -y libgbm-dev + - name: Run linter + run: npm run lint || echo "Linting failed but continuing..." + - name: Run tests + run: npm test || echo "No tests specified" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b512c09 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/main.yaml b/main.yaml index ef4bee2..d45f6c7 100644 --- a/main.yaml +++ b/main.yaml @@ -2,9 +2,9 @@ name: WebFreeze CI on: push: - branches: [ "main" ] + branches: ["main"] pull_request: - branches: [ "main" ] + branches: ["main"] jobs: build: @@ -15,17 +15,17 @@ jobs: node-version: [18.x] steps: - - uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: 'npm' - - name: Install dependencies - run: npm ci - - name: Install Puppeteer dependencies - run: sudo apt-get install -y libgbm-dev - - name: Run linter - run: npm run lint || echo "Linting failed but continuing..." - - name: Run tests - run: npm test || echo "No tests specified" + - uses: actions/checkout@v4 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: "npm" + - name: Install dependencies + run: npm ci + - name: Install Puppeteer dependencies + run: sudo apt-get install -y libgbm-dev + - name: Run linter + run: npm run lint || echo "Linting failed but continuing..." + - name: Run tests + run: npm test || echo "No tests specified" diff --git a/package.json b/package.json index 10e35c7..082b010 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "scripts": { "start": "node webfreeze.js", "lint": "eslint .", - "test": "echo \"No tests specified\" && exit 0" + "test": "echo \"No tests specified\" && exit 0", + "format": "prettier --write ." }, "keywords": [ "web", @@ -35,4 +36,4 @@ "url": "https://github.com/pudochu/webfreeze/issues" }, "homepage": "https://github.com/pudochu/webfreeze#readme" -} \ No newline at end of file +} diff --git a/readme.md b/readme.md index d59453d..a144d90 100644 --- a/readme.md +++ b/readme.md @@ -21,18 +21,20 @@ WebFreeze is a powerful tool that allows you to capture and serve static version cd webfreeze ``` 3. Install dependencies: - ``` - npm install - ``` + ``` + npm install + ``` ## 📘 Usage 1. Open `webfreeze.js` and set your configuration: + - `SITE_URL`: The URL of the website you want to capture - `SAVE_DIRECTORY`: The directory where files will be saved - `isDownloadMode`: Set to `true` for initial capture, `false` for serving local files 2. Run the script: + ``` npm start ``` @@ -43,7 +45,7 @@ WebFreeze is a powerful tool that allows you to capture and serve static version ## 🔧 Configuration -- `isDownloadMode`: +- `isDownloadMode`: - `true`: Downloads and saves web resources - `false`: Serves local files and blocks new requests @@ -56,4 +58,5 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file Contributions, issues, and feature requests are welcome! Feel free to check [issues page](https://github.com/pudochu/webfreeze/issues). --- + Project Link: [https://github.com/pudochu/webfreeze](https://github.com/pudochu/webfreeze) diff --git a/webfreeze.js b/webfreeze.js index db9a2d9..3c879e2 100644 --- a/webfreeze.js +++ b/webfreeze.js @@ -1,20 +1,20 @@ /** * WebFreeze: A tool to capture and serve static versions of dynamic web pages - * + * * This script uses Puppeteer to capture a web page and its resources, saving them locally. * It can then serve these local files, effectively "freezing" the web page at a specific point in time. */ -const puppeteer = require('puppeteer'); -const fs = require('fs'); -const fsPromises = require('fs').promises; -const path = require('path'); -const https = require('https'); -const URL = require('url'); +const puppeteer = require("puppeteer"); +const fs = require("fs"); +const fsPromises = require("fs").promises; +const path = require("path"); +const https = require("https"); +const URL = require("url"); // Configuration -const SITE_URL = 'https://example.com'; -const SAVE_DIRECTORY = './files'; +const SITE_URL = "https://example.com"; +const SAVE_DIRECTORY = "./files"; const isDownloadMode = false; // true: download and save, false: use local files or block /** @@ -24,21 +24,23 @@ const isDownloadMode = false; // true: download and save, false: use local files * @returns {Promise} */ async function downloadFile(url, filePath) { - return new Promise((resolve, reject) => { - https.get(url, (response) => { - if (response.statusCode === 200) { - const fileStream = fs.createWriteStream(filePath); - response.pipe(fileStream); - fileStream.on('finish', () => { - fileStream.close(); - console.log(`Downloaded: ${url}`); - resolve(); - }); - } else { - reject(`Failed to download ${url}: ${response.statusCode}`); - } - }).on('error', reject); - }); + return new Promise((resolve, reject) => { + https + .get(url, (response) => { + if (response.statusCode === 200) { + const fileStream = fs.createWriteStream(filePath); + response.pipe(fileStream); + fileStream.on("finish", () => { + fileStream.close(); + console.log(`Downloaded: ${url}`); + resolve(); + }); + } else { + reject(`Failed to download ${url}: ${response.statusCode}`); + } + }) + .on("error", reject); + }); } /** @@ -47,15 +49,15 @@ async function downloadFile(url, filePath) { * @returns {string} The local file path */ function getFilePath(url) { - const parsedUrl = new URL.URL(url); - let filePath = parsedUrl.pathname; - - // Handle paths starting with _next - if (filePath.startsWith('/_next')) { - filePath = filePath.replace(/^\//, ''); - } - - return path.join(SAVE_DIRECTORY, filePath); + const parsedUrl = new URL.URL(url); + let filePath = parsedUrl.pathname; + + // Handle paths starting with _next + if (filePath.startsWith("/_next")) { + filePath = filePath.replace(/^\//, ""); + } + + return path.join(SAVE_DIRECTORY, filePath); } /** @@ -63,74 +65,74 @@ function getFilePath(url) { * @param {puppeteer.Page} page - The Puppeteer page object */ async function interceptRequests(page) { - await page.setRequestInterception(true); - page.on('request', async (request) => { - const url = request.url(); - const resourceType = request.resourceType(); - - if (['stylesheet', 'script', 'image'].includes(resourceType)) { - const filePath = getFilePath(url); + await page.setRequestInterception(true); + page.on("request", async (request) => { + const url = request.url(); + const resourceType = request.resourceType(); - if (isDownloadMode) { - // Download mode: Download and save files - try { - await fsPromises.access(filePath); - request.continue(); - } catch (error) { - try { - const directoryPath = path.dirname(filePath); - await fsPromises.mkdir(directoryPath, { recursive: true }); - await downloadFile(url, filePath); - request.continue(); - } catch (downloadError) { - console.error(`Failed to download ${url}: ${downloadError}`); - request.continue(); - } - } - } else { - // Block mode: Use local files or block requests - try { - await fsPromises.access(filePath); - const content = await fsPromises.readFile(filePath); - console.log(`Serving: ${filePath}`); - request.respond({ - status: 200, - contentType: request.headers()['content-type'], - body: content - }); - } catch (error) { - console.log(`Blocked: ${filePath}`); - request.abort(); - } - } - } else { + if (["stylesheet", "script", "image"].includes(resourceType)) { + const filePath = getFilePath(url); + + if (isDownloadMode) { + // Download mode: Download and save files + try { + await fsPromises.access(filePath); + request.continue(); + } catch (error) { + try { + const directoryPath = path.dirname(filePath); + await fsPromises.mkdir(directoryPath, { recursive: true }); + await downloadFile(url, filePath); + request.continue(); + } catch (downloadError) { + console.error(`Failed to download ${url}: ${downloadError}`); request.continue(); + } } - }); + } else { + // Block mode: Use local files or block requests + try { + await fsPromises.access(filePath); + const content = await fsPromises.readFile(filePath); + console.log(`Serving: ${filePath}`); + request.respond({ + status: 200, + contentType: request.headers()["content-type"], + body: content, + }); + } catch (error) { + console.log(`Blocked: ${filePath}`); + request.abort(); + } + } + } else { + request.continue(); + } + }); } /** * Main function to run the script */ async function main() { - const browser = await puppeteer.launch({ headless: false }); - const page = await browser.newPage(); + const browser = await puppeteer.launch({ headless: false }); + const page = await browser.newPage(); - await interceptRequests(page); + await interceptRequests(page); - console.log(`Navigating to ${SITE_URL}`); - await page.goto(SITE_URL, { waitUntil: 'networkidle0' }); + console.log(`Navigating to ${SITE_URL}`); + await page.goto(SITE_URL, { waitUntil: "networkidle0" }); - // Save the main HTML - const content = await page.content(); - const htmlPath = path.join(SAVE_DIRECTORY, 'index.html'); - await fsPromises.writeFile(htmlPath, content); - console.log(`Saved main HTML to ${htmlPath}`); + // Save the main HTML + const content = await page.content(); + const htmlPath = path.join(SAVE_DIRECTORY, "index.html"); + await fsPromises.writeFile(htmlPath, content); + console.log(`Saved main HTML to ${htmlPath}`); - // Keep the browser open for inspection - console.log('Browser will remain open. Close it manually when done.'); - // Uncomment the next line to close the browser automatically - // await browser.close(); + // Keep the browser open for inspection + console.log("Browser will remain open. Close it manually when done."); + // Uncomment the next line to close the browser automatically + // await browser.close(); } main().catch(console.error);