diff --git a/scripts/create-everything-list.js b/scripts/create-everything-list.js index d2409af2..c306e11b 100644 --- a/scripts/create-everything-list.js +++ b/scripts/create-everything-list.js @@ -1,57 +1,45 @@ const fs = require("node:fs").promises; const path = require("node:path"); +// Lists to include in the "everything" list const listsToIncludeInEverythingList = [ - "abuse", - "ads", - "crypto", - "drugs", - "facebook", - "fraud", - "gambling", - "malware", - "phishing", - "piracy", - "porn", - "ransomware", - "redirect", - "scam", - "tiktok", - "torrent", - "tracking", - - // The following lists are in beta and therefore not included in the everything list: - - // "smart-tv", - // "basic", - // "whatsapp", - // "vaping" + "abuse", "ads", "crypto", "drugs", "facebook", "fraud", + "gambling", "malware", "phishing", "piracy", "porn", + "ransomware", "redirect", "scam", "tiktok", "torrent", "tracking", + // Beta lists (excluded from "everything" list) + // "smart-tv", "basic", "whatsapp", "vaping" ]; (async () => { try { - const files = (await fs.readdir(path.join(__dirname, ".."))).filter( - (file) => - file.endsWith(".txt") && - listsToIncludeInEverythingList.some((val) => file.startsWith(val)), + const baseDir = path.join(__dirname, ".."); + + // Filter and collect relevant .txt files + const files = (await fs.readdir(baseDir)).filter((file) => + file.endsWith(".txt") && + listsToIncludeInEverythingList.some((prefix) => file.startsWith(prefix)) ); + + // Use a Set to store unique domains const domains = new Set(); + // Process each file to extract domains await Promise.all( files.map(async (file) => { - const fileContents = await fs.readFile( - path.join(__dirname, "..", file), - "utf8", - ); - for (const line of fileContents.split("\n")) { + const filePath = path.join(baseDir, file); + const fileContents = await fs.readFile(filePath, "utf8"); + + // Extract domains starting with "0.0.0.0" + fileContents.split("\n").forEach((line) => { if (line.startsWith("0.0.0.0 ")) { - domains.add(line.replace("0.0.0.0 ", "")); + domains.add(line.slice(8)); // Add the domain after "0.0.0.0 " } - } - }), + }); + }) ); - let everythingListContent = `# ------------------------------------[UPDATE]-------------------------------------- + // Generate content for the "everything" list + const header = `# ------------------------------------[UPDATE]-------------------------------------- # Title: The Block List Project - Everything List # Expires: 1 day # Homepage: https://blocklistproject.github.io/Lists/ @@ -69,15 +57,16 @@ const listsToIncludeInEverythingList = [ # ------------------------------------[FILTERS]------------------------------------- `; - for (const val of domains) { - everythingListContent += `0.0.0.0 ${val}\n`; - } + // Concatenate domains into a single list with the header + const everythingListContent = `${header}${Array.from(domains) + .map((domain) => `0.0.0.0 ${domain}`) + .join("\n")}\n`; - await fs.writeFile( - path.join(__dirname, "..", "everything.txt"), - everythingListContent, - "utf8", - ); + // Write the final "everything" list file + const outputFilePath = path.join(baseDir, "everything.txt"); + await fs.writeFile(outputFilePath, everythingListContent, "utf8"); + + console.log("Everything list generated successfully."); } catch (error) { console.error("Error processing files:", error); }