giving up

scripts/update.sh

@@ -1,28 +1,29 @@
 #!/usr/bin/env bash
 
-# Use this script to generate all the blocklists after `domains.txt` is updated.
+# Use this script to generate all the blocklists after `.txt` files in the `source` folder are updated.
 # e.g.
 # ./scripts/update.sh
 
 # Remove `www.` from sources.
 find ./sources -type f -name "*.txt" -exec sed -i 's/^www\.//i' {} \;
 
-# Remove duplicates from each source file.
+# Remove duplicates from each source file. (TO DO: keep "duplicate" line jumps and comments)
 find ./sources -type f -name "*.txt" -exec bash -c '
-    awk '/^#/ {print; next} {printf "%s%s", sep, $0; sep = RS} END {print ""}' "$0" > "$0.temp";
-    mv "$0.temp" "$0";
+    cat -n "$0" | sort -uk2 | sort -nk1 | cut -f2- > "$0_temp.txt";
+    mv "$0_temp.txt" "$0";
 ' {} \;
-# should ignore line jumps and comments
 
-# Combine all sources into `domains.txt`.
-find ./sources -type f -name "*.txt" -exec cat {} \; > sources.txt
-sed -i '/^#/d; s/#.*//; /^ *$/d' sources.txt
+# Combine all sources into a domains list.
+find ./sources -type f -name "*.txt" -exec cat {} \; > domains.txt
 
-# Remove duplicates from `domains.txt`.
+# Remove comments, inline comments, spaces and empty lines from the domains list.
+sed -i '/^#/d; s/#.*//; s/ //g; /^ *$/d' domains.txt
+
+# Remove duplicates from the domains list.
 cat -n domains.txt | sort -uk2 | sort -nk1 | cut -f2- > domains_temp.txt
 mv domains_temp.txt domains.txt
 
-# Generate blocklists from `domains.txt`.
+# Generate blocklists from the domain list.
 python scripts/domains_to_hosts.py > hosts.txt
 python scripts/domains_to_hosts_ipv6.py > hosts.txt.ipv6
 python scripts/domains_to_dnsmasq.py > dnsmasq_hosts.txt