EmailVerification.module

<?php

/**
 * ProcessWire Module Email Verification
 *
 * E-Mail Verification and Domain Validation via API.
 * checks an email addresses against blacklist and availability of mailhost.
 * Blacklisted domains are simply stored in a text file. Easy to edit.
 *
 * @author Christoph Thelen aka @kixe 2014/10/05
 * @copyright © 2014 Christoph Thelen
 * @license Licensed under GNU/GPL v3
 * @link https://processwire.com/talk/topic/...
 * @version 2.0.4
 * @since 1.2.0 2018/02/18 - update email check via fiddlemail.com/ mogelmail.de
 * @since 1.2.1 2018/02/18 - update return value add() and addToBlacklist()
 * @since 1.2.2 2018/02/18 - fixed minor bugs
 * @since 1.2.3 2018/03/28 - fixed bug array to string conversion update() added json_encode()
 * @since 1.2.4 2018/03/29 - better mail host validation validHost();
 * @since 1.2.5 2018/05/22 - switch from fiddlemail.com/ mogelmail.de service to trashmail-blacklist.org which is free
 * @since 1.2.6 2018/05/22 - fixed bug
 * @since 2.0.0 2018/05/25 - major update
 * @since 2.0.1 2018/06/08 - protect internal functions, throw exception if domain is both: blacklisted/ whitelisted
 * @since 2.0.2 2018/07/18 - function validHost() - use gethostbyname() only for secondlevel domains
 * @since 2.0.3 2018/07/18 - fixed dot issue checkdnsrr() @see https://secure.php.net/manual/de/function.checkdnsrr.php#119969
 * @since 2.0.4 2018/09/09 - fixed bug: check failed if domain is substring of listed domain e.g. gmail.com -> nobugmail.com
 * 
 * 
 * ProcessWire 2.x/ 3.x 
 * Copyright (C) 2013 by Ryan Cramer 
 * Licensed under GNU/GPL v2, see LICENSE.TXT
 * 
 * http://processwire.com
 *
 */

class EmailVerification extends Wire implements Module {

	public static function getModuleInfo() {

		return array(

			'title' => 'Email Verification', 
			'version' => 204, 
			'summary' => __('E-Mail Verification and Domain Validation via API. Checks an email address against blacklist and availability of mailhost. Blacklisted domains and email addresses are simply stored in a text file. Easy to edit.'),
			'href' => 'https://processwire.com/talk/topic/7826-module-email-verification/',
			'author' => 'kixe',
			'icon' => 'envelope-o',
			'license' => 'GNU-GPLv3',
			'hreflicense' => 'http://www.gnu.org/licenses/gpl-3.0.html'
			);
	}

	protected static $files = array(
		'b' => __DIR__ .'/blacklist.txt',
		'w' => __DIR__ .'/whitelist.txt'
	);

	public function __construct() {
	    // intentionally empty
	}

	/*
	public static function allowedExtraChars() {
		return array(
			'de' => 'àáâãäåāăąæçćĉċčďđèéêëēĕėęěŋðĝğġģĥħìíîïĩīĭįıĵķĸĺļľłñńņňòóôõöøōŏőœŕŗřśŝşšţťŧþùúûüũūŭůűųŵýÿŷźżž',
			'no' => 'áàäčçđéèêŋńñóòôöšŧüžæøå',
			'se' => 'àáäåæçèéêëìíîïðñòóôöøùúüýþćčđěłńŋřśšţŧźžǎǐǒǔǥǧǩǯəʒ', // plus Hebrew
			'dk' => 'äåæéöøü',
			'is' => 'áéýúíóþæöð',
			'it' => 'àâäèéêëìîïòôöùûüæœçÿ',
			'fr' => 'àáâãäåæçèéêëìíîïñòóôõöùúûüýÿœ',
			'nu' => 'àáâäåāæçèéêëēìíîïīðñŋòóôõöøōùúûüūýþÿ',
			'be' => 'àáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿœ',
			'de' => 'àáâãäåāăąæçćĉċčďđèéêëēĕėęěŋðĝğġģĥħìíîïĩīĭįıĵķĸĺļľłñńņňòóôõöøōŏőœŕŗřśŝşšţťŧþùúûüũūŭůűųŵýÿŷźżž',
			'biz' => 'àáäåæéêíðñòóôöøúüýþ'
		);
	}
	*/

	public function init() {
		// cyclical cleanup of white- and blacklist files
		foreach(self::$files as $file) {
			if (file_exists($file) && time() - filemtime($file) > 604800) {
				if ($message = $this->clean($file)) {
					$message = 'Updated file ' . $file . json_encode($message);
					$this->message($message, Notice::logOnly);
				}
			}
		}
	}

	/**
	 * get array of all TLDs by IANA punycoded, uppercase, local text file updated monthly (30 days)
	 *
	 * @param $cycle int cycle time to update the tld file
	 * @return bool/ array
	 *
	 */
	public function getTLDs($cycle = 2592000) {
		$source = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt";
		$file = __DIR__.'/tlds-alpha-by-domain.txt';
		if (!file_exists($file) || time() - filemtime($file) > $cycle) {
			if (!$this->checkRemoteFileAccess($source)) return false;
			$update = file_get_contents($source);
			if (file_put_contents($file, $update, LOCK_EX) === false) {
				$this->error("Update of $file failed", Notice::logOnly);;
				return false;
			}
		}
		$tld = array_filter(file($file,FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES), function($e) { return (substr($e,0,1) == '#')? false :true; });
		return $tld;
	}

	/**
	 * check if this is a valid top level domain
	 *
	 * @param $tld string top level domain
	 * @return bool valid TLD
	 *
	 */
	public function validTLD($tld) {
		return in_array(strtoupper($this->idn_to_ascii($tld)), $this->getTLDs())? true : false;
	}

	/**
	 * convert domain name to IDNA ASCII form
	 * PHP::idn_to_ascii() needs intl extension installed
	 *
	 * @param $value string top level domain
	 * @return string IDNA ASCII form
	 *
	 */
	protected function idn_to_ascii($value) {
		if(function_exists("idn_to_ascii")) {
			// use native php function if available
			return \idn_to_ascii($value, IDNA_NONTRANSITIONAL_TO_ASCII, INTL_IDNA_VARIANT_UTS46);
		} else {
			// otherwise use PW Punycode class
			$pc = new Punycode();
			return $pc->encode($value);
		}
	}

	/**
	 * extract the part of a string after the last @ if exists
	 *
	 * @param $str string (email)
	 * @return null/ string (domain)
	 *
	 */
	public function getDomain($str) {
		$domain = strpos($str,'@')?substr(strrchr($str,'@'),1) : $str;
		if (!$this->validDomainName($domain)) return null;
		return $domain;
	}

	/**
	 * validate a domain name
	 *
	 * @param string (domainname)
	 * @return boolean
	 *
	 */
	public function validDomainName($domainname = '') {		
		if (strlen($domainname) > 255 || strlen($domainname) < 2) return false;
		return ($this->validHostName($this->idn_to_ascii($domainname)))? true : false;
	}

	/**
	 * to verify a domain name you have to convert to punycode before
	 * @param string (hostname) allowed characters A-Z, a-z, 0-9 and hyphen
	 * @return boolean/ string
	 *
	 */
	public function validHostName($hostname = '') {
		if (strlen($hostname) > 255 || strlen($hostname) < 2) return false;
		$matches = array();
		$pattern = '/^((?!-)[a-z0-9-]{1,63}(?<!-)\.)+([a-z]{2,63}|xn--[a-z0-9]{2,63}){1}$/i';
		if (!preg_match($pattern,$hostname,$matches)) return false;
		if (!$this->validTLD($matches[count($matches)-1])) return false;
		return $hostname;
	}

	/**
	 * Determine if given domain or Email is blacklisted, update blacklist file (max. once per week)
	 *
	 * @param $input string (domain/ email address)
	 * @return null/ boolean/ string
	 * @deprecated use check() or info() instead
	 *
	 */
	public function blacklisted($input = null, $mx = false) {
		if (!$input) return null;
		$domain = $this->getDomain($input);
		if ($this->getDomain($input) == null) return null; // invalid domain
		$b = $this->has($domain, self::$files['b']);
		$w = $this->has($domain, self::$files['w']);
		if ($b && $w) throw new WireException("Please manually remove domain $domain from either blacklist.txt or whitelist.txt");
		if ($b) return $domain;
		if ($w) return false;
		$validHost = $this->validHost($domain);
		if ($validHost === false) return null; // missing mail server
		$check = $this->isTrashmail($domain);
		// add to local files if known
		if ($check === true) return $this->blacklist($domain);
		if ($check === false && $this->whitelist($domain)) return false;
		if ($mx === false && $validHost === 'MX') {
			getmxrr($domain, $hosts);
			if (!empty($hosts)) {
				$_mx = null;
				foreach ($hosts as $host) {
					$mx = $this->blacklisted($host, true);
					if ($mx === false) return false;
					else if ($mx === null) continue;
					else $_mx = $mx;
				}
				if ($_mx) {
					$this->blacklist($_mx);
					return $domain;
				}
			}
		}
		return null;	
	}

	/**
	 * Determine if given domain or Email is blacklisted, whitelisted or unknown
	 *
	 * @param $input string (domain/ email address)
	 * @return null/ boolean/ string
	 *
	 */
	public function check($input = null) {}

	/**
	 * get detailed info about email address and domain
	 *
	 * @param $input string (domain/ email address)
	 * @return string
	 *
	 */
	public function info($input = null) {}

	/**
	 * Check DNS records corresponding to a given Internet host name or IP address
	 *
	 * @param $input string (domain/ email address)
	 * @return null/ boolean/ string
	 *
	 */
	public function validHost($input = null) {
		if (!$input) return null;
		$domain = $this->getDomain($input);
		if (!$this->validDomainName($domain)) return false;
		// check if invalid domain is redirected to ISPs own servers (e.g. search)
		$invalidHostName = 'example.thisdomaindoesnotexistatall';
		if (substr_count($domain, '.') > 1) {
			$parts = explode('.', $domain);
			$mainDomain = implode('.',array_slice($parts, -2));
		}
		if (is_array(gethostbynamel($invalidHostName)) && in_array(gethostbyname($mainDomain),gethostbynamel($invalidHostName))) return false;

		// must not point to any CNAME records
		if (checkdnsrr(trim($domain,'.').'.', "CNAME")) return false;
		// must point to either A or AAAA record
		$recordA = checkdnsrr(trim($domain,'.').'.', "A");
		$record4A = checkdnsrr(trim($domain,'.').'.', "AAAA");
		$recordMX = checkdnsrr(trim($domain,'.').'.', "MX");
		if ($recordA === false && $record4A === false && $recordMX === true) {
			getmxrr($domain, $hosts);
			if (!empty($hosts)) {
				foreach ($hosts as $host) {
					$record4A = checkdnsrr(trim($host,'.').'.', "AAAA");
					$recordA = checkdnsrr(trim($host,'.').'.', "A");
					if ($recordA === true || $record4A === true) return "MX";
				}
				$hosts = json_encode($hosts);
				$this->error("Domain [$domain] itself or one related MX RR $hosts must point to either A or AAAA record.");
			}
		}
		if ($recordMX) return "MX";
		if ($recordA) return "A";
		if ($record4A) return "AAAA";
		return false;
	}

	/**
	 * free service to check trashmails
	 * @param $input string (domain/ email address)
	 * @return null/ boolean
	 * true = blacklisted
	 * false = whitelisted
	 * null = unknown
	 *
	 * @see https://trashmail-blacklist.org
	 * 
	 */
	public function isTrashmail($input) {
		$domain = $this->getDomain($input);
		$source = "https://v2.trashmail-blacklist.org/check/json/$domain"; // return json string {"status":"unknown","domain":"domain.org","added":1456095603,"lastchecked":1456095603}

		if (!$this->checkRemoteFileAccess($source)) return null;
		$json = file_get_contents($source);
		$result = json_decode($json);
		if ($result->status == 'blacklisted') return true;
		else if ($result->status == 'whitelisted') return false;
		else return null;
	}

	/**
	 * manually add a single e-mail address or maildomain to blacklist
	 * remove from whitelist
	 * 
	 * @param $input string (domain/ email address)
	 * @return boolean/ string
	 *
	 */
	public function blacklist($input) {
		$domain = $this->getDomain($input);
		if (!$this->validHost($domain)) return false;
		if ($this->add($domain, self::$files['b']) && $this->remove($domain, self::$files['w'])) return $domain;
		return false;
	}

	/**
	 * manually add a single e-mail address or maildomain to whitelist
	 * remove from blacklist
	 * 
	 * @param $input string (domain/ email address)
	 * @return boolean/ string
	 *
	 */
	public function whitelist($input) {
		$domain = $this->getDomain($input);
		if (!$this->validHost($domain)) return false;
		if ($this->add($domain, self::$files['w']) && $this->remove($domain, self::$files['b'])) return $domain;
		return false;
	}

	/**
	 * is domain present in list
	 * 
	 * @param $domain
	 * @param $path file path
	 * @param $remove bool
	 * @return boolean/ array
	 *
	 */
	protected function has($domain, $path, $remove = false) {
		if (!file_exists($path)) return false;
		$array = file($path, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
		if ($array === false) throw new WireException("Failed to open file: $path");
		$flip_array = array_flip($array);
		$has = isset($flip_array[$domain]);
		if (!$remove) return $has;
		if (!$has) return false;
		unset($flip_array[$domain]);
		return array_keys($flip_array);
	}	

	/**
	 * add domain to a list
	 * 
	 * @param $domain
	 * @param $path file path
	 * @return string/ boolean
	 *
	 */
	protected function add($domain, $path) {
		if ($this->has($domain, $path)) return $domain; // nothing to do
		if ($this->save($domain, $path)) return $domain;
		return false;
	}	

	/**
	 * remove domain from a list
	 * 
	 * @param $domain
	 * @param $path file path
	 * @return string/ boolean
	 *
	 */
	protected function remove($domain, $path) {
		$uniques = $this->has($domain, $path, true);
		if (false === $uniques) return $domain; // nothing to do
		$uniques = implode("\n",$uniques);
		if ($this->save($uniques, $path, false)) return $domain;
		return false;
	}

	/**
	 * append a string to a file and save
	 * file will be created if not exist
	 * 
	 * @param $string
	 * @param $path file path
	 * @param $append bool append string to file or truncate/ create file
	 * @return boolean
	 *
	 */
	protected function save($string, $path, $append = true) {
		if (!file_exists($path)) $append = false;
		$nl = $append? "\n" : '';
		$mode = $append? 'a' : 'w'; // append string to file or truncate/ create file
		if ($file = fopen($path, $mode)) {
			flock($file, LOCK_EX);
			fwrite($file, $nl.$string);
			flock($file, LOCK_UN);
			fclose($file);
			return true;
		}
		return false;
	}

	/**
	 * checks accessibility of remote file, expects http status codes: 200 or 302 only
	 * logs error if not accessible
	 *
	 * @param $httpUrl
	 * @return bool
	 *
	 */
	protected function checkRemoteFileAccess($httpUrl) {
		$file_headers = @get_headers($httpUrl);
		if (preg_match('/200|302/',$file_headers[0])) return true;
		$error = "Connection to external datasource failed: '$httpUrl'";
		if ($file_headers) $error .= " ($file_headers[0])";
		$this->error($error, Notice::logOnly);
		return false;
	}

	/**
	 * Check DNS records corresponding to domain name in blacklist file
	 * max 50 entries allowed
	 *
	 * @param $from int array key from where to check
	 * @param $to int array key until where to check (max diff to $from = 50)
	 * @return array/ boolean
	 * @see function validHost()
	 *
	 *
	public function checkBlacklistHosts($from = 0, $to = 10) {
		if ($to - $from >= 50) throw new WireException('Maximum number of tests exceeded');
		if (!file_exists($this->filepath)) return false;
		$lines = file($this->filepath);
		$lines = array_map('trim',array_slice($lines, $from, $to));
		$num = count($lines);
		if ($num) {
			$list = array();
			foreach ($lines as $nastymail) {
				$list[$nastymail] = $this->validHost($nastymail);
			}		
			return $list;
		}
		return false;
	}
	*/

	/**
	 * clean up files
	 *
	 * delete surrounding spaces and tabs
	 * remove empty lines
	 * delete duplicates
	 * sort alphabetic
	 *
	 * @param string (path to file)
	 * @return multiple array of total number of entries, deleted emptylines and deleted duplicates (value => number)
	 *
	 */
	protected function clean($filepath) {
		if(!is_writable($filepath)) return $this->error("File or permission missing " . $filepath);
		$return = array();

		$file = file($filepath);
		$file = array_map('trim',$file); // delete surrounding spaces and remove linebreaks
		$uniques = array_unique($file); // delete duplicates
		$dupes = array_diff_key( $file, $uniques); // array of duplicates

		sort($uniques); // sort alphabetic
		if($uniques[0] == "") {
			array_shift($uniques); // delete empty string
			$return['deleted_emptystrings'] = 1;
		} else {
			$return['deleted_emptystrings'] = 0;		
		}
		$return['totalnum'] = count($uniques);
		$uniques = implode("\n",$uniques); // add linebreaks
		$this->save(trim($uniques), $filepath, false); // overwrite

		$return['deleted_duplicates'] = array_count_values($dupes);
		if (array_key_exists('', $return['deleted_duplicates'])) $return['deleted_emptystrings'] += $return['deleted_duplicates'][''];
		unset($return['deleted_duplicates']['']);

		return $return;		
	}
}