X-Git-Url: http://git.osdn.net/view?p=pukiwiki%2Fpukiwiki_sandbox.git;a=blobdiff_plain;f=spam%2Fspam.php;h=4170a87d908a12e6c4b744a2ffe52642e0e078d5;hp=60b8ad9ca8e660f290644123419cc6adbc58c1d5;hb=994bd33af87aa256bc133ff0d2eaf41e6c4c135e;hpb=e23db20ef955f6d4a76131eab25813c30ad0dcce

diff --git a/spam/spam.php b/spam/spam.php
index 60b8ad9..ed4a2b6 100644
--- a/spam/spam.php
+++ b/spam/spam.php
@@ -1,661 +1,732 @@
 <?php
-// $Id: spam.php,v 1.37 2006/11/25 14:07:42 henoheno Exp $
-// Copyright (C) 2006 PukiWiki Developers Team
+// $Id: spam.php,v 1.221 2011/01/24 14:51:50 henoheno Exp $
+// Copyright (C) 2006-2009, 2011 PukiWiki Developers Team
 // License: GPL v2 or (at your option) any later version
-
+//
 // Functions for Concept-work of spam-uri metrics
+//
+// (PHP 4 >= 4.3.0): preg_match_all(PREG_OFFSET_CAPTURE): $method['uri_XXX'] related feature
 
-// Return an array of URIs in the $string
-// [OK] http://nasty.example.org#nasty_string
-// [OK] http://nasty.example.org:80/foo/xxx#nasty_string/bar
-// [OK] ftp://nasty.example.org:80/dfsdfs
-// [OK] ftp://cnn.example.com&story=breaking_news@10.0.0.1/top_story.htm (from RFC3986)
-function uri_pickup($string = '', $normalize = TRUE,
-	$preserve_rawuri = FALSE, $preserve_chunk = TRUE)
-{
-	// Not available for: IDN(ignored)
-	$array = array();
-	preg_match_all(
-		// scheme://userinfo@host:port/path/or/pathinfo/maybefile.and?query=string#fragment
-		// Refer RFC3986 (Regex below is not strict)
-		'#(\b[a-z][a-z0-9.+-]{1,8})://' .	// 1: Scheme
-		'(?:' .
-			'([^\s<>"\'\[\]/\#?@]*)' .		// 2: Userinfo (Username)
-		'@)?' .
-		'(' .
-			// 3: Host
-			'\[[0-9a-f:.]+\]' . '|' .				// IPv6([colon-hex and dot]): RFC2732
-			'(?:[0-9]{1-3}\.){3}[0-9]{1-3}' . '|' .	// IPv4(dot-decimal): 001.22.3.44
-			'[^\s<>"\'\[\]:/\#?]+' . 				// FQDN: foo.example.org
-		')' .
-		'(?::([0-9]*))?' .					// 4: Port
-		'((?:/+[^\s<>"\'\[\]/\#]+)*/+)?' .	// 5: Directory path or path-info
-		'([^\s<>"\'\[\]\#]+)?' .			// 6: File and query string
-		'(?:\#([a-z0-9._~%!$&\'()*+,;=:@-]*))?' .	// 7: Fragment
-		'#i',
-		 $string, $array, PREG_SET_ORDER | PREG_OFFSET_CAPTURE
-	);
-	//var_dump(recursive_map('htmlspecialchars', $array));
 
-	// Shrink $array
-	static $parts = array(
-		1 => 'scheme', 2 => 'userinfo', 3 => 'host', 4 => 'port',
-		5 => 'path', 6 => 'file', 7 => 'fragment'
-	);
-	$default = array('');
-	foreach(array_keys($array) as $uri) {
-		array_rename_keys($array[$uri], $parts, TRUE, $default);
-		$offset = $array[$uri]['scheme'][1]; // Scheme's offset
-
-		foreach(array_keys($array[$uri]) as $part) {
-			// Remove offsets for each part
-			$array[$uri][$part] = & $array[$uri][$part][0];
-		}
+if (! defined('LIB_DIR'))   define('LIB_DIR', './');
+require(LIB_DIR . 'spam_pickup.php');
+require(LIB_DIR . 'spam_util.php');
 
-		if ($normalize) {
-			$array[$uri]['scheme'] = scheme_normalize($array[$uri]['scheme']);
-			//if ($array[$uri]['scheme'] === '') {
-			//	// Ignore
-			//	unset ($array[$uri]);
-			//	continue;
-			//}
-			
-			$array[$uri]['host']   = strtolower($array[$uri]['host']);
-			$array[$uri]['port']   = port_normalize($array[$uri]['port'], $array[$uri]['scheme'], FALSE);
-			$array[$uri]['path']   = path_normalize($array[$uri]['path']);
-			//$array[$uri]['uri']    = uri_array_implode($array[$uri]);
-			if ($preserve_rawuri) $array[$uri]['rawuri'] = & $array[$uri][0];
-		} else {
-			$array[$uri]['uri'] = & $array[$uri][0]; // Raw
-		}
-		unset($array[$uri][0]); // Matched string itself
-		if (! $preserve_chunk) {
-			unset(
-				$array[$uri]['scheme'],
-				$array[$uri]['userinfo'],
-				$array[$uri]['host'],
-				$array[$uri]['port'],
-				$array[$uri]['path'],
-				$array[$uri]['file'],
-				$array[$uri]['fragment']
-			);
-		}
+if (! defined('SPAM_INI_FILE'))   define('SPAM_INI_FILE',   'spam.ini.php');
 
-		$array[$uri]['offset'] = $offset;
-		$array[$uri]['area']   = 0;
-	}
 
-	return $array;
-}
+// ---------------------
+// Regex
 
-// Domain exposure callback (See spam_uri_pickup_preprocess())
-// http://victim.example.org/?foo+site:nasty.example.com+bar
-// => http://nasty.example.com/?refer=victim.example.org
-// NOTE: 'refer=' is not so good for (at this time).
-// Consider about using IP address of the victim, try to avoid that.
-function _preg_replace_callback_domain_exposure($matches = array())
+// Rough implementation of globbing
+//
+// USAGE: $regex = '/^' . generate_glob_regex('*.txt', '/') . '$/i';
+//
+function generate_glob_regex($string = '', $divider = '/')
 {
-	$result = '';
-
-	// Preserve the victim URI as a complicity or ...
-	if (isset($matches[5])) {
-		$result =
-			$matches[1] . '://' .	// scheme
-			$matches[2] . '/' .		// victim.example.org
-			$matches[3];			// The rest of all (before victim)
-	}
+	static $from = array(
+			 1 => '*',
+			11 => '?',
+	//		22 => '[',	// Maybe cause regex compilation error (e.g. '[]')
+	//		23 => ']',	//
+		);
+	static $mid = array(
+			 1 => '_AST_',
+			11 => '_QUE_',
+	//		22 => '_RBR_',
+	//		23 => '_LBR_',
+		);
+	static $to = array(
+			 1 => '.*',
+			11 => '.',
+	//		22 => '[',
+	//		23 => ']',
+		);
 
-	// Flipped URI
-	$result = 
-		$matches[1] . '://' .	// scheme
-		$matches[4] .			// nasty.example.com
-		'/?refer=' . strtolower($matches[2]) .	// victim.example.org
-		' ' . $result;
+	if (! is_string($string)) return '';
 
-	return $result;
+	$string = str_replace($from, $mid, $string); // Hide
+	$string = preg_quote($string, $divider);
+	$string = str_replace($mid, $to, $string);   // Unhide
+
+	return $string;
 }
 
-// Preprocess: rawurldecode() and adding space(s) to detect/count some URIs _if possible_
-// NOTE: It's maybe danger to var_dump(result). [e.g. 'javascript:']
-// [OK] http://victim.example.org/go?http%3A%2F%2Fnasty.example.org
-// [OK] http://victim.example.org/http://nasty.example.org
-function spam_uri_pickup_preprocess($string = '')
+// Generate host (FQDN, IPv4, ...) regex
+// 'localhost'     : Matches with 'localhost' only
+// 'example.org'   : Matches with 'example.org' only (See host_normalize() about 'www')
+// '.example.org'  : Matches with ALL FQDN ended with '.example.org'
+// '*.example.org' : Almost the same of '.example.org' except 'www.example.org'
+// '10.20.30.40'   : Matches with IPv4 address '10.20.30.40' only
+// [TODO] '192.'   : Matches with all IPv4 hosts started with '192.'
+// TODO: IPv4, CIDR?, IPv6
+function generate_host_regex($string = '', $divider = '/')
 {
 	if (! is_string($string)) return '';
 
-	$string = rawurldecode($string);
-
-	// Domain exposure (See _preg_replace_callback_domain_exposure())
-	$string = preg_replace_callback(
-		array(
-			// Something Google: http://www.google.com/supported_domains
-			'#(http)://([a-z0-9.]+\.google\.[a-z]{2,3}(?:\.[a-z]{2})?)/' .
-			'([a-z0-9?=&.%_+-]+)' .		// ?query=foo+
-			'\bsite:([a-z0-9.%_-]+)' .	// site:nasty.example.com
-			'()' .	// Preserve?
-			'#i',
-		),
-		'_preg_replace_callback_domain_exposure',
-		$string
-	);
+ 	if (mb_strpos($string, '.') === FALSE || is_ip($string)) {
+		// "localhost", IPv4, etc
+		return generate_glob_regex($string, $divider);
+	}
 
-	// URI exposure (uriuri => uri uri)
-	$string = preg_replace(
-		array(
-			'#(?<! )(?:https?|ftp):/#',
-		//	'#[a-z][a-z0-9.+-]{1,8}://#i',
-		//	'#[a-z][a-z0-9.+-]{1,8}://#i'
-		),
-		' $0',
-		$string
-	);
+	// FQDN or something
+	$part = explode('.', $string, 2);
+	if ($part[0] == '') {
+		// ".example.org"
+		$part[0] = '(?:.*\.)?';
+	} else if ($part[0] == '*') {
+		// "*.example.org"
+		$part[0] = '.*\.';
+	} else {
+		// example.org, etc
+		return generate_glob_regex($string, $divider);
+	}
 
-	return $string;
+	$part[1] = generate_glob_regex($part[1], $divider);
+
+	return implode('', $part);
 }
 
-// TODO: Area selection (Check BBCode only, check anchor only, check ...)
-// Main function of spam-uri pickup
-function spam_uri_pickup($string = '')
+
+// ---------------------
+// Load
+
+// Load SPAM_INI_FILE and return parsed one
+function get_blocklist($list = '')
 {
-	$string = spam_uri_pickup_preprocess($string);
-
-	$array  = uri_pickup($string);
-
-	// Area elevation for '(especially external)link' intension
-	if (! empty($array)) {
-		// Anchor tags by preg_match_all()
-		// [OK] <a href="http://nasty.example.com">visit http://nasty.example.com/</a>
-		// [OK] <a href=\'http://nasty.example.com/\' >discount foobar</a> 
-		// [NG] <a href="http://ng.example.com">visit http://ng.example.com _not_ended_
-		// [NG] <a href=  >Good site!</a> <a href= "#" >test</a>
-		$areas = array();
-		preg_match_all('#<a\b[^>]*href[^>]*>.*?</a\b[^>]*(>)#i',
-			 $string, $areas, PREG_SET_ORDER | PREG_OFFSET_CAPTURE);
-		//var_dump(recursive_map('htmlspecialchars', $areas));
-		foreach(array_keys($areas) as $area) {
-			$areas[$area] =  array(
-				$areas[$area][0][1], // Area start (<a href>)
-				$areas[$area][1][1], // Area end   (</a>)
-			);
-		}
-		area_measure($areas, $array);
-
-		// phpBB's "BBCode" by preg_match_all()
-		// [url]http://nasty.example.com/[/url]
-		// [link]http://nasty.example.com/[/link]
-		// [url=http://nasty.example.com]visit http://nasty.example.com/[/url]
-		// [link http://nasty.example.com/]buy something[/link]
-		// ?? [url=][/url]
-		$areas = array();
-		preg_match_all('#\[(url|link)\b[^\]]*\].*?\[/\1\b[^\]]*(\])#i',
-			 $string, $areas, PREG_SET_ORDER | PREG_OFFSET_CAPTURE);
-		//var_dump(recursive_map('htmlspecialchars', $areas));
-		foreach(array_keys($areas) as $area) {
-			$areas[$area] = array(
-				$areas[$area][0][1], // Area start ([url])
-				$areas[$area][2][1], // Area end   ([/url])
-			);
+	static $regexes;
+
+	if ($list === NULL) {
+		$regexes = NULL;	// Unset
+		return array();
+	}
+
+	if (! isset($regexes)) {
+		$regexes = array();
+		if (file_exists(SPAM_INI_FILE)) {
+			$blocklist = array();
+
+			include(SPAM_INI_FILE);
+			//	$blocklist['list'] = array(
+			//  	//'goodhost' => FALSE;
+			//  	'badhost' => TRUE;
+			// );
+			//	$blocklist['badhost'] = array(
+			//		'*.blogspot.com',	// Blog services's subdomains (only)
+			//		'IANA-examples' => '#^(?:.*\.)?example\.(?:com|net|org)$#',
+			//	);
+
+			foreach(array(
+					'pre',
+					'list',
+				) as $special) {
+
+				if (! isset($blocklist[$special])) continue;
+
+				$regexes[$special] = $blocklist[$special];
+
+				foreach(array_keys($blocklist[$special]) as $_list) {
+					if (! isset($blocklist[$_list])) continue;
+
+					foreach ($blocklist[$_list] as $key => $value) {
+						if (is_array($value)) {
+							$regexes[$_list][$key] = array();
+							foreach($value as $_key => $_value) {
+								get_blocklist_add($regexes[$_list][$key], $_key, $_value);
+							}
+						} else {
+							get_blocklist_add($regexes[$_list], $key, $value);
+						}
+					}
+
+					unset($blocklist[$_list]);
+				}
+			}
 		}
-		area_measure($areas, $array);
-
-		// Various Wiki syntax
-		// [text_or_uri>text_or_uri]
-		// [text_or_uri:text_or_uri]
-		// [text_or_uri|text_or_uri]
-		// [text_or_uri->text_or_uri]
-		// [text_or_uri text_or_uri] // MediaWiki
-		// MediaWiki: [http://nasty.example.com/ visit http://nasty.example.com/]
-
-		// Remove 'offset's for area_measure()
-		//foreach(array_keys($array) as $key)
-		//	unset($array[$key]['offset']);
 	}
 
-	return $array;
+	if ($list === '') {
+		return $regexes;		// ALL of
+	} else if (isset($regexes[$list])) {
+		return $regexes[$list];	// A part of
+	} else {
+		return array();			// Found nothing
+	}
 }
 
-// $array['something'] => $array['wanted']
-function array_rename_keys(& $array, $keys = array('from' => 'to'), $force = FALSE, $default = '')
+// Subroutine of get_blocklist(): Add new regex to the $array
+function get_blocklist_add(& $array, $key = 0, $value = '*.example.org/path/to/file.html')
 {
-	if (! is_array($array) || ! is_array($keys))
-		return FALSE;
-
-	// Nondestructive test
-	if (! $force)
-		foreach(array_keys($keys) as $from)
-			if (! isset($array[$from]))
-				return FALSE;
-
-	foreach($keys as $from => $to) {
-		if ($from === $to) continue;
-		if (! $force || isset($array[$from])) {
-			$array[$to] = & $array[$from];
-			unset($array[$from]);
-		} else  {
-			$array[$to] = $default;
+	if (is_string($key)) {
+		$array[$key]   = & $value; // Treat $value as a regex for FQDN(host)s
+	} else {
+		$regex = generate_host_regex($value, '#');
+		if (! empty($regex)) {
+			$array[$value] = '#^' . $regex . '$#i';
 		}
 	}
-
-	return TRUE;
 }
 
-// If in doubt, it's a little doubtful
-function area_measure($areas, & $array, $belief = -1, $a_key = 'area', $o_key = 'offset')
+// Blocklist metrics: Separate $host, to $blocked and not blocked
+function blocklist_distiller(& $hosts, $keys = array('goodhost', 'badhost'), $asap = FALSE)
 {
-	if (! is_array($areas) || ! is_array($array)) return;
-
-	$areas_keys = array_keys($areas);
-	foreach(array_keys($array) as $u_index) {
-		$offset = isset($array[$u_index][$o_key]) ?
-			intval($array[$u_index][$o_key]) : 0;
-		foreach($areas_keys as $a_index) {
-			if (isset($array[$u_index][$a_key])) {
-				$offset_s = intval($areas[$a_index][0]);
-				$offset_e = intval($areas[$a_index][1]);
-				// [Area => inside <= Area]
-				if ($offset_s < $offset && $offset < $offset_e) {
-					$array[$u_index][$a_key] += $belief;
+	if (! is_array($hosts)) $hosts = array($hosts);
+	if (! is_array($keys))  $keys  = array($keys);
+
+	$list = get_blocklist('list');
+	$blocked = array();
+
+	foreach($keys as $key){
+		foreach (get_blocklist($key) as $label => $regex) {
+			if (is_array($regex)) {
+				foreach($regex as $_label => $_regex) {
+					$group = preg_grep($_regex, $hosts);
+					if ($group) {
+						$hosts = array_diff($hosts, $group);
+						$blocked[$key][$label][$_label] = $group;
+						if ($asap && $list[$key]) break;
+					}
+				}
+			} else {
+				$group = preg_grep($regex, $hosts);
+				if ($group) {
+					$hosts = array_diff($hosts, $group);
+					$blocked[$key][$label] = $group;
+					if ($asap && $list[$key]) break;
 				}
 			}
 		}
 	}
+
+	return $blocked;
 }
 
 
 // ---------------------
-// Part Two
 
-// Scheme normalization: Renaming the schemes
-// snntp://example.org =>  nntps://example.org
-// NOTE: Keep the static lists simple. See also port_normalize().
-function scheme_normalize($scheme = '', $considerd_harmfull = TRUE)
-{
-	// Abbreviations considerable they don't have link intension
-	static $abbrevs = array(
-		'ttp'	=> 'http',
-		'ttps'	=> 'https',
-	);
 
-	// Alias => normalized
-	static $aliases = array(
-		'pop'	=> 'pop3',
-		'news'	=> 'nntp',
-		'imap4'	=> 'imap',
-		'snntp'	=> 'nntps',
-		'snews'	=> 'nntps',
-		'spop3'	=> 'pop3s',
-		'pops'	=> 'pop3s',
+// Default (enabled) methods and thresholds (for content insertion)
+function check_uri_spam_method($times = 1, $t_area = 0, $rule = TRUE)
+{
+	$times  = intval($times);
+	$t_area = intval($t_area);
+
+	$positive = array(
+		// Thresholds
+		'quantity'     =>  8 * $times,	// Allow N URIs
+		'non_uniqhost' =>  3 * $times,	// Allow N duped (and normalized) Hosts
+		//'non_uniquri'=>  3 * $times,	// Allow N duped (and normalized) URIs
+
+		// Areas
+		'area_anchor'  => $t_area,	// Using <a href> HTML tag
+		'area_bbcode'  => $t_area,	// Using [url] or [link] BBCode
+		//'uri_anchor' => $t_area,	// URI inside <a href> HTML tag
+		//'uri_bbcode' => $t_area,	// URI inside [url] or [link] BBCode
 	);
+	if ($rule) {
+		$bool = array(
+			// Rules
+			//'asap'   => TRUE,	// Quit or return As Soon As Possible
+			'uniqhost' => TRUE,	// Show uniq host (at block notification mail)
+			'badhost'  => TRUE,	// Check badhost
+		);
+	} else {
+		$bool = array();
+	}
 
-	$scheme = strtolower(trim($scheme));
-	if (isset($abbrevs[$scheme])) {
-		if ($considerd_harmfull) {
-			$scheme = $abbrevs[$scheme];
-		} else {
-			$scheme = '';
-		}
+	// Remove non-$positive values
+	foreach (array_keys($positive) as $key) {
+		if ($positive[$key] < 0) unset($positive[$key]);
 	}
-	if (isset($aliases[$scheme])) $scheme = $aliases[$scheme];
 
-	return $scheme;
+	return $positive + $bool;
 }
 
-// Port normalization: Suppress the (redundant) default port
-// HTTP://example.org:80/ => http://example.org/
-// HTTP://example.org:8080/ => http://example.org:8080/
-// HTTPS://example.org:443/ => https://example.org/
-function port_normalize($port, $scheme, $scheme_normalize = TRUE)
+// Simple/fast spam check
+function check_uri_spam($target = '', $method = array())
 {
-	// Schemes that users _maybe_ want to add protocol-handlers
-	// to their web browsers. (and attackers _maybe_ want to use ...)
-	// Reference: http://www.iana.org/assignments/port-numbers
-	static $array = array(
-		// scheme => default port
-		'ftp'     =>    21,
-		'ssh'     =>    22,
-		'telnet'  =>    23,
-		'smtp'    =>    25,
-		'tftp'    =>    69,
-		'gopher'  =>    70,
-		'finger'  =>    79,
-		'http'    =>    80,
-		'pop3'    =>   110,
-		'sftp'    =>   115,
-		'nntp'    =>   119,
-		'imap'    =>   143,
-		'irc'     =>   194,
-		'wais'    =>   210,
-		'https'   =>   443,
-		'nntps'   =>   563,
-		'rsync'   =>   873,
-		'ftps'    =>   990,
-		'telnets' =>   992,
-		'imaps'   =>   993,
-		'ircs'    =>   994,
-		'pop3s'   =>   995,
-		'mysql'   =>  3306,
+	// Return value
+	$progress = array(
+		'method'  => array(
+			// Theme to do  => Dummy, optional value, or optional array()
+			//'quantity'    => 8,
+			//'uniqhost'    => TRUE,
+			//'non_uniqhost'=> 3,
+			//'non_uniquri' => 3,
+			//'badhost'     => TRUE,
+			//'area_anchor' => 0,
+			//'area_bbcode' => 0,
+			//'uri_anchor'  => 0,
+			//'uri_bbcode'  => 0,
+		),
+		'sum' => array(
+			// Theme        => Volume found (int)
+		),
+		'is_spam' => array(
+			// Flag. If someting defined here,
+			// one or more spam will be included
+			// in this report
+		),
+		'blocked' => array(
+			// Hosts blocked
+			//'category' => array(
+			//	'host',
+			//)
+		),
+		'hosts' => array(
+			// Hosts not blocked
+		),
 	);
 
-	$port = trim($port);
-	if ($port === '') return $port;
+	// ----------------------------------------
+	// Aliases
 
-	if ($scheme_normalize) $scheme = scheme_normalize($scheme);
-	if (isset($array[$scheme]) && $port == $array[$scheme])
-		$port = ''; // Ignore the defaults
+	$sum     = & $progress['sum'];
+	$is_spam = & $progress['is_spam'];
+	$progress['method'] = & $method;	// Argument
+	$blocked = & $progress['blocked'];
+	$hosts   = & $progress['hosts'];
+	$asap    = isset($method['asap']);
 
-	return $port;
-}
+	// ----------------------------------------
+	// Init
 
-// Path normalization
-// http://example.org => http://example.org/
-// http://example.org#hoge => http://example.org/#hoge
-// http://example.org/path/a/b/./c////./d => http://example.org/path/a/b/c/d
-// http://example.org/path/../../a/../back => http://example.org/back
-function path_normalize($path = '', $divider = '/', $addroot = TRUE)
-{
-	if (! is_string($path) || $path == '') {
-		$path = $addroot ? $divider : '';
-	} else {
-		$path = trim($path);
-		$last = ($path[strlen($path) - 1] == $divider) ? $divider : '';
-		$array = explode($divider, $path);
-
-		// Remove paddings
-		foreach(array_keys($array) as $key) {
-			if ($array[$key] == '' || $array[$key] == '.')
-				 unset($array[$key]);
-		}
-		// Back-track
-		$tmp = array();
-		foreach($array as $value) {
-			if ($value == '..') {
-				array_pop($tmp);
-			} else {
-				array_push($tmp, $value);
+	if (! is_array($method) || empty($method)) {
+		$method = check_uri_spam_method();
+	}
+	foreach(array_keys($method) as $key) {
+		if (! isset($sum[$key])) $sum[$key] = 0;
+	}
+	if (! isset($sum['quantity'])) $sum['quantity'] = 0;
+
+	// ----------------------------------------
+	// Recurse
+
+	if (is_array($target)) {
+		foreach($target as $str) {
+			if (! is_string($str)) continue;
+
+			$_progress = check_uri_spam($str, $method);	// Recurse
+
+			// Merge $sum
+			$_sum = & $_progress['sum'];
+			foreach (array_keys($_sum) as $key) {
+				if (! isset($sum[$key])) {
+					$sum[$key] = & $_sum[$key];
+				} else {
+					$sum[$key] += $_sum[$key];
+				}
+			}
+
+			// Merge $is_spam
+			$_is_spam = & $_progress['is_spam'];
+			foreach (array_keys($_is_spam) as $key) {
+				$is_spam[$key] = TRUE;
+				if ($asap) break;
 			}
+			if ($asap && $is_spam) break;
+
+			// Merge only
+			$blocked = array_merge_leaves($blocked, $_progress['blocked'], FALSE);
+			$hosts   = array_merge_leaves($hosts,   $_progress['hosts'],   FALSE);
 		}
-		$array = & $tmp;
 
-		$path = $addroot ? $divider : '';
-		if (! empty($array)) $path .= implode($divider, $array) . $last;
+		// Unique values
+		$blocked = array_unique_recursive($blocked);
+		$hosts   = array_unique_recursive($hosts);
+
+		// Recount $sum['badhost']
+		$sum['badhost'] = array_count_leaves($blocked);
+
+		return $progress;
 	}
 
-	return $path;
-}
+	// ----------------------------------------
+	// Area measure
 
-// An URI array => An URI (See uri_pickup())
-function uri_array_implode($uri = array())
-{
-	if (empty($uri) || ! is_array($uri)) return NULL;
+	if (! $asap || ! $is_spam) {
 	
-	$tmp = array();
-	if (isset($uri['scheme']) && $uri['scheme'] !== '') {
-		$tmp[] = & $uri['scheme'];
-		$tmp[] = '://';
+		// Method pickup
+		$_method = array();
+		foreach(array(
+				'area_anchor',	// There's HTML anchor tag
+				'area_bbcode',	// There's 'BBCode' linking tag
+			) as $key) {
+			if (isset($method[$key])) $_method[$key] = TRUE;
+		}
+
+		if ($_method) {
+			$_asap   = isset($method['asap']) ? array('asap' => TRUE) : array();
+			$_result = area_pickup($target, $_method + $_asap);
+			$_asap   = NULL;
+		} else {
+			$_result = FALSE;
+		}
+
+		if ($_result) {
+			foreach(array_keys($_method) as $key) {
+				if (isset($_result[$key])) {
+					$sum[$key] = $_result[$key];
+					if (isset($method[$key]) && $sum[$key] > $method[$key]) {
+						$is_spam[$key] = TRUE;
+					}
+				}
+			}
+		}
+
+		unset($_asap, $_method, $_result);
+	}
+
+	// Return if ...
+	if ($asap && $is_spam) return $progress;
+
+	// ----------------------------------------
+	// URI: Pickup
+
+	$pickups = spam_uri_pickup($target, $method);
+
+
+	// Return if ...
+	if (empty($pickups)) return $progress;
+
+	// Normalize all
+	$pickups = uri_pickup_normalize($pickups);
+
+	// ----------------------------------------
+	// Pickup some part of URI
+
+	$hosts = array();
+	foreach ($pickups as $key => $pickup) {
+		$hosts[$key] = & $pickup['host'];
 	}
-	if (isset($uri['userinfo']) && $uri['userinfo'] !== '') {
-		$tmp[] = & $uri['userinfo'];
-		$tmp[] = '@';
+
+	// ----------------------------------------
+	// URI: Bad host <pre-filter> (Separate good/bad hosts from $hosts)
+
+	if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
+		$list    = get_blocklist('pre');
+		$blocked = blocklist_distiller($hosts, array_keys($list), $asap);
+		foreach($list as $key => $type){
+			if (! $type) unset($blocked[$key]); // Ignore goodhost etc
+		}
+		unset($list);
+		if (! empty($blocked)) $is_spam['badhost'] = TRUE;
 	}
-	if (isset($uri['host']) && $uri['host'] !== '') {
-		$tmp[] = & $uri['host'];
+
+	// Return if ...
+	if ($asap && $is_spam) return $progress;
+
+	// Remove blocked from $pickups
+	foreach(array_keys($pickups) as $key) {
+		if (! isset($hosts[$key])) {
+			unset($pickups[$key]);
+		}
 	}
-	if (isset($uri['port']) && $uri['port'] !== '') {
-		$tmp[] = ':';
-		$tmp[] = & $uri['port'];
+
+	// ----------------------------------------
+	// URI: Check quantity
+
+	$sum['quantity'] += count($pickups);
+		// URI quantity
+	if ((! $asap || ! $is_spam) && isset($method['quantity']) &&
+		$sum['quantity'] > $method['quantity']) {
+		$is_spam['quantity'] = TRUE;
 	}
-	if (isset($uri['path']) && $uri['path'] !== '') {
-		$tmp[] = & $uri['path'];
+
+	// ----------------------------------------
+	// URI: used inside HTML anchor tag pair
+
+	if ((! $asap || ! $is_spam) && isset($method['uri_anchor'])) {
+		$key = 'uri_anchor';
+		foreach($pickups as $pickup) {
+			if (isset($pickup['area'][$key])) {
+				$sum[$key] += $pickup['area'][$key];
+				if(isset($method[$key]) &&
+					$sum[$key] > $method[$key]) {
+					$is_spam[$key] = TRUE;
+					if ($asap && $is_spam) break;
+				}
+				if ($asap && $is_spam) break;
+			}
+		}
 	}
-	if (isset($uri['file']) && $uri['file'] !== '') {
-		$tmp[] = & $uri['file'];
+
+	// ----------------------------------------
+	// URI: used inside 'BBCode' pair
+
+	if ((! $asap || ! $is_spam) && isset($method['uri_bbcode'])) {
+		$key = 'uri_bbcode';
+		foreach($pickups as $pickup) {
+			if (isset($pickup['area'][$key])) {
+				$sum[$key] += $pickup['area'][$key];
+				if(isset($method[$key]) &&
+					$sum[$key] > $method[$key]) {
+					$is_spam[$key] = TRUE;
+					if ($asap && $is_spam) break;
+				}
+				if ($asap && $is_spam) break;
+			}
+		}
 	}
-	if (isset($uri['fragment']) && $uri['fragment'] !== '') {
-		$tmp[] = '#';
-		$tmp[] = & $uri['fragment'];
+
+	// ----------------------------------------
+	// URI: Uniqueness (and removing non-uniques)
+
+	if ((! $asap || ! $is_spam) && isset($method['non_uniquri'])) {
+
+		$uris = array();
+		foreach (array_keys($pickups) as $key) {
+			$uris[$key] = uri_pickup_implode($pickups[$key]);
+		}
+		$count = count($uris);
+		$uris  = array_unique($uris);
+		$sum['non_uniquri'] += $count - count($uris);
+		if ($sum['non_uniquri'] > $method['non_uniquri']) {
+			$is_spam['non_uniquri'] = TRUE;
+		}
+		if (! $asap || ! $is_spam) {
+			foreach (array_diff(array_keys($pickups),
+				array_keys($uris)) as $remove) {
+				unset($pickups[$remove]);
+			}
+		}
+		unset($uris);
 	}
 
-	return implode('', $tmp);
-}
+	// Return if ...
+	if ($asap && $is_spam) return $progress;
 
-// ---------------------
-// Part One : Checker
+	// ----------------------------------------
+	// Host: Uniqueness (uniq / non-uniq)
 
-function generate_glob_regex($string = '', $divider = '/')
-{
-	static $from = array(
-			0 => '*',
-			1 => '?',
-			2 => '\[',
-			3 => '\]',
-			4 => '[',
-			5 => ']',
-		);
-	static $mid = array(
-			0 => '_AST_',
-			1 => '_QUE_',
-			2 => '_eRBR_',
-			3 => '_eLBR_',
-			4 => '_RBR_',
-			5 => '_LBR_',
-		);
-	static $to = array(
-			0 => '.*',
-			1 => '.',
-			2 => '\[',
-			3 => '\]',
-			4 => '[',
-			5 => ']',
-		);
+	$hosts = array_unique($hosts);
 
-	$string = str_replace($from, $mid, $string); // Hide
-	$string = preg_quote($string, $divider);
-	$string = str_replace($mid, $to, $string);   // Unhide
+	if (isset($sum['uniqhost'])) $sum['uniqhost'] += count($hosts);
+	if ((! $asap || ! $is_spam) && isset($method['non_uniqhost'])) {
+		$sum['non_uniqhost'] = $sum['quantity'] - $sum['uniqhost'];
+		if ($sum['non_uniqhost'] > $method['non_uniqhost']) {
+			$is_spam['non_uniqhost'] = TRUE;
+		}
+	}
 
-	return $string;
-}
+	// Return if ...
+	if ($asap && $is_spam) return $progress;
 
-// TODO: Ignore list
-// TODO: require_or_include_once(another file)
-function is_badhost($hosts = '', $asap = TRUE)
-{
-	static $blocklist_regex;
-
-	if (! isset($blocklist_regex)) {
-		$blocklist_regex = array();
-		$blocklist = array(
-			// Deny all uri
-			//'*',
-
-			// IP address or ...
-			//'10.20.*.*',	// 10.20.example.com also matches
-			//'\[1\]',
-			
-			// Too much malicious sub-domains
-			'*.blogspot.com',
-
-			// 2006-11 dev
-			'wwwtahoo.com',
-
-			// 2006-11 dev
-			'*.infogami.com',
-
-			// 2006/11/19 17:50 dev
-			'*.google0site.org',
-			'*.bigpricesearch.org',
-			'*.osfind.org',
-			'*.bablomira.biz',
+	// ----------------------------------------
+	// URI: Bad host (Separate good/bad hosts from $hosts)
+
+	if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
+		$list    = get_blocklist('list');
+		$blocked = array_merge_leaves(
+			$blocked,
+			blocklist_distiller($hosts, array_keys($list), $asap),
+			FALSE
 		);
-		foreach ($blocklist as $part) {
-			$blocklist_regex[] = '#^' . generate_glob_regex($part, '#') . '$#i';
+		foreach($list as $key=>$type){
+			if (! $type) unset($blocked[$key]); // Ignore goodhost etc
 		}
+		unset($list);
+		if (! empty($blocked)) $is_spam['badhost'] = TRUE;
 	}
 
-	$result = 0;
-	if (! is_array($hosts)) $hosts = array($hosts);
-	foreach($hosts as $host) {
-		if (! is_string($host)) $host = '';
-		foreach ($blocklist_regex as $regex) {
-			if (preg_match($regex, $host)) {
-				++$result;
-				if ($asap) {
-					return $result;
-				} else {
-					break; // Check next host
+	// Return if ...
+	//if ($asap && $is_spam) return $progress;
+
+	// ----------------------------------------
+	// End
+
+	return $progress;
+}
+
+// ---------------------
+// Reporting
+
+// Summarize $progress (blocked only)
+function summarize_spam_progress($progress = array(), $blockedonly = FALSE)
+{
+	if ($blockedonly) {
+		$tmp = array_keys($progress['is_spam']);
+	} else {
+		$tmp = array();
+		$method = & $progress['method'];
+		if (isset($progress['sum'])) {
+			foreach ($progress['sum'] as $key => $value) {
+				if (isset($method[$key]) && $value) {
+					$tmp[] = $key . '(' . $value . ')';
 				}
 			}
 		}
 	}
 
-	return $result;
+	return implode(', ', $tmp);
 }
 
-// TODO return TRUE or FALSE!
-// Simple/fast spam check
-function check_uri_spam($target = '', $method = array(), $asap = TRUE)
+function summarize_detail_badhost($progress = array())
 {
-	$is_spam  = FALSE;
-	$progress = array(
-		'quantity' => 0,
-		'area'     => 0,
-		'non_uniq' => 0,
-		'uniqhost' => 0,
-		'badhost'  => 0,
-		);
-
-	if (! is_array($method) || empty($method)) {
-		// Default
-		$method = array(
-			'quantity' => 8,		// Allow N URIs
-			'area'     => TRUE,
-			'non_uniq' => 3,		// Allow N times dupe
-			'badhost'  => TRUE,
-		);
+	if (! isset($progress['blocked']) || empty($progress['blocked'])) return '';
+
+	// Flat per group
+	$blocked = array();
+	foreach($progress['blocked'] as $list => $lvalue) {
+		foreach($lvalue as $group => $gvalue) {
+			$flat = implode(', ', array_flat_leaves($gvalue));
+			if ($flat === $group) {
+				$blocked[$list][]       = $flat;
+			} else {
+				$blocked[$list][$group] = $flat;
+			}
+		}
 	}
 
-	if (is_array($target)) {
-		foreach($target as $str) {
-			// Recurse
-			list($is_spam, $_progress) = check_uri_spam($str, $method);
-			$progress['quantity'] += $_progress['quantity'];
-			$progress['non_uniq'] += $_progress['non_uniq'];
-			if ($asap || $is_spam) break;
+	// Shrink per list
+	// From: 'A-1' => array('ie.to')
+	// To:   'A-1' => 'ie.to'
+	foreach($blocked as $list => $lvalue) {
+		if (is_array($lvalue) &&
+		   count($lvalue) == 1 &&
+		   is_numeric(key($lvalue))) {
+		    $blocked[$list] = current($lvalue);
 		}
-	} else {
-		$pickups = spam_uri_pickup($target);
-		$progress['quantity'] += count($pickups);
-
-		if (! empty($pickups)) {
+	}
 
-			// URI quantity
-			if ((! $is_spam || ! $asap) && isset($method['quantity']) &&
-				$progress['quantity'] > $method['quantity']) {
-				$is_spam = TRUE;
-			}
-			//var_dump($method['quantity'], $is_spam);
-
-			// Using invalid area
-			if ((! $is_spam || ! $asap) && isset($method['area'])) {
-				foreach($pickups as $pickup) {
-					if ($pickup['area'] < 0) {
-						++$progress['area'];
-						$is_spam = TRUE;
-						if ($asap) break;
-					}
-				}
-			}
-			//var_dump($method['area'], $is_spam);
+	return var_export_shrink($blocked, TRUE, TRUE);
+}
 
-			// URI uniqueness (and removing non-uniques)
-			if ((! $is_spam || ! $asap) && isset($method['non_uniq'])) {
-				$uris = array();
-				foreach ($pickups as $key => $pickup) {
-					$uris[$key] = uri_array_implode($pickup);
-				}
-				$count = count($uris);
-				$uris = array_unique($uris);
-				$progress['non_uniq'] += $count - count($uris);
-				if ($progress['non_uniq'] > $method['non_uniq']) {
-					$is_spam = TRUE;
-				}
-				if (! $asap || ! $is_spam) {
-					foreach (array_diff(array_keys($pickups),
-						array_keys($uris)) as $remove) {
-						unset($pickups[$remove]);
-					}
-				}
-				unset($uris);
-				//var_dump($uris, $pickups);
-			}
-			//var_dump($method['non_uniq'], $is_spam);
+function summarize_detail_newtral($progress = array())
+{
+	if (! isset($progress['hosts'])    ||
+	    ! is_array($progress['hosts']) ||
+	    empty($progress['hosts'])) return '';
+
+	// Generate a responsible $trie
+	$trie = array();
+	foreach($progress['hosts'] as $value) {
+		// 'A.foo.bar.example.com'
+		$resp = whois_responsibility($value);	// 'example.com'
+		if (empty($resp)) {
+			// One or more test, or do nothing here
+			$resp = strval($value);
+			$rest = '';
+		} else {
+			$rest = rtrim(substr($value, 0, - strlen($resp)), '.');	// 'A.foo.bar'
+		}
+		$trie = array_merge_leaves($trie, array($resp => array($rest => NULL)), FALSE);
+	}
 
-			// Bad host
-			if ((! $is_spam || ! $asap) && isset($method['badhost'])) {
-				$hosts = array();
-				foreach ($pickups as $pickup) {
-					$hosts[] = & $pickup['host'];
+	// Format: var_export_shrink() -like output
+	$result = array();
+	ksort_by_domain($trie);
+	foreach(array_keys($trie) as $key) {
+		ksort_by_domain($trie[$key]);
+		if (count($trie[$key]) == 1 && key($trie[$key]) == '') {
+			// Just one 'responsibility.example.com'
+			$result[] = '  \'' . $key . '\',';
+		} else {
+			// One subdomain-or-host, or several ones
+			$subs = array();
+			foreach(array_keys($trie[$key]) as $sub) {
+				if ($sub == '') {
+					$subs[] = $key;			// 'example.com'
+				} else {
+					$subs[] = $sub . '. ';	// 'A.foo.bar. '
 				}
-				$hosts = array_unique($hosts);
-				$progress['uniqhost'] += count($hosts);
-				$count = is_badhost($hosts, $asap);
-				$progress['badhost'] += $count;
-				if ($count !== 0) $is_spam = TRUE;
 			}
-			//var_dump($method['badhost'], $is_spam);
+			$result[] = '  \'' . $key . '\' => \'' . implode(', ', $subs) . '\',';
 		}
+		unset($trie[$key]);
 	}
-
-	return array($is_spam, $progress);
+	return
+		'array (' . "\n" .
+			implode("\n", $result) . "\n" .
+		')';
 }
 
+
 // ---------------------
+// Exit
 
-// Check User-Agent (not testing yet)
-function is_invalid_useragent($ua_name = '' /*, $ua_vars = ''*/ )
+// Freeing memories
+function spam_dispose()
 {
-	return $ua_name === '';
+	get_blocklist(NULL);
+	whois_responsibility(NULL);
 }
 
+// Common bahavior for blocking
+// NOTE: Call this function from various blocking feature, to disgueise the reason 'why blocked'
+function spam_exit($mode = '', $data = array())
+{
+	$exit = TRUE;
+
+	switch ($mode) {
+		case '':
+			echo("\n");
+			break;
+		case 'dump':
+			echo('<pre>' . "\n");
+			echo htmlsc(var_export($data, TRUE));
+			echo('</pre>' . "\n");
+			break;
+	};
+
+	if ($exit) exit;	// Force exit
+}
+
+
 // ---------------------
+// Simple filtering
 
-// TODO: Separate check-part(s) and mail part
-// TODO: Mail to administrator with more measurement data?
+// TODO: Record them
 // Simple/fast spam filter ($target: 'a string' or an array())
-function pkwk_spamfilter($action, $page, $target = array('title' => ''), $method = array())
+function pkwk_spamfilter($action, $page, $target = array('title' => ''), $method = array(), $exitmode = '')
 {
-	$is_spam = FALSE;
+	$progress = check_uri_spam($target, $method);
 
-	//$is_spam =  is_invalid_useragent('NOTYET');
-	if ($is_spam) {
-		$action .= ' (Invalid User-Agent)';
+	if (empty($progress['is_spam'])) {
+		spam_dispose();
 	} else {
-		list($is_spam) = check_uri_spam($target, $method);
-	}
 
-	if ($is_spam) {
-		// Mail to administrator(s)
-		global $notify, $notify_subject;
-		if ($notify) {
-			$footer['ACTION'] = $action;
-			$footer['PAGE']   = '[blocked] ' . $page;
-			$footer['URI']    = get_script_uri() . '?' . rawurlencode($page);
-			$footer['USER_AGENT']  = TRUE;
-			$footer['REMOTE_ADDR'] = TRUE;
-			pkwk_mail_notify($notify_subject,  var_export($target, TRUE), $footer);
-			unset($footer);
-		}
-	}
+// TODO: detect encoding from $target for mbstring functions
+//		$tmp = array();
+//		foreach(array_keys($target) as $key) {
+//			$tmp[strings($key, 0, FALSE, TRUE)] = strings($target[$key], 0, FALSE, TRUE);	// Removing "\0" etc
+//		}
+//		$target = & $tmp;
 
-	if ($is_spam) spam_exit();
+		pkwk_spamnotify($action, $page, $target, $progress, $method);
+		spam_exit($exitmode, $progress);
+	}
 }
 
 // ---------------------
+// PukiWiki original
 
-// Common bahavior for blocking
-// NOTE: Call this function from various blocking feature, to disgueise the reason 'why blocked'
-function spam_exit()
+// Mail to administrator(s)
+function pkwk_spamnotify($action, $page, $target = array('title' => ''), $progress = array(), $method = array())
 {
-	die("\n");
+	global $notify, $notify_subject;
+
+	if (! $notify) return;
+
+	$asap = isset($method['asap']);
+
+	$summary['ACTION']  = 'Blocked by: ' . summarize_spam_progress($progress, TRUE);
+	if (! $asap) {
+		$summary['METRICS'] = summarize_spam_progress($progress);
+	}
+
+	$tmp = summarize_detail_badhost($progress);
+	if ($tmp != '') $summary['DETAIL_BADHOST'] = $tmp;
+
+	$tmp = summarize_detail_newtral($progress);
+	if (! $asap && $tmp != '') $summary['DETAIL_NEUTRAL_HOST'] = $tmp;
+
+	$summary['COMMENT'] = $action;
+	$summary['PAGE']    = '[blocked] ' . (is_pagename($page) ? $page : '');
+	$summary['URI']     = get_script_uri() . '?' . rawurlencode($page);
+	$summary['USER_AGENT']  = TRUE;
+	$summary['REMOTE_ADDR'] = TRUE;
+	pkwk_mail_notify($notify_subject,  var_export($target, TRUE), $summary, TRUE);
 }
 
 ?>