X-Git-Url: http://git.osdn.net/view?a=blobdiff_plain;f=spam%2Fspam.php;h=0850859a935c0204282ca2886f6016c101cd9452;hb=0804eeeb75480d620b89e325f6daaa1b99bd0173;hp=0f15961a4a0bb6f666af0e891b78b4a681c6ef01;hpb=3c9d634a3d81e3271d7f88e3d88dfcf350a8b2b4;p=pukiwiki%2Fpukiwiki_sandbox.git diff --git a/spam/spam.php b/spam/spam.php index 0f15961..0850859 100644 --- a/spam/spam.php +++ b/spam/spam.php @@ -1,8 +1,10 @@ = 4.3.0): preg_match_all(PREG_OFFSET_CAPTURE): $method['uri_XXX'] related feature if (! defined('SPAM_INI_FILE')) define('SPAM_INI_FILE', 'spam.ini.php'); @@ -43,22 +45,23 @@ function preg_grep_invert($pattern = '//', $input = array()) // [OK] http://nasty.example.org:80/foo/xxx#nasty_string/bar // [OK] ftp://nasty.example.org:80/dfsdfs // [OK] ftp://cnn.example.com&story=breaking_news@10.0.0.1/top_story.htm (from RFC3986) -function uri_pickup($string = '', $normalize = TRUE, - $preserve_rawuri = FALSE, $preserve_chunk = TRUE) +function uri_pickup($string = '') { + if (! is_string($string)) return array(); + // Not available for: IDN(ignored) $array = array(); preg_match_all( // scheme://userinfo@host:port/path/or/pathinfo/maybefile.and?query=string#fragment // Refer RFC3986 (Regex below is not strict) - '#(\b[a-z][a-z0-9.+-]{1,8})://' . // 1: Scheme + '#(\b[a-z][a-z0-9.+-]{1,8}):/+' . // 1: Scheme '(?:' . '([^\s<>"\'\[\]/\#?@]*)' . // 2: Userinfo (Username) '@)?' . '(' . // 3: Host '\[[0-9a-f:.]+\]' . '|' . // IPv6([colon-hex and dot]): RFC2732 - '(?:[0-9]{1-3}\.){3}[0-9]{1-3}' . '|' . // IPv4(dot-decimal): 001.22.3.44 + '(?:[0-9]{1,3}\.){3}[0-9]{1,3}' . '|' . // IPv4(dot-decimal): 001.22.3.44 '[a-z0-9.-]+' . // hostname(FQDN) : foo.example.org ')' . '(?::([0-9]*))?' . // 4: Port @@ -70,7 +73,7 @@ function uri_pickup($string = '', $normalize = TRUE, $string, $array, PREG_SET_ORDER | PREG_OFFSET_CAPTURE ); - // Shrink $array + // Format the $array static $parts = array( 1 => 'scheme', 2 => 'userinfo', 3 => 'host', 4 => 'port', 5 => 'path', 6 => 'file', 7 => 'query', 8 => 'fragment' @@ -79,70 +82,63 @@ function uri_pickup($string = '', $normalize = TRUE, foreach(array_keys($array) as $uri) { $_uri = & $array[$uri]; array_rename_keys($_uri, $parts, TRUE, $default); - - $offset = $_uri['scheme'][1]; // Scheme's offset + $offset = $_uri['scheme'][1]; // Scheme's offset = URI's offset foreach(array_keys($_uri) as $part) { - // Remove offsets for each part - $_uri[$part] = & $_uri[$part][0]; + $_uri[$part] = & $_uri[$part][0]; // Remove offsets } + } - if ($normalize) { - $_uri['scheme'] = scheme_normalize($_uri['scheme']); - if ($_uri['scheme'] === '') { - unset($array[$uri]); - continue; - } - $_uri['host'] = strtolower($_uri['host']); - $_uri['port'] = port_normalize($_uri['port'], $_uri['scheme'], FALSE); - $_uri['path'] = path_normalize($_uri['path']); - if ($preserve_rawuri) $_uri['rawuri'] = & $_uri[0]; - - // DEBUG - //$_uri['uri'] = uri_array_implode($_uri); - } else { - $_uri['uri'] = & $_uri[0]; // Raw + foreach(array_keys($array) as $uri) { + $_uri = & $array[$uri]; + if ($_uri['scheme'] === '') { + unset($array[$uri]); // Considererd harmless + continue; } unset($_uri[0]); // Matched string itself - if (! $preserve_chunk) { - unset( - $_uri['scheme'], - $_uri['userinfo'], - $_uri['host'], - $_uri['port'], - $_uri['path'], - $_uri['file'], - $_uri['query'], - $_uri['fragment'] - ); - } - - // Area offset for area_measure() - $_uri['area']['offset'] = $offset; + $_uri['area']['offset'] = $offset; // Area offset for area_measure() } return $array; } -// Destructive normalize of URI array -// NOTE: Give me the uri_pickup() result with chunks -function uri_array_normalize(& $pickups, $preserve = TRUE) +// Normalize an array of URI arrays +// NOTE: Give me the uri_pickup() results +function uri_pickup_normalize(& $pickups, $destructive = TRUE) { if (! is_array($pickups)) return $pickups; - foreach (array_keys($pickups) as $key) { - $_key = & $pickups[$key]; - $_key['path'] = isset($_key['path']) ? strtolower($_key['path']) : ''; - $_key['file'] = isset($_key['file']) ? file_normalize($_key['file']) : ''; - $_key['query'] = isset($_key['query']) ? query_normalize(strtolower($_key['query']), TRUE) : ''; - $_key['fragment'] = (isset($_key['fragment']) && $preserve) ? - strtolower($_key['fragment']) : ''; // Just ignore + if ($destructive) { + foreach (array_keys($pickups) as $key) { + $_key = & $pickups[$key]; + $_key['scheme'] = isset($_key['scheme']) ? scheme_normalize($_key['scheme']) : ''; + $_key['host'] = isset($_key['host']) ? host_normalize($_key['host']) : ''; + $_key['port'] = isset($_key['port']) ? port_normalize($_key['port'], $_key['scheme'], FALSE) : ''; + $_key['path'] = isset($_key['path']) ? strtolower(path_normalize($_key['path'])) : ''; + $_key['file'] = isset($_key['file']) ? file_normalize($_key['file']) : ''; + $_key['query'] = isset($_key['query']) ? query_normalize($_key['query']) : ''; + $_key['fragment'] = isset($_key['fragment']) ? strtolower($_key['fragment']) : ''; + } + } else { + foreach (array_keys($pickups) as $key) { + $_key = & $pickups[$key]; + $_key['scheme'] = isset($_key['scheme']) ? scheme_normalize($_key['scheme']) : ''; + $_key['host'] = isset($_key['host']) ? strtolower($_key['host']) : ''; + $_key['port'] = isset($_key['port']) ? port_normalize($_key['port'], $_key['scheme'], FALSE) : ''; + $_key['path'] = isset($_key['path']) ? path_normalize($_key['path']) : ''; + } } return $pickups; } // An URI array => An URI (See uri_pickup()) -function uri_array_implode($uri = array()) +// USAGE: +// $pickups = uri_pickup('a string include some URIs'); +// $uris = array(); +// foreach (array_keys($pickups) as $key) { +// $uris[$key] = uri_pickup_implode($pickups[$key]); +// } +function uri_pickup_implode($uri = array()) { if (empty($uri) || ! is_array($uri)) return NULL; @@ -337,6 +333,7 @@ function _preg_replace_callback_domain_exposure($matches = array()) // NOTE: It's maybe danger to var_dump(result). [e.g. 'javascript:'] // [OK] http://victim.example.org/go?http%3A%2F%2Fnasty.example.org // [OK] http://victim.example.org/http://nasty.example.org +// TODO: link.toolbot.com, urlx.org function spam_uri_pickup_preprocess($string = '') { if (! is_string($string)) return ''; @@ -346,9 +343,17 @@ function spam_uri_pickup_preprocess($string = '') // Domain exposure (See _preg_replace_callback_domain_exposure()) $string = preg_replace_callback( array( - // Something Google: http://www.google.com/supported_domains - '#(http)://([a-z0-9.]+\.google\.[a-z]{2,3}(?:\.[a-z]{2})?)/' . - '([a-z0-9?=&.%_+-]+)' . // ?query=foo+ + '#(http)://' . + '(' . + // Something Google: http://www.google.com/supported_domains + '(?:[a-z0-9.]+\.)?google\.[a-z]{2,3}(?:\.[a-z]{2})?' . + '|' . + // AltaVista + '(?:[a-z0-9.]+\.)?altavista.com' . + + ')' . + '/' . + '([a-z0-9?=&.%_/\'\\\+-]+)' . // path/?query=foo+bar+ '\bsite:([a-z0-9.%_-]+\.[a-z0-9.%_-]+)' . // site:nasty.example.com //'()' . // Preserve or remove? '#i', @@ -371,7 +376,8 @@ function spam_uri_pickup_preprocess($string = '') return $string; } -// Main function of spam-uri pickup +// Main function of spam-uri pickup, +// A wrapper function of uri_pickup() function spam_uri_pickup($string = '', $method = array()) { if (! is_array($method) || empty($method)) { @@ -418,15 +424,15 @@ function spam_uri_pickup($string = '', $method = array()) // Scheme normalization: Renaming the schemes // snntp://example.org => nntps://example.org // NOTE: Keep the static lists simple. See also port_normalize(). -function scheme_normalize($scheme = '', $considerd_harmfull = TRUE) +function scheme_normalize($scheme = '', $abbrevs_harmfull = TRUE) { - // Abbreviations considerable they don't have link intension + // Abbreviations they have no intention of link static $abbrevs = array( 'ttp' => 'http', 'ttps' => 'https', ); - // Alias => normalized + // Aliases => normalized ones static $aliases = array( 'pop' => 'pop3', 'news' => 'nntp', @@ -437,24 +443,44 @@ function scheme_normalize($scheme = '', $considerd_harmfull = TRUE) 'pops' => 'pop3s', ); - $scheme = strtolower(trim($scheme)); + if (! is_string($scheme)) return ''; + + $scheme = strtolower($scheme); if (isset($abbrevs[$scheme])) { - if ($considerd_harmfull) { - $scheme = $abbrevs[$scheme]; - } else { - $scheme = ''; - } + $scheme = $abbrevs_harmfull ? $abbrevs[$scheme] : ''; + } + if (isset($aliases[$scheme])) { + $scheme = $aliases[$scheme]; } - if (isset($aliases[$scheme])) $scheme = $aliases[$scheme]; return $scheme; } +// Hostname normlization (Destructive) +// www.foo => www.foo ('foo' seems TLD) +// www.foo.bar => foo.bar +// www.10.20 => www.10.20 (Invalid hostname) +// NOTE: +// 'www' is mostly used as traditional hostname of WWW server. +// 'www.foo.bar' may be identical with 'foo.bar'. +function host_normalize($host = '') +{ + if (! is_string($host)) return ''; + + $host = strtolower($host); + $matches = array(); + if (preg_match('/^www\.(.+\.[a-z]+)$/', $host, $matches)) { + return $matches[1]; + } else { + return $host; + } +} + // Port normalization: Suppress the (redundant) default port // HTTP://example.org:80/ => http://example.org/ // HTTP://example.org:8080/ => http://example.org:8080/ // HTTPS://example.org:443/ => https://example.org/ -function port_normalize($port, $scheme, $scheme_normalize = TRUE) +function port_normalize($port, $scheme, $scheme_normalize = FALSE) { // Schemes that users _maybe_ want to add protocol-handlers // to their web browsers. (and attackers _maybe_ want to use ...) @@ -486,9 +512,11 @@ function port_normalize($port, $scheme, $scheme_normalize = TRUE) 'mysql' => 3306, ); - $port = trim($port); - if ($port === '') return $port; + // intval() converts '0-1' to '0', so preg_match() rejects these invalid ones + if (! is_numeric($port) || $port < 0 || preg_match('/[^0-9]/i', $port)) + return ''; + $port = intval($port); if ($scheme_normalize) $scheme = scheme_normalize($scheme); if (isset($array[$scheme]) && $port == $array[$scheme]) $port = ''; // Ignore the defaults @@ -501,21 +529,33 @@ function port_normalize($port, $scheme, $scheme_normalize = TRUE) // http://example.org#hoge => http://example.org/#hoge // http://example.org/path/a/b/./c////./d => http://example.org/path/a/b/c/d // http://example.org/path/../../a/../back => http://example.org/back -function path_normalize($path = '', $divider = '/', $addroot = TRUE) +function path_normalize($path = '', $divider = '/', $add_root = TRUE) { - if (! is_string($path) || $path == '') - return $addroot ? $divider : ''; + if (! is_string($divider)) return is_string($path) ? $path : ''; + + if ($add_root) { + $first_div = & $divider; + } else { + $first_div = ''; + } + if (! is_string($path) || $path == '') return $first_div; + + if (strpos($path, $divider, strlen($path) - strlen($divider)) === FALSE) { + $last_div = ''; + } else { + $last_div = & $divider; + } - $path = trim($path); - $last = ($path[strlen($path) - 1] == $divider) ? $divider : ''; $array = explode($divider, $path); - // Remove paddings + // Remove paddings ('//' and '/./') foreach(array_keys($array) as $key) { - if ($array[$key] == '' || $array[$key] == '.') + if ($array[$key] == '' || $array[$key] == '.') { unset($array[$key]); + } } - // Back-track + + // Remove back-tracks ('/../') $tmp = array(); foreach($array as $value) { if ($value == '..') { @@ -526,56 +566,184 @@ function path_normalize($path = '', $divider = '/', $addroot = TRUE) } $array = & $tmp; - $path = $addroot ? $divider : ''; - if (! empty($array)) $path .= implode($divider, $array) . $last; - - return $path; + if (empty($array)) { + return $first_div; + } else { + return $first_div . implode($divider, $array) . $last_div; + } } // DirectoryIndex normalize (Destructive and rough) -function file_normalize($string = 'index.html.en') +// TODO: sample.en.ja.html.gz => sample.html +function file_normalize($file = 'index.html.en') { - static $array = array( - 'index' => TRUE, // Some system can omit the suffix - 'index.htm' => TRUE, - 'index.html' => TRUE, - 'index.shtml' => TRUE, - 'index.jsp' => TRUE, - 'index.php' => TRUE, - 'index.php3' => TRUE, - 'index.php4' => TRUE, - //'index.pl' => TRUE, - //'index.py' => TRUE, - //'index.rb' => TRUE, - 'index.cgi' => TRUE, + static $simple_defaults = array( 'default.htm' => TRUE, 'default.html' => TRUE, 'default.asp' => TRUE, 'default.aspx' => TRUE, + 'index' => TRUE, // Some system can omit the suffix ); - // Content-negothiation filter: - // Roughly removing ISO 639 -like - // 2-letter suffixes (See RFC3066) - $matches = array(); - if (preg_match('/(.*)\.[a-z][a-z](?:-[a-z][a-z])?$/i', $string, $matches)) { - $_string = $matches[1]; - } else { - $_string = & $string; - } + static $content_suffix = array( + // index.xxx, sample.xxx + 'htm' => TRUE, + 'html' => TRUE, + 'shtml' => TRUE, + 'jsp' => TRUE, + 'php' => TRUE, + 'php3' => TRUE, + 'php4' => TRUE, + 'pl' => TRUE, + 'py' => TRUE, + 'rb' => TRUE, + 'cgi' => TRUE, + 'xml' => TRUE, + ); - if (isset($array[strtolower($_string)])) { - return ''; - } else { - return $string; + static $language_suffix = array( + // Reference: Apache 2.0.59 'AddLanguage' default + 'ca' => TRUE, + 'cs' => TRUE, // cs + 'cz' => TRUE, // cs + 'de' => TRUE, + 'dk' => TRUE, // da + 'el' => TRUE, + 'en' => TRUE, + 'eo' => TRUE, + 'es' => TRUE, + 'et' => TRUE, + 'fr' => TRUE, + 'he' => TRUE, + 'hr' => TRUE, + 'it' => TRUE, + 'ja' => TRUE, + 'ko' => TRUE, + 'ltz' => TRUE, + 'nl' => TRUE, + 'nn' => TRUE, + 'no' => TRUE, + 'po' => TRUE, + 'pt' => TRUE, + 'pt-br' => TRUE, + 'ru' => TRUE, + 'sv' => TRUE, + 'zh-cn' => TRUE, + 'zh-tw' => TRUE, + + // Reference: Apache 2.0.59 default 'index.html' variants + 'ee' => TRUE, + 'lb' => TRUE, + 'var' => TRUE, + ); + + static $charset_suffix = array( + // Reference: Apache 2.0.59 'AddCharset' default + 'iso8859-1' => TRUE, // ISO-8859-1 + 'latin1' => TRUE, // ISO-8859-1 + 'iso8859-2' => TRUE, // ISO-8859-2 + 'latin2' => TRUE, // ISO-8859-2 + 'cen' => TRUE, // ISO-8859-2 + 'iso8859-3' => TRUE, // ISO-8859-3 + 'latin3' => TRUE, // ISO-8859-3 + 'iso8859-4' => TRUE, // ISO-8859-4 + 'latin4' => TRUE, // ISO-8859-4 + 'iso8859-5' => TRUE, // ISO-8859-5 + 'latin5' => TRUE, // ISO-8859-5 + 'cyr' => TRUE, // ISO-8859-5 + 'iso-ru' => TRUE, // ISO-8859-5 + 'iso8859-6' => TRUE, // ISO-8859-6 + 'latin6' => TRUE, // ISO-8859-6 + 'arb' => TRUE, // ISO-8859-6 + 'iso8859-7' => TRUE, // ISO-8859-7 + 'latin7' => TRUE, // ISO-8859-7 + 'grk' => TRUE, // ISO-8859-7 + 'iso8859-8' => TRUE, // ISO-8859-8 + 'latin8' => TRUE, // ISO-8859-8 + 'heb' => TRUE, // ISO-8859-8 + 'iso8859-9' => TRUE, // ISO-8859-9 + 'latin9' => TRUE, // ISO-8859-9 + 'trk' => TRUE, // ISO-8859-9 + 'iso2022-jp'=> TRUE, // ISO-2022-JP + 'jis' => TRUE, // ISO-2022-JP + 'iso2022-kr'=> TRUE, // ISO-2022-KR + 'kis' => TRUE, // ISO-2022-KR + 'iso2022-cn'=> TRUE, // ISO-2022-CN + 'cis' => TRUE, // ISO-2022-CN + 'big5' => TRUE, + 'cp-1251' => TRUE, // ru, WINDOWS-1251 + 'win-1251' => TRUE, // ru, WINDOWS-1251 + 'cp866' => TRUE, // ru + 'koi8-r' => TRUE, // ru, KOI8-r + 'koi8-ru' => TRUE, // ru, KOI8-r + 'koi8-uk' => TRUE, // ru, KOI8-ru + 'ua' => TRUE, // ru, KOI8-ru + 'ucs2' => TRUE, // ru, ISO-10646-UCS-2 + 'ucs4' => TRUE, // ru, ISO-10646-UCS-4 + 'utf8' => TRUE, + + // Reference: Apache 2.0.59 default 'index.html' variants + 'euc-kr' => TRUE, + 'gb2312' => TRUE, + ); + + // May uncompress by web browsers on the fly + // Must be at the last of the filename + // Reference: Apache 2.0.59 'AddEncoding' + static $encoding_suffix = array( + 'z' => TRUE, + 'gz' => TRUE, + ); + + if (! is_string($file)) return ''; + $_file = strtolower($file); + if (isset($simple_defaults[$_file])) return ''; + + + // Roughly removing language/character-set/encoding suffixes + // References: + // * Apache 2 document about 'Content-negotiaton', 'mod_mime' and 'mod_negotiation' + // http://httpd.apache.org/docs/2.0/content-negotiation.html + // http://httpd.apache.org/docs/2.0/mod/mod_mime.html + // http://httpd.apache.org/docs/2.0/mod/mod_negotiation.html + // * http://www.iana.org/assignments/character-sets + // * RFC3066: Tags for the Identification of Languages + // http://www.ietf.org/rfc/rfc3066.txt + // * ISO 639: codes of 'language names' + $suffixes = explode('.', $_file); + $body = array_shift($suffixes); + if ($suffixes) { + // Remove the last .gz/.z + $last_key = end(array_keys($suffixes)); + if (isset($encoding_suffix[$suffixes[$last_key]])) { + unset($suffixes[$last_key]); + } + } + // Cut language and charset suffixes + foreach($suffixes as $key => $value){ + if (isset($language_suffix[$value]) || isset($charset_suffix[$value])) { + unset($suffixes[$key]); + } } + if (empty($suffixes)) return $body; + + // Index.xxx + $count = count($suffixes); + reset($suffixes); + $current = current($suffixes); + if ($body == 'index' && $count == 1 && isset($content_suffix[$current])) return ''; + + return $file; } // Sort query-strings if possible (Destructive and rough) // [OK] &&&&f=d&b&d&c&a=0dd => a=0dd&b&c&d&f=d // [OK] nothing==&eg=dummy&eg=padding&eg=foobar => eg=foobar -function query_normalize($string = '', $equal = FALSE, $equal_cutempty = TRUE) +function query_normalize($string = '', $equal = TRUE, $equal_cutempty = TRUE, $stortolower = TRUE) { + if (! is_string($string)) return ''; + if ($stortolower) $string = strtolower($string); + $array = explode('&', $string); // Remove '&' paddings @@ -614,6 +782,10 @@ function query_normalize($string = '', $equal = FALSE, $equal_cutempty = TRUE) // --------------------- // Part One : Checker +// Rough implementation of globbing +// +// USAGE: $regex = '/^' . generate_glob_regex('*.txt', '/') . '$/i'; +// function generate_glob_regex($string = '', $divider = '/') { static $from = array( @@ -635,98 +807,178 @@ function generate_glob_regex($string = '', $divider = '/') // 23 => ']', ); - if (is_array($string)) { - // Recurse - return '(?:' . - implode('|', // OR - array_map('generate_glob_regex', - $string, - array_pad(array(), count($string), $divider) - ) - ) . - ')'; + if (! is_string($string)) return ''; + + $string = str_replace($from, $mid, $string); // Hide + $string = preg_quote($string, $divider); + $string = str_replace($mid, $to, $string); // Unhide + + return $string; +} + +// Rough hostname checker +// [OK] 192.168. +// TODO: Strict digit, 0x, CIDR, IPv6 +function is_ip($string = '') +{ + if (preg_match('/^' . + '(?:[0-9]{1,3}\.){3}[0-9]{1,3}' . '|' . + '(?:[0-9]{1,3}\.){1,3}' . '$/', + $string)) { + return 4; // Seems IPv4(dot-decimal) } else { - $string = str_replace($from, $mid, $string); // Hide - $string = preg_quote($string, $divider); - $string = str_replace($mid, $to, $string); // Unhide - return $string; + return 0; // Seems not IP } } -function get_blocklist($list = '') +// Generate host (FQDN, IPv4, ...) regex +// 'localhost' : Matches with 'localhost' only +// 'example.org' : Matches with 'example.org' only (See host_normalize() about 'www') +// '.example.org' : Matches with ALL FQDN ended with '.example.org' +// '*.example.org' : Almost the same of '.example.org' except 'www.example.org' +// '10.20.30.40' : Matches with IPv4 address '10.20.30.40' only +// [TODO] '192.' : Matches with all IPv4 hosts started with '192.' +// TODO: IPv4, CIDR?, IPv6 +function generate_host_regex($string = '', $divider = '/') { - static $regex; - - if (! isset($regex)) { - $regex = array(); - - // Sample - if (FALSE) { - $blocklist['badhost'] = array( - //'*', // Deny all uri - //'10.20.*.*', // 10.20.example.com also matches - //'*.blogspot.com', // Blog services subdomains - //array('blogspot.com', '*.blogspot.com') - ); - foreach ($blocklist['badhost'] as $part) { - $_part = is_array($part) ? implode('/', $part) : $part; - $regex['badhost'][$_part] = '/^' . generate_glob_regex($part) . '$/i'; - } + if (! is_string($string)) return ''; + + if (mb_strpos($string, '.') === FALSE) + return generate_glob_regex($string, $divider); + + $result = ''; + if (is_ip($string)) { + // IPv4 + return generate_glob_regex($string, $divider); + } else { + // FQDN or something + $part = explode('.', $string, 2); + if ($part[0] == '') { + $part[0] = '(?:.*\.)?'; // And all related FQDN + } else if ($part[0] == '*') { + $part[0] = '.*\.'; // All subdomains/hosts only + } else { + return generate_glob_regex($string, $divider); } + $part[1] = generate_glob_regex($part[1], $divider); + return implode('', $part); + } +} + +function get_blocklist($list = '') +{ + static $regexs; - // Load + if (! isset($regexs)) { + $regexs = array(); if (file_exists(SPAM_INI_FILE)) { $blocklist = array(); - require(SPAM_INI_FILE); - foreach(array('goodhost', 'badhost') as $key) { - if (! isset($blocklist[$key])) continue; - foreach ($blocklist[$key] as $part) { - $_part = is_array($part) ? implode('/', $part) : $part; - $regex[$key][$_part] = '/^' . generate_glob_regex($part) . '$/i'; + include(SPAM_INI_FILE); + // $blocklist['badhost'] = array( + // '*.blogspot.com', // Blog services's subdomains (only) + // 'IANA-examples' => '#^(?:.*\.)?example\.(?:com|net|org)$#', + // ); + if (isset($blocklist['list'])) { + $regexs['list'] = & $blocklist['list']; + } else { + // Default + $blocklist['list'] = array( + 'goodhost' => FALSE, + 'badhost' => TRUE, + ); + } + foreach(array_keys($blocklist['list']) as $_list) { + if (! isset($blocklist[$_list])) continue; + foreach ($blocklist[$_list] as $key => $value) { + if (is_array($value)) { + $regexs[$_list][$key] = array(); + foreach($value as $_key => $_value) { + get_blocklist_add($regexs[$_list][$key], $_key, $_value); + } + } else { + get_blocklist_add($regexs[$_list], $key, $value); + } } + unset($blocklist[$_list]); } } } if ($list == '') { - return $regex; - } else if (isset($regex[$list])) { - return $regex[$list]; + return $regexs; // ALL + } else if (isset($regexs[$list])) { + return $regexs[$list]; } else { return array(); } } +// Subroutine of get_blocklist() +function get_blocklist_add(& $array, $key = 0, $value = '*.example.org') +{ + if (is_string($key)) { + $array[$key] = & $value; // Treat $value as a regex + } else { + $array[$value] = '/^' . generate_host_regex($value, '/') . '$/i'; + } +} + function is_badhost($hosts = array(), $asap = TRUE, & $remains) { $result = array(); if (! is_array($hosts)) $hosts = array($hosts); foreach(array_keys($hosts) as $key) { - if (! is_string($hosts[$key])) unset($hosts[$key]); - } - if (empty($hosts)) return $result; - - foreach (get_blocklist('goodhost') as $_regex) { - $hosts = preg_grep_invert($_regex, $hosts); + if (! is_string($hosts[$key])) { + unset($hosts[$key]); + } } if (empty($hosts)) return $result; - $tmp = array(); - foreach (get_blocklist('badhost') as $part => $_regex) { - $result[$part] = preg_grep($_regex, $hosts); - if (empty($result[$part])) { - unset($result[$part]); + foreach(get_blocklist('list') as $key=>$value){ + if ($value) { + foreach (get_blocklist($key) as $label => $regex) { + if (is_array($regex)) { + $result[$label] = array(); + foreach($regex as $_label => $_regex) { + if (is_badhost_avail($_label, $_regex, $hosts, $result[$label]) && $asap) { + break; + } + } + if (empty($result[$label])) unset($result[$label]); + } else { + if (is_badhost_avail($label, $regex, $hosts, $result) && $asap) { + break; + } + } + } } else { - $hosts = array_diff($hosts, $result[$part]); - if ($asap) break; + foreach (get_blocklist($key) as $regex) { + $hosts = preg_grep_invert($regex, $hosts); + } + if (empty($hosts)) return $result; } } $remains = $hosts; - return $result; } +// Subroutine for is_badhost() +function is_badhost_avail($label = '*.example.org', $regex = '/^.*\.example\.org$/', & $hosts, & $result) +{ + $group = preg_grep($regex, $hosts); + if ($group) { + + // DEBUG var_dump($group); // badhost detail + + $result[$label] = & $group; + $hosts = array_diff($hosts, $result[$label]); + return TRUE; + } else { + return FALSE; + } +} + // Default (enabled) methods and thresholds (for content insertion) function check_uri_spam_method($times = 1, $t_area = 0, $rule = TRUE) { @@ -785,13 +1037,15 @@ function check_uri_spam($target = '', $method = array()) 'is_spam' => array(), 'method' => & $method, 'remains' => array(), + 'error' => array(), ); $sum = & $progress['sum']; $is_spam = & $progress['is_spam']; $remains = & $progress['remains']; + $error = & $progress['error']; $asap = isset($method['asap']); - // Return if ... + // Recurse if (is_array($target)) { foreach($target as $str) { // Recurse @@ -799,6 +1053,7 @@ function check_uri_spam($target = '', $method = array()) $_sum = & $_progress['sum']; $_is_spam = & $_progress['is_spam']; $_remains = & $_progress['remains']; + $_error = & $_progress['error']; foreach (array_keys($_sum) as $key) { $sum[$key] += $_sum[$key]; } @@ -818,9 +1073,14 @@ function check_uri_spam($target = '', $method = array()) } foreach ($_remains as $key=>$value) { foreach ($value as $_key=>$_value) { - $remains[$key][$_key] = $_value; + if (is_int($_key)) { + $remains[$key][] = $_value; + } else { + $remains[$key][$_key] = $_value; + } } } + if (! empty($_error)) $error += $_error; if ($asap && $is_spam) break; } return $progress; @@ -853,14 +1113,14 @@ function check_uri_spam($target = '', $method = array()) } // Return if ... - if ($asap && $is_spam) { - return $progress; - } - // URI Init - $pickups = spam_uri_pickup($target, $method); - if (empty($pickups)) { - return $progress; - } + if ($asap && $is_spam) return $progress; + + // URI: Pickup + $pickups = uri_pickup_normalize(spam_uri_pickup($target, $method)); + //$remains['uri_pickup'] = & $pickups; + + // Return if ... + if (empty($pickups)) return $progress; // URI: Check quantity $sum['quantity'] += count($pickups); @@ -905,12 +1165,9 @@ function check_uri_spam($target = '', $method = array()) // URI: Uniqueness (and removing non-uniques) if ((! $asap || ! $is_spam) && isset($method['non_uniquri'])) { - // Destructive normalize of URIs - uri_array_normalize($pickups); - $uris = array(); foreach (array_keys($pickups) as $key) { - $uris[$key] = uri_array_implode($pickups[$key]); + $uris[$key] = uri_pickup_implode($pickups[$key]); } $count = count($uris); $uris = array_unique($uris); @@ -928,14 +1185,13 @@ function check_uri_spam($target = '', $method = array()) } // Return if ... - if ($asap && $is_spam) { - return $progress; - } + if ($asap && $is_spam) return $progress; // Host: Uniqueness (uniq / non-uniq) $hosts = array(); foreach ($pickups as $pickup) $hosts[] = & $pickup['host']; $hosts = array_unique($hosts); + //$remains['uniqhost'] = & $hosts; $sum['uniqhost'] += count($hosts); if ((! $asap || ! $is_spam) && isset($method['non_uniqhost'])) { $sum['non_uniqhost'] = $sum['quantity'] - $sum['uniqhost']; @@ -945,26 +1201,23 @@ function check_uri_spam($target = '', $method = array()) } // Return if ... - if ($asap && $is_spam) { - return $progress; - } + if ($asap && $is_spam) return $progress; // URI: Bad host if ((! $asap || ! $is_spam) && isset($method['badhost'])) { $__remains = array(); - if ($asap) { - $badhost = is_badhost($hosts, $asap, $__remains); - } else { - $badhost = is_badhost($hosts, $asap, $__remains); + $badhost = is_badhost($hosts, $asap, $__remains); + if (! $asap) { if ($__remains) { - $progress['remains']['badhost'] = array(); + $remains['badhost'] = array(); foreach ($__remains as $value) { - $progress['remains']['badhost'][$value] = TRUE; + $remains['badhost'][$value] = TRUE; } } } unset($__remains); if (! empty($badhost)) { + //var_dump($badhost); // BADHOST detail $sum['badhost'] += array_count_leaves($badhost); foreach(array_keys($badhost) as $keys) { $is_spam['badhost'][$keys] = @@ -1092,7 +1345,7 @@ function pkwk_spamnotify($action, $page, $target = array('title' => ''), $progre $summary['URI'] = get_script_uri() . '?' . rawurlencode($page); $summary['USER_AGENT'] = TRUE; $summary['REMOTE_ADDR'] = TRUE; - pkwk_mail_notify($notify_subject, var_export($target, TRUE), $summary); + pkwk_mail_notify($notify_subject, var_export($target, TRUE), $summary, TRUE); } ?>