X-Git-Url: http://git.osdn.net/view?a=blobdiff_plain;f=spam%2Fspam.php;h=5681b51da320758a65473ba36fde026aad485fe0;hb=192960fcb58dcb981250871667e112e22958a484;hp=d81880b9831fa1e0031357bf0c40db8eadd2d473;hpb=ea368101c8ce40a473576284616ba801cd66f033;p=pukiwiki%2Fpukiwiki_sandbox.git diff --git a/spam/spam.php b/spam/spam.php index d81880b..5681b51 100644 --- a/spam/spam.php +++ b/spam/spam.php @@ -1,5 +1,5 @@ \n *array \(#', ' => array (', + $result + ); + + if ($ignore_numeric_keys) { + $result =preg_replace( + // Remove numeric keys + '#^( *)[0-9]+ => #m', '$1', + $result + ); + } + + if ($return) { + return $result; + } else { + echo $result; + return NULL; + } +} + +// Remove redundant values from array() +function array_unique_recursive($array = array()) +{ + if (! is_array($array)) return $array; + + $tmp = array(); + foreach($array as $key => $value){ + if (is_array($value)) { + $array[$key] = array_unique_recursive($value); + } else { + if (isset($tmp[$value])) { + unset($array[$key]); + } else { + $tmp[$value] = TRUE; + } + } + } + + return $array; +} + +// Renumber all numeric keys from 0 +function array_renumber_numeric_keys(& $array) +{ + if (! is_array($array)) return $array; + + $count = -1; + $tmp = array(); + foreach($array as $key => $value){ + if (is_array($value)) array_renumber_numeric_keys($array[$key]); // Recurse + if (is_numeric($key)) $tmp[$key] = ++$count; + } + array_rename_keys($array, $tmp); + + return $array; +} + // Roughly strings(1) using PCRE // This function is useful to: // * Reduce the size of data, from removing unprintable binary data @@ -44,40 +111,56 @@ function preg_grep_invert($pattern = '//', $input = array()) // References: // http://www.freebsd.org/cgi/man.cgi?query=strings (Man-page of GNU strings) // http://www.pcre.org/pcre.txt -function strings($binary = '', $min_len = 4, $ignore_space = FALSE) +function strings($binary = '', $min_len = 4, $ignore_space = FALSE, $multibyte = TRUE) { + // String only + $binary = (is_array($binary) || $binary === TRUE) ? '' : strval($binary); + + $regex = $ignore_space ? + '[^[:graph:] \t\n]+' : // Remove "\0" etc, and readable spaces + '[^[:graph:][:space:]]+'; // Preserve readable spaces if possible + + $binary = $multibyte ? + mb_ereg_replace($regex, "\n", $binary) : + preg_replace('/' . $regex . '/s', "\n", $binary); + if ($ignore_space) { $binary = preg_replace( array( - '/(?:[^[:graph:] \t\n]|[\r])+/s', '/[ \t]{2,}/', '/^[ \t]/m', '/[ \t]$/m', ), array( - "\n", ' ', '', '' ), $binary); - } else { - $binary = preg_replace('/(?:[^[:graph:][:space:]]|[\r])+/s', "\n", $binary); } if ($min_len > 1) { + // The last character seems "\n" or not + $br = (! empty($binary) && $binary[strlen($binary) - 1] == "\n") ? "\n" : ''; + $min_len = min(1024, intval($min_len)); - $binary = - implode("\n", - preg_grep('/^.{' . $min_len . ',}/S', - explode("\n", $binary) - ) - ); + $regex = '/^.{' . $min_len . ',}/S'; + $binary = implode("\n", preg_grep($regex, explode("\n", $binary))) . $br; } return $binary; } +// Reverse $string with specified delimiter +function delimiter_reverse($string = 'foo.bar.example.com', $from_delim = '.', $to_delim = '.') +{ + if (! is_string($string) || ! is_string($from_delim) || ! is_string($to_delim)) + return $string; + + // com.example.bar.foo + return implode($to_delim, array_reverse(explode($from_delim, $string))); +} + // --------------------- // URI pickup @@ -96,7 +179,7 @@ function uri_pickup($string = '') preg_match_all( // scheme://userinfo@host:port/path/or/pathinfo/maybefile.and?query=string#fragment // Refer RFC3986 (Regex below is not strict) - '#(\b[a-z][a-z0-9.+-]{1,8}):/+' . // 1: Scheme + '#(\b[a-z][a-z0-9.+-]{1,8}):[/\\\]+' . // 1: Scheme '(?:' . '([^\s<>"\'\[\]/\#?@]*)' . // 2: Userinfo (Username) '@)?' . @@ -104,7 +187,7 @@ function uri_pickup($string = '') // 3: Host '\[[0-9a-f:.]+\]' . '|' . // IPv6([colon-hex and dot]): RFC2732 '(?:[0-9]{1,3}\.){3}[0-9]{1,3}' . '|' . // IPv4(dot-decimal): 001.22.3.44 - '[a-z0-9.-]+' . // hostname(FQDN) : foo.example.org + '[a-z0-9][a-z0-9.-]+[a-z0-9]' . // hostname(FQDN) : foo.example.org ')' . '(?::([0-9]*))?' . // 4: Port '((?:/+[^\s<>"\'\[\]/\#]+)*/+)?' . // 5: Directory path or path-info @@ -370,6 +453,27 @@ function _preg_replace_callback_domain_exposure($matches = array()) return $result; } +// Preprocess: Removing uninterest part for URI detection +function spam_uri_removing_hocus_pocus($binary = '', $method = array()) +{ + $length = 4 ; // 'http'(1) and '://'(2) and 'fqdn'(1) + if (is_array($method)) { + // ''(1) or ''(4) + // '[uri'(4) or ']'(1) or '[/uri]'(6) + if (isset($method['area_anchor']) || isset($method['uri_anchor']) || + isset($method['area_bbcode']) || isset($method['uri_bbcode'])) + $length = 1; // Seems not effective + } + + // Removing sequential spaces and too short lines + $binary = strings($binary, $length, TRUE, TRUE); + + // Remove words (has no '<>[]:') between spaces + $binary = preg_replace('/[ \t][\w.,()\ \t]+[ \t]/', ' ', $binary); + + return $binary; +} + // Preprocess: rawurldecode() and adding space(s) and something // to detect/count some URIs _if possible_ // NOTE: It's maybe danger to var_dump(result). [e.g. 'javascript:'] @@ -377,17 +481,18 @@ function _preg_replace_callback_domain_exposure($matches = array()) // [OK] http://victim.example.org/nasty.example.org // [OK] http://victim.example.org/go?http%3A%2F%2Fnasty.example.org // [OK] http://victim.example.org/http://nasty.example.org -function spam_uri_pickup_preprocess($string = '') +function spam_uri_pickup_preprocess($string = '', $method = array()) { if (! is_string($string)) return ''; - $string = rawurldecode($string); + $string = spam_uri_removing_hocus_pocus(rawurldecode($string), $method); + //var_dump(htmlspecialchars($string)); // Domain exposure (simple) // http://victim.example.org/nasty.example.org/path#frag // => http://nasty.example.org/?refer=victim.example.org and original $string = preg_replace( - '#http://' . + '#h?ttp://' . '(' . 'ime\.nu' . '|' . // 2ch.net 'ime\.st' . '|' . // 2ch.net @@ -395,7 +500,24 @@ function spam_uri_pickup_preprocess($string = '') 'urlx\.org' . ')' . '/([a-z0-9.%_-]+\.[a-z0-9.%_-]+)#i', // nasty.example.org - 'http://$2/?refer=$1 $0', // Insert + 'http://$2/?refer=$1 $0', // Preserve $0 or remove? + $string + ); + + // Domain exposure (gate-big5) + // http://victim.example.org/gate/big5/nasty.example.org/path + // => http://nasty.example.org/?refer=victim.example.org and original + $string = preg_replace( + '#h?ttp://' . + '(' . + 'big5.51job.com' . '|' . + 'big5.china.com' . '|' . + 'big5.xinhuanet.com' . '|' . + ')' . + '/gate/big5' . + '/([a-z0-9.%_-]+\.[a-z0-9.%_-]+)' . + '#i', // nasty.example.org + 'http://$2/?refer=$1 $0', // Preserve $0 or remove? $string ); @@ -443,7 +565,7 @@ function spam_uri_pickup($string = '', $method = array()) $method = check_uri_spam_method(); } - $string = spam_uri_pickup_preprocess($string); + $string = spam_uri_pickup_preprocess($string, $method); $array = uri_pickup($string); @@ -926,10 +1048,15 @@ function generate_host_regex($string = '', $divider = '/') function get_blocklist($list = '') { - static $regexs; + static $regexes; - if (! isset($regexs)) { - $regexs = array(); + if ($list === NULL) { + $regexes = NULL; // Unset + return array(); + } + + if (! isset($regexes)) { + $regexes = array(); if (file_exists(SPAM_INI_FILE)) { $blocklist = array(); include(SPAM_INI_FILE); @@ -938,7 +1065,7 @@ function get_blocklist($list = '') // 'IANA-examples' => '#^(?:.*\.)?example\.(?:com|net|org)$#', // ); if (isset($blocklist['list'])) { - $regexs['list'] = & $blocklist['list']; + $regexes['list'] = & $blocklist['list']; } else { // Default $blocklist['list'] = array( @@ -950,12 +1077,12 @@ function get_blocklist($list = '') if (! isset($blocklist[$_list])) continue; foreach ($blocklist[$_list] as $key => $value) { if (is_array($value)) { - $regexs[$_list][$key] = array(); + $regexes[$_list][$key] = array(); foreach($value as $_key => $_value) { - get_blocklist_add($regexs[$_list][$key], $_key, $_value); + get_blocklist_add($regexes[$_list][$key], $_key, $_value); } } else { - get_blocklist_add($regexs[$_list], $key, $value); + get_blocklist_add($regexes[$_list], $key, $value); } } unset($blocklist[$_list]); @@ -963,11 +1090,11 @@ function get_blocklist($list = '') } } - if ($list == '') { - return $regexs; // ALL - } else if (isset($regexs[$list])) { - return $regexs[$list]; - } else { + if ($list === '') { + return $regexes; // ALL + } else if (isset($regexes[$list])) { + return $regexes[$list]; + } else { return array(); } } @@ -982,7 +1109,6 @@ function get_blocklist_add(& $array, $key = 0, $value = '*.example.org') } } - // Blocklist metrics: Separate $host, to $blocked and not blocked function blocklist_distiller(& $hosts, $keys = array('goodhost', 'badhost'), $asap = FALSE) { @@ -1017,31 +1143,6 @@ function blocklist_distiller(& $hosts, $keys = array('goodhost', 'badhost'), $as return $blocked; } -function is_badhost($hosts = array(), $asap = TRUE, & $remains) -{ - $remains = array(); - - if (! is_array($hosts)) $hosts = array($hosts); - foreach(array_keys($hosts) as $key) { - if (! is_string($hosts[$key])) { - unset($hosts[$key]); - } - } - if (empty($hosts)) return $result; - - $list = get_blocklist('list'); - $blocked = blocklist_distiller($hosts, array_keys($list), $asap); - foreach($list as $key=>$type){ - if (! $type) { - unset($blocked[$key]); // Ignore goodhost etc - } - } - $remains = $hosts; - - return $blocked; -} - - // Default (enabled) methods and thresholds (for content insertion) function check_uri_spam_method($times = 1, $t_area = 0, $rule = TRUE) { @@ -1082,70 +1183,92 @@ function check_uri_spam_method($times = 1, $t_area = 0, $rule = TRUE) // Simple/fast spam check function check_uri_spam($target = '', $method = array()) { - if (! is_array($method) || empty($method)) { - $method = check_uri_spam_method(); - } + // Return value $progress = array( + 'method' => array( + // Theme to do => Dummy, optional value, or optional array() + //'quantity' => 8, + //'uniqhost' => TRUE, + //'non_uniqhost'=> 3, + //'non_uniquri' => 3, + //'badhost' => TRUE, + //'area_anchor' => 0, + //'area_bbcode' => 0, + //'uri_anchor' => 0, + //'uri_bbcode' => 0, + ), 'sum' => array( - 'quantity' => 0, - 'uniqhost' => 0, - 'non_uniqhost'=> 0, - 'non_uniquri' => 0, - 'badhost' => 0, - 'area_anchor' => 0, - 'area_bbcode' => 0, - 'uri_anchor' => 0, - 'uri_bbcode' => 0, + // Theme => Volume found (int) + ), + 'is_spam' => array( + // Flag. If someting defined here, + // one or more spam will be included + // in this report + ), + 'blocked' => array( + // Hosts blocked + //'category' => array( + // 'host', + //) + ), + 'hosts' => array( + // Hosts not blocked ), - 'is_spam' => array(), - 'method' => & $method, - 'remains' => array(), - 'error' => array(), ); + + // Aliases $sum = & $progress['sum']; $is_spam = & $progress['is_spam']; - $remains = & $progress['remains']; - $error = & $progress['error']; + $progress['method'] = & $method; // Argument + $blocked = & $progress['blocked']; + $hosts = & $progress['hosts']; $asap = isset($method['asap']); - // Recurse + // Init + if (! is_array($method) || empty($method)) { + $method = check_uri_spam_method(); + } + foreach(array_keys($method) as $key) { + if (! isset($sum[$key])) $sum[$key] = 0; + } + if (! isset($sum['quantity'])) $sum['quantity'] = 0; + if (is_array($target)) { foreach($target as $str) { - // Recurse - $_progress = check_uri_spam($str, $method); - $_sum = & $_progress['sum']; - $_is_spam = & $_progress['is_spam']; - $_remains = & $_progress['remains']; - $_error = & $_progress['error']; + if (! is_string($str)) continue; + + $_progress = check_uri_spam($str, $method); // Recurse + + // Merge $sum + $_sum = & $_progress['sum']; foreach (array_keys($_sum) as $key) { - $sum[$key] += $_sum[$key]; - } - foreach (array_keys($_is_spam) as $key) { - if (is_array($_is_spam[$key])) { - // Marge keys (badhost) - foreach(array_keys($_is_spam[$key]) as $_key) { - if (! isset($is_spam[$key][$_key])) { - $is_spam[$key][$_key] = $_is_spam[$key][$_key]; - } else { - $is_spam[$key][$_key] += $_is_spam[$key][$_key]; - } - } + if (! isset($sum[$key])) { + $sum[$key] = & $_sum[$key]; } else { - $is_spam[$key] = TRUE; + $sum[$key] += $_sum[$key]; } } - foreach ($_remains as $key=>$value) { - foreach ($value as $_key=>$_value) { - if (is_int($_key)) { - $remains[$key][] = $_value; - } else { - $remains[$key][$_key] = $_value; - } - } + + // Merge $is_spam + $_is_spam = & $_progress['is_spam']; + foreach (array_keys($_is_spam) as $key) { + $is_spam[$key] = TRUE; + if ($asap) break; } - if (! empty($_error)) $error += $_error; if ($asap && $is_spam) break; + + // Merge only + $blocked = array_merge_recursive($blocked, $_progress['blocked']); + $hosts = array_merge_recursive($hosts, $_progress['hosts']); } + + // Unique values + $blocked = array_unique_recursive($blocked); + $hosts = array_unique_recursive($hosts); + + // Recount $sum['badhost'] + $sum['badhost'] = array_count_leaves($blocked); + return $progress; } @@ -1180,7 +1303,6 @@ function check_uri_spam($target = '', $method = array()) // URI: Pickup $pickups = uri_pickup_normalize(spam_uri_pickup($target, $method)); - //$remains['uri_pickup'] = & $pickups; // Return if ... if (empty($pickups)) return $progress; @@ -1251,10 +1373,8 @@ function check_uri_spam($target = '', $method = array()) if ($asap && $is_spam) return $progress; // Host: Uniqueness (uniq / non-uniq) - $hosts = array(); foreach ($pickups as $pickup) $hosts[] = & $pickup['host']; $hosts = array_unique($hosts); - //$remains['uniqhost'] = & $hosts; $sum['uniqhost'] += count($hosts); if ((! $asap || ! $is_spam) && isset($method['non_uniqhost'])) { $sum['non_uniqhost'] = $sum['quantity'] - $sum['uniqhost']; @@ -1266,30 +1386,18 @@ function check_uri_spam($target = '', $method = array()) // Return if ... if ($asap && $is_spam) return $progress; - // URI: Bad host + // URI: Bad host (Separate good/bad hosts from $hosts) if ((! $asap || ! $is_spam) && isset($method['badhost'])) { - $__remains = array(); - $badhost = is_badhost($hosts, $asap, $__remains); - if (! $asap) { - if ($__remains) { - $remains['badhost'] = array(); - foreach ($__remains as $value) { - $remains['badhost'][$value] = TRUE; - } - } - } - unset($__remains); - if (! empty($badhost)) { - //var_dump($badhost); // BADHOST detail - - $sum['badhost'] += array_count_leaves($badhost); - foreach(array_keys($badhost) as $keys) { - $is_spam['badhost'][$keys] = - array_count_leaves($badhost[$keys]); - } - unset($badhost); + // is_badhost() + $list = get_blocklist('list'); + $blocked = blocklist_distiller($hosts, array_keys($list), $asap); + foreach($list as $key=>$type){ + if (! $type) unset($blocked[$key]); // Ignore goodhost etc } + unset($list); + + if (! empty($blocked)) $is_spam['badhost'] = TRUE; } return $progress; @@ -1308,10 +1416,52 @@ function array_count_leaves($array = array(), $count_empty = FALSE) return $count; } +// An array-leaves to a flat array +function array_flat_leaves($array, $unique = TRUE) +{ + if (! is_array($array)) return $array; + + $tmp = array(); + foreach(array_keys($array) as $key) { + if (is_array($array[$key])) { + // Recurse + foreach(array_flat_leaves($array[$key]) as $_value) { + $tmp[] = $_value; + } + } else { + $tmp[] = & $array[$key]; + } + } + + return $unique ? array_values(array_unique($tmp)) : $tmp; +} + +// An array() to an array leaf +function array_leaf($array = array('A', 'B', 'C.D'), $stem = FALSE, $edge = TRUE) +{ + if (! is_array($array)) return $array; + + $leaf = array(); + $tmp = & $leaf; + foreach($array as $arg) { + if (! is_string($arg) && ! is_int($arg)) continue; + $tmp[$arg] = array(); + $parent = & $tmp; + $tmp = & $tmp[$arg]; + } + if ($stem) { + $parent[key($parent)] = & $edge; + } else { + $parent = key($parent); + } + + return $leaf; // array('A' => array('B' => 'C.D')) +} + + // --------------------- // Reporting -// TODO: Don't show unused $method! // Summarize $progress (blocked only) function summarize_spam_progress($progress = array(), $blockedonly = FALSE) { @@ -1322,7 +1472,7 @@ function summarize_spam_progress($progress = array(), $blockedonly = FALSE) $method = & $progress['method']; if (isset($progress['sum'])) { foreach ($progress['sum'] as $key => $value) { - if (isset($method[$key])) { + if (isset($method[$key]) && $value) { $tmp[] = $key . '(' . $value . ')'; } } @@ -1332,15 +1482,575 @@ function summarize_spam_progress($progress = array(), $blockedonly = FALSE) return implode(', ', $tmp); } +function summarize_detail_badhost($progress = array()) +{ + if (! isset($progress['blocked']) || empty($progress['blocked'])) return ''; + + // Flat per group + $blocked = array(); + foreach($progress['blocked'] as $list => $lvalue) { + foreach($lvalue as $group => $gvalue) { + $flat = implode(', ', array_flat_leaves($gvalue)); + if ($flat === $group) { + $blocked[$list][] = $flat; + } else { + $blocked[$list][$group] = $flat; + } + } + } + + // Shrink per list + // From: 'A-1' => array('ie.to') + // To: 'A-1' => 'ie.to' + foreach($blocked as $list => $lvalue) { + if (is_array($lvalue) && + count($lvalue) == 1 && + is_numeric(key($lvalue))) { + $blocked[$list] = current($lvalue); + } + } + + return var_export_shrink($blocked, TRUE, TRUE); +} + +function summarize_detail_newtral($progress = array()) +{ + if (! isset($progress['hosts']) || + ! is_array($progress['hosts']) || + empty($progress['hosts'])) return ''; + + $result = ''; + + // Generate a $trie + $trie = array(); + foreach($progress['hosts'] as $value) { + + // Try to shorten (pre) -- array('example.com', 'bar', 'foo') + $resp = whois_responsibility($value); // 'example.com' + $rest = rtrim(substr($value, 0, - strlen($resp)), '.'); // 'foo.bar' + if ($rest) { + $parts = explode('.', delimiter_reverse('.' . $rest)); + array_unshift($parts, $resp); + } else { + $parts = array($resp, $rest); + } + + $trie = array_merge_recursive( + $trie, + array_leaf($parts, TRUE, $value) + ); + } + + // Try to shorten (post, non-recursive) -- 'foo.bar.example.com' + array_joinbranch_leaf($trie, '.', 0, TRUE); + + // Sort and flatten -- 'A.foo.bar.example.com, B.foo.bar.example.com' + foreach(array_keys($trie) as $key) { + if (is_array($trie[$key])) { + ksort_by_domain($trie[$key]); + $trie[$key] = implode(', ', array_flat_leaves($trie[$key])); + } + } + + // TODO: ltrim('.') from $trie + + ksort_by_domain($trie); + + // TODO: from array('foobar' => 'foobar') to 'foobar' + + return var_export_shrink($trie, TRUE, TRUE); +} + +// ksort() by domain +function ksort_by_domain(& $array) +{ + $sort = array(); + foreach(array_keys($array) as $key) { + $sort[delimiter_reverse($key)] = $key; + } + ksort($sort, SORT_STRING); + $result = array(); + foreach($sort as $key) { + $result[$key] = & $array[$key]; + } + $array = $result; +} + +// array('F' => array('B' => array('C' => array('d' => array('' => 'foobar'))))) +// to +// array('F.B.C.d.' => 'foobar') +function array_joinbranch_leaf(& $array, $delim = '.', $limit = 0, $reverse = FALSE) +{ + $result = array(); + if (! is_array($array)) return $result; // Nothing to do + + $limit = max(0, intval($limit)); + $cstack = array(); + + foreach(array_keys($array) as $key) { + $kstack = array(); + $k = -1; + + $single = array($key => & $array[$key]); // Keep it single + $cursor = & $single; + while(is_array($cursor) && count($cursor) == 1) { // Once + ++$k; + $kstack[] = key($cursor); + $cursor = & $cursor[$kstack[$k]]; + if ($limit != 0 && $k == $limit) break; + } + + // Relink + if ($k != 0) { + if ($reverse) $kstack = array_reverse($kstack); + $joinkey = implode($delim, $kstack); + + unset($array[$key]); + $array[$joinkey] = & $cursor; + $result[$joinkey] = $k + 1; // Key seems not an single array => joined length + } + } + + return $result; +} + + +// Check responsibility-root of the FQDN +// 'foo.bar.example.com' => 'example.com' (.com has the last whois for it) +// 'foo.bar.example.au' => 'example.au' (.au has the last whois for it) +// 'foo.bar.example.edu.au' => 'example.edu.au' (.edu.au has the last whois for it) +// 'foo.bar.example.act.edu.au' => 'example.act.edu.au' (.act.edu.au has the last whois for it) +function whois_responsibility($fqdn = 'foo.bar.example.com', $parent = FALSE, $implicit = TRUE) +{ + // Domains who have 2nd and/or 3rd level domains + static $domain = array( + + // ccTLD: Australia + // http://www.auda.org.au/ + // NIC : http://www.aunic.net/ + // Whois: http://www.ausregistry.com.au/ + 'au' => array( + // .au Second Level Domains + // http://www.auda.org.au/domains/ + 'asn' => TRUE, + 'com' => TRUE, + 'conf' => TRUE, + 'csiro' => TRUE, + 'edu' => array( // http://www.domainname.edu.au/ + // Geographic + 'act' => TRUE, + 'nt' => TRUE, + 'nsw' => TRUE, + 'qld' => TRUE, + 'sa' => TRUE, + 'tas' => TRUE, + 'vic' => TRUE, + 'wa' => TRUE, + ), + 'gov' => array( + // Geographic + 'act' => TRUE, // Australian Capital Territory + 'nt' => TRUE, // Northern Territory + 'nsw' => TRUE, // New South Wales + 'qld' => TRUE, // Queensland + 'sa' => TRUE, // South Australia + 'tas' => TRUE, // Tasmania + 'vic' => TRUE, // Victoria + 'wa' => TRUE, // Western Australia + ), + 'id' => TRUE, + 'net' => TRUE, + 'org' => TRUE, + 'info' => TRUE, + ), + + // ccTLD: China + // NIC : http://www.cnnic.net.cn/en/index/ + // Whois: http://ewhois.cnnic.cn/ + 'cn' => array( + // Provisional Administrative Rules for Registration of Domain Names in China + // http://www.cnnic.net.cn/html/Dir/2003/11/27/1520.htm + + // Organizational + 'ac' => TRUE, + 'com' => TRUE, + 'edu' => TRUE, + 'gov' => TRUE, + 'net' => TRUE, + 'org' => TRUE, + + // Geographic + 'ah' => TRUE, + 'bj' => TRUE, + 'cq' => TRUE, + 'fj' => TRUE, + 'gd' => TRUE, + 'gs' => TRUE, + 'gx' => TRUE, + 'gz' => TRUE, + 'ha' => TRUE, + 'hb' => TRUE, + 'he' => TRUE, + 'hi' => TRUE, + 'hk' => TRUE, + 'hl' => TRUE, + 'hn' => TRUE, + 'jl' => TRUE, + 'js' => TRUE, + 'jx' => TRUE, + 'ln' => TRUE, + 'mo' => TRUE, + 'nm' => TRUE, + 'nx' => TRUE, + 'qh' => TRUE, + 'sc' => TRUE, + 'sd' => TRUE, + 'sh' => TRUE, + 'sn' => TRUE, + 'sx' => TRUE, + 'tj' => TRUE, + 'tw' => TRUE, + 'xj' => TRUE, + 'xz' => TRUE, + 'yn' => TRUE, + 'zj' => TRUE, + ), + + // ccTLD: South Korea + // NIC : http://www.nic.or.kr/english/ + // Whois: http://whois.nida.or.kr/english/ + 'kr' => array( + // .kr domain policy [appendix 1] : Qualifications for Second Level Domains + // http://domain.nida.or.kr/eng/policy.jsp + + // Organizational + 'co' => TRUE, + 'ne ' => TRUE, + 'or ' => TRUE, + 're ' => TRUE, + 'pe' => TRUE, + 'go ' => TRUE, + 'mil' => TRUE, + 'ac' => TRUE, + 'hs' => TRUE, + 'ms' => TRUE, + 'es' => TRUE, + 'sc' => TRUE, + 'kg' => TRUE, + + // Geographic + 'seoul' => TRUE, + 'busan' => TRUE, + 'daegu' => TRUE, + 'incheon' => TRUE, + 'gwangju' => TRUE, + 'daejeon' => TRUE, + 'ulsan' => TRUE, + 'gyeonggi' => TRUE, + 'gangwon' => TRUE, + 'chungbuk' => TRUE, + 'chungnam' => TRUE, + 'jeonbuk' => TRUE, + 'jeonnam' => TRUE, + 'gyeongbuk' => TRUE, + 'gyeongnam' => TRUE, + 'jeju' => TRUE, + ), + + // ccTLD: Japan + // NIC : http://jprs.co.jp/en/ + // Whois: http://whois.jprs.jp/en/ + 'jp' => array( + // Guide to JP Domain Name + // http://jprs.co.jp/en/jpdomain.html + + // Organizational + 'ac' => TRUE, + 'ad' => TRUE, + 'co' => TRUE, + 'go' => TRUE, + 'gr' => TRUE, + 'lg' => TRUE, + 'ne' => TRUE, + 'or' => TRUE, + + // Geographic + // + // Examples for 3rd level domains + //'kumamoto' => array( + // // http://www.pref.kumamoto.jp/link/list.asp#4 + // 'amakusa' => TRUE, + // 'hitoyoshi' => TRUE, + // 'jonan' => TRUE, + // 'kumamoto' => TRUE, + // ... + //), + 'aichi' => TRUE, + 'akita' => TRUE, + 'aomori' => TRUE, + 'chiba' => TRUE, + 'ehime' => TRUE, + 'fukui' => TRUE, + 'fukuoka' => TRUE, + 'fukushima' => TRUE, + 'gifu' => TRUE, + 'gunma' => TRUE, + 'hiroshima' => TRUE, + 'hokkaido' => TRUE, + 'hyogo' => TRUE, + 'ibaraki' => TRUE, + 'ishikawa' => TRUE, + 'iwate' => TRUE, + 'kagawa' => TRUE, + 'kagoshima' => TRUE, + 'kanagawa' => TRUE, + 'kawasaki' => TRUE, + 'kitakyushu'=> TRUE, + 'kobe' => TRUE, + 'kochi' => TRUE, + 'kumamoto' => TRUE, + 'kyoto' => TRUE, + 'mie' => TRUE, + 'miyagi' => TRUE, + 'miyazaki' => TRUE, + 'nagano' => TRUE, + 'nagasaki' => TRUE, + 'nagoya' => TRUE, + 'nara' => TRUE, + 'niigata' => TRUE, + 'oita' => TRUE, + 'okayama' => TRUE, + 'okinawa' => TRUE, + 'osaka' => TRUE, + 'saga' => TRUE, + 'saitama' => TRUE, + 'sapporo' => TRUE, + 'sendai' => TRUE, + 'shiga' => TRUE, + 'shimane' => TRUE, + 'shizuoka' => TRUE, + 'tochigi' => TRUE, + 'tokushima' => TRUE, + 'tokyo' => TRUE, + 'tottori' => TRUE, + 'toyama' => TRUE, + 'wakayama' => TRUE, + 'yamagata' => TRUE, + 'yamaguchi' => TRUE, + 'yamanashi' => TRUE, + 'yokohama' => TRUE, + ), + + // ccTLD: Ukraine + // NIC : http://www.nic.net.ua/ + // Whois: http://whois.com.ua/ + 'ua' => array( + // policy for alternative 2nd level domain names (a2ld) + // http://www.nic.net.ua/doc/a2ld + // http://whois.com.ua/ + 'cherkassy' => TRUE, + 'chernigov' => TRUE, + 'chernovtsy' => TRUE, + 'ck' => TRUE, + 'cn' => TRUE, + 'com' => TRUE, + 'crimea' => TRUE, + 'cv' => TRUE, + 'dn' => TRUE, + 'dnepropetrovsk' => TRUE, + 'donetsk' => TRUE, + 'dp' => TRUE, + 'edu' => TRUE, + 'gov' => TRUE, + 'if' => TRUE, + 'ivano-frankivsk' => TRUE, + 'kh' => TRUE, + 'kharkov' => TRUE, + 'kherson' => TRUE, + 'kiev' => TRUE, + 'kirovograd' => TRUE, + 'km' => TRUE, + 'kr' => TRUE, + 'ks' => TRUE, + 'lg' => TRUE, + 'lugansk' => TRUE, + 'lutsk' => TRUE, + 'lviv' => TRUE, + 'mk' => TRUE, + 'net' => TRUE, + 'nikolaev' => TRUE, + 'od' => TRUE, + 'odessa' => TRUE, + 'org' => TRUE, + 'pl' => TRUE, + 'poltava' => TRUE, + 'rovno' => TRUE, + 'rv' => TRUE, + 'sebastopol' => TRUE, + 'sumy' => TRUE, + 'te' => TRUE, + 'ternopil' => TRUE, + 'uz' => TRUE, + 'uzhgorod' => TRUE, + 'vinnica' => TRUE, + 'vn' => TRUE, + 'zaporizhzhe' => TRUE, + 'zhitomir' => TRUE, + 'zp' => TRUE, + 'zt' => TRUE, + ), + + // ccTLD: United Kingdom + // NIC : http://www.nic.uk/ + 'uk' => array( + // Second Level Domains + // http://www.nic.uk/registrants/aboutdomainnames/sld/ + 'co' => TRUE, + 'ltd' => TRUE, + 'me' => TRUE, + 'net' => TRUE, + 'nic' => TRUE, + 'org' => TRUE, + 'plc' => TRUE, + 'sch' => TRUE, + + // Delegated Second Level Domains + // http://www.nic.uk/registrants/aboutdomainnames/sld/delegated/ + 'ac' => TRUE, + 'gov' => TRUE, + 'mil' => TRUE, + 'mod' => TRUE, + 'nhs' => TRUE, + 'police' => TRUE, + ), + + // ccTLD: United States of America + // NIC : http://nic.us/ + // Whois: http://whois.us/ + 'us' => array( + // See RFC1480 + + // Organizational + 'dni', + 'fed', + 'isa', + 'kids', + 'nsn', + + // Geographical + // United States Postal Service: State abbreviations (for postal codes) + // http://www.usps.com/ncsc/lookups/abbreviations.html + 'ak' => TRUE, // Alaska + 'al' => TRUE, // Alabama + 'ar' => TRUE, // Arkansas + 'as' => TRUE, // American samoa + 'az' => TRUE, // Arizona + 'ca' => TRUE, // California + 'co' => TRUE, // Colorado + 'ct' => TRUE, // Connecticut + 'dc' => TRUE, // District of Columbia + 'de' => TRUE, // Delaware + 'fl' => TRUE, // Florida + 'fm' => TRUE, // Federated states of Micronesia + 'ga' => TRUE, // Georgia + 'gu' => TRUE, // Guam + 'hi' => TRUE, // Hawaii + 'ia' => TRUE, // Iowa + 'id' => TRUE, // Idaho + 'il' => TRUE, // Illinois + 'in' => TRUE, // Indiana + 'ks' => TRUE, // Kansas + 'ky' => TRUE, // Kentucky + 'la' => TRUE, // Louisiana + 'ma' => TRUE, // Massachusetts + 'md' => TRUE, // Maryland + 'me' => TRUE, // Maine + 'mh' => TRUE, // Marshall Islands + 'mi' => TRUE, // Michigan + 'mn' => TRUE, // Minnesota + 'mo' => TRUE, // Missouri + 'mp' => TRUE, // Northern mariana islands + 'ms' => TRUE, // Mississippi + 'mt' => TRUE, // Montana + 'nc' => TRUE, // North Carolina + 'nd' => TRUE, // North Dakota + 'ne' => TRUE, // Nebraska + 'nh' => TRUE, // New Hampshire + 'nj' => TRUE, // New Jersey + 'nm' => TRUE, // New Mexico + 'nv' => TRUE, // Nevada + 'ny' => TRUE, // New York + 'oh' => TRUE, // Ohio + 'ok' => TRUE, // Oklahoma + 'or' => TRUE, // Oregon + 'pa' => TRUE, // Pennsylvania + 'pr' => TRUE, // Puerto Rico + 'pw' => TRUE, // Palau + 'ri' => TRUE, // Rhode Island + 'sc' => TRUE, // South Carolina + 'sd' => TRUE, // South Dakota + 'tn' => TRUE, // Tennessee + 'tx' => TRUE, // Texas + 'ut' => TRUE, // Utah + 'va' => TRUE, // Virginia + 'vi' => TRUE, // Virgin Islands + 'vt' => TRUE, // Vermont + 'wa' => TRUE, // Washington + 'wi' => TRUE, // Wisconsin + 'wv' => TRUE, // West Virginia + 'wy' => TRUE, // Wyoming + ), + ); + + if (! is_string($fqdn)) return ''; + + $result = array(); + $dcursor = & $domain; + $array = array_reverse(explode('.', $fqdn)); + $i = 0; + while(TRUE) { + $acursor = $array[$i]; + if (is_array($dcursor) && isset($dcursor[$acursor])) { + $result[] = & $array[$i]; + $dcursor = & $dcursor[$acursor]; + } else { + if (! $parent && isset($acursor)) { + $result[] = & $array[$i]; // Whois servers must know this subdomain + } + break; + } + ++$i; + } + + // Implicit responsibility: Top-Level-Domains must not be yours + // 'bar.foo.something' => 'foo.something' + if ($implicit && count($result) == 1 && count($array) > 1) { + $result[] = & $array[1]; + } + + return $result ? implode('.', array_reverse($result)) : ''; +} + + // --------------------- // Exit +// Freeing memories +function spam_dispose() +{ + get_blocklist(NULL); +} + // Common bahavior for blocking // NOTE: Call this function from various blocking feature, to disgueise the reason 'why blocked' function spam_exit($mode = '', $data = array()) { + + $exit = TRUE; switch ($mode) { - case '': echo("\n"); break; + case '': + echo("\n"); + break; case 'dump': echo('
' . "\n");
 			echo htmlspecialchars(var_export($data, TRUE));
@@ -1348,8 +2058,7 @@ function spam_exit($mode = '', $data = array())
 			break;
 	};
 
-	// Force exit
-	exit;
+	if ($exit) exit;	// Force exit
 }
 
 
@@ -1362,11 +2071,11 @@ function pkwk_spamfilter($action, $page, $target = array('title' => ''), $method
 {
 	$progress = check_uri_spam($target, $method);
 
-	if (! empty($progress['is_spam'])) {
-		// Mail to administrator(s)
+	if (empty($progress['is_spam'])) {
+		spam_dispose();
+	} else {
+		$target = strings($target, 0);	// Removing "\0" etc
 		pkwk_spamnotify($action, $page, $target, $progress, $method);
-
-		// Exit
 		spam_exit($exitmode, $progress);
 	}
 }
@@ -1387,23 +2096,13 @@ function pkwk_spamnotify($action, $page, $target = array('title' => ''), $progre
 	if (! $asap) {
 		$summary['METRICS'] = summarize_spam_progress($progress);
 	}
-	if (isset($progress['is_spam']['badhost'])) {
-		$badhost = array();
-		foreach($progress['is_spam']['badhost'] as $glob=>$number) {
-			$badhost[] = $glob . '(' . $number . ')';
-		}
-		$summary['DETAIL_BADHOST'] = implode(', ', $badhost);
-	}
-	if (! $asap && $progress['remains']['badhost']) {
-		$count = count($progress['remains']['badhost']);
-		$summary['DETAIL_NEUTRAL_HOST'] = $count .
-			' (' .
-				preg_replace(
-					'/[^, a-z0-9.-]/i', '',
-					implode(', ', array_keys($progress['remains']['badhost']))
-				) .
-			')';
-	}
+
+	$tmp = summarize_detail_badhost($progress);
+	if ($tmp != '') $summary['DETAIL_BADHOST'] = $tmp;
+
+	$tmp = summarize_detail_newtral($progress);
+	if (! $asap && $tmp != '') $summary['DETAIL_NEUTRAL_HOST'] = $tmp;
+
 	$summary['COMMENT'] = $action;
 	$summary['PAGE']    = '[blocked] ' . (is_pagename($page) ? $page : '');
 	$summary['URI']     = get_script_uri() . '?' . rawurlencode($page);