(Separate good/bad hosts from $hosts)
+
+ if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
+ $list = get_blocklist('pre');
+ $blocked = blocklist_distiller($hosts, array_keys($list), $asap);
+ foreach($list as $key => $type){
+ if (! $type) unset($blocked[$key]); // Ignore goodhost etc
+ }
+ unset($list);
+ if (! empty($blocked)) $is_spam['badhost'] = TRUE;
+ }
+
+ // Return if ...
+ if ($asap && $is_spam) return $progress;
+
+ // Remove blocked from $pickups
+ foreach(array_keys($pickups) as $key) {
+ if (! isset($hosts[$key])) {
+ unset($pickups[$key]);
+ }
+ }
+
+ // ----------------------------------------
// URI: Check quantity
+
$sum['quantity'] += count($pickups);
// URI quantity
if ((! $asap || ! $is_spam) && isset($method['quantity']) &&
@@ -1287,7 +433,9 @@ function check_uri_spam($target = '', $method = array())
$is_spam['quantity'] = TRUE;
}
+ // ----------------------------------------
// URI: used inside HTML anchor tag pair
+
if ((! $asap || ! $is_spam) && isset($method['uri_anchor'])) {
$key = 'uri_anchor';
foreach($pickups as $pickup) {
@@ -1303,7 +451,9 @@ function check_uri_spam($target = '', $method = array())
}
}
+ // ----------------------------------------
// URI: used inside 'BBCode' pair
+
if ((! $asap || ! $is_spam) && isset($method['uri_bbcode'])) {
$key = 'uri_bbcode';
foreach($pickups as $pickup) {
@@ -1319,7 +469,9 @@ function check_uri_spam($target = '', $method = array())
}
}
+ // ----------------------------------------
// URI: Uniqueness (and removing non-uniques)
+
if ((! $asap || ! $is_spam) && isset($method['non_uniquri'])) {
$uris = array();
@@ -1344,10 +496,12 @@ function check_uri_spam($target = '', $method = array())
// Return if ...
if ($asap && $is_spam) return $progress;
+ // ----------------------------------------
// Host: Uniqueness (uniq / non-uniq)
- foreach ($pickups as $pickup) $hosts[] = & $pickup['host'];
+
$hosts = array_unique($hosts);
- $sum['uniqhost'] += count($hosts);
+
+ if (isset($sum['uniqhost'])) $sum['uniqhost'] += count($hosts);
if ((! $asap || ! $is_spam) && isset($method['non_uniqhost'])) {
$sum['non_uniqhost'] = $sum['quantity'] - $sum['uniqhost'];
if ($sum['non_uniqhost'] > $method['non_uniqhost']) {
@@ -1358,79 +512,32 @@ function check_uri_spam($target = '', $method = array())
// Return if ...
if ($asap && $is_spam) return $progress;
+ // ----------------------------------------
// URI: Bad host (Separate good/bad hosts from $hosts)
- if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
- // is_badhost()
- $list = get_blocklist('list');
- $blocked = blocklist_distiller($hosts, array_keys($list), $asap);
+ if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
+ $list = get_blocklist('list');
+ $blocked = array_merge_leaves(
+ $blocked,
+ blocklist_distiller($hosts, array_keys($list), $asap),
+ FALSE
+ );
foreach($list as $key=>$type){
if (! $type) unset($blocked[$key]); // Ignore goodhost etc
}
unset($list);
-
if (! empty($blocked)) $is_spam['badhost'] = TRUE;
}
- return $progress;
-}
-
-// Count leaves (A leaf = value that is not an array, or an empty array)
-function array_count_leaves($array = array(), $count_empty = FALSE)
-{
- if (! is_array($array) || (empty($array) && $count_empty)) return 1;
-
- // Recurse
- $count = 0;
- foreach ($array as $part) {
- $count += array_count_leaves($part, $count_empty);
- }
- return $count;
-}
-
-// An array-leaves to a flat array
-function array_flat_leaves($array, $unique = TRUE)
-{
- if (! is_array($array)) return $array;
-
- $tmp = array();
- foreach(array_keys($array) as $key) {
- if (is_array($array[$key])) {
- // Recurse
- foreach(array_flat_leaves($array[$key]) as $_value) {
- $tmp[] = $_value;
- }
- } else {
- $tmp[] = & $array[$key];
- }
- }
-
- return $unique ? array_values(array_unique($tmp)) : $tmp;
-}
+ // Return if ...
+ //if ($asap && $is_spam) return $progress;
-// An array() to an array leaf
-function array_leaf($array = array('A', 'B', 'C.D'), $stem = FALSE, $edge = TRUE)
-{
- if (! is_array($array)) return $array;
-
- $leaf = array();
- $tmp = & $leaf;
- foreach($array as $arg) {
- if (! is_string($arg) && ! is_int($arg)) continue;
- $tmp[$arg] = array();
- $parent = & $tmp;
- $tmp = & $tmp[$arg];
- }
- if ($stem) {
- $parent[key($parent)] = & $edge;
- } else {
- $parent = key($parent);
- }
+ // ----------------------------------------
+ // End
- return $leaf; // array('A' => array('B' => 'C.D'))
+ return $progress;
}
-
// ---------------------
// Reporting
@@ -1491,385 +598,47 @@ function summarize_detail_newtral($progress = array())
! is_array($progress['hosts']) ||
empty($progress['hosts'])) return '';
- $result = '';
- if (FALSE) {
- // Sort by domain
- $tmp = array();
- foreach($progress['hosts'] as $value) {
- $tmp[delimiter_reverse($value)] = $value;
- }
- ksort($tmp, SORT_STRING);
- $result = count($tmp) . ' (' .implode(', ', $tmp) . ')';
- } else {
- $tmp = array();
- foreach($progress['hosts'] as $value) {
- $tmp = array_merge_recursive(
- $tmp,
- array_leaf(explode('.', delimiter_reverse($value)), TRUE, $value)
- );
- }
-
-//var_dump($tmp);
-// TODO: IP address 1.2.3.4 => "0"-3-2-1 by array_shrinkbranch_leaves()
-
- array_shrinkbranch_leaves($tmp, '.', TRUE); // "domain.tld"
- array_joinbranch_leaf($tmp, '.', 0, TRUE);
- foreach($tmp as $key => $value) {
- if (is_array($value)) {
- ksort($tmp[$key], SORT_STRING);
- $tmp[$key] = implode(', ', array_flat_leaves($value));
- }
+ // Generate a responsible $trie
+ $trie = array();
+ foreach($progress['hosts'] as $value) {
+ // 'A.foo.bar.example.com'
+ $resp = whois_responsibility($value); // 'example.com'
+ if (empty($resp)) {
+ // One or more test, or do nothing here
+ $resp = strval($value);
+ $rest = '';
+ } else {
+ $rest = rtrim(substr($value, 0, - strlen($resp)), '.'); // 'A.foo.bar'
}
- ksort($tmp, SORT_STRING);
-
- $result = var_export_shrink($tmp, TRUE, TRUE);
+ $trie = array_merge_leaves($trie, array($resp => array($rest => NULL)), FALSE);
}
- return $result;
-}
-
-// array('F' => array('B' => array('C' => array('d' => array('' => 'foobar')))))
-// to
-// array('F.B.C.d.' => 'foobar')
-function array_joinbranch_leaf(& $array, $delim = '.', $limit = 0, $reverse = FALSE)
-{
+ // Format: var_export_shrink() -like output
$result = array();
- if (! is_array($array)) return $result; // Nothing to do
-
- $limit = max(0, intval($limit));
- $cstack = array();
-
- foreach(array_keys($array) as $key) {
- $kstack = array();
- $k = -1;
-
- $single = array($key => & $array[$key]); // Keep it single
- $cursor = & $single;
- while(is_array($cursor) && count($cursor) == 1) { // Once
- ++$k;
- $kstack[] = key($cursor);
- $cursor = & $cursor[$kstack[$k]];
- if ($limit != 0 && $k == $limit) break;
- }
-
- // Relink
- if ($k != 0) {
- if ($reverse) $kstack = array_reverse($kstack);
- $joinkey = implode($delim, $kstack);
-
- unset($array[$key]);
- $array[$joinkey] = & $cursor;
- $result[$joinkey] = $k + 1; // Key seems not an single array => joined length
- }
- }
-
- return $result;
-}
-
-
-// array('A' => array('B' => 'C')) to
-// array('A.B' => 'C')
-// array(
-// 'A' => array(
-// 'B' => array(
-// 'C' => array(
-// 'D' => '1'
-// ),
-// ),
-// ),
-// 'G' => array(
-// 'H' => '2'
-// ),
-// )
-// to
-// array (
-// 'G.H' => '2',
-// 'A.B.C.D' => '1',
-// )
-function array_shrinkbranch_leaves(& $array, $delim = '.', $reverse = FALSE, $recurse = FALSE)
-{
- $result = 0;
- if (! is_array($array) || empty($array)) return $result;
-
- foreach(array_keys($array) as $key) {
- $branch = & $array[$key];
- if (! is_array($branch) || empty($branch)) continue;
-
- foreach(array_keys($branch) as $bkey) {
- $joinkey = $reverse ?
- $bkey . $delim . $key :
- $key . $delim . $bkey;
- $array[$joinkey] = & $branch[$bkey];
- unset($array[$key]);
- ++$result;
- }
- }
-
- // Rescan (Recurse)
- if ($recurse && $result) {
- $result = array_shrinkbranch_leaves($array, $delim, $reverse, $recurse);
- }
-
- return $result; // Tell me how many
-}
-//$a = array (
-// 'edu' => array (
-// 'berkeley' => array (
-// 'polisci' => array (
-// '' => 'polisci.berkeley.edu',
-// ),
-// ),
-// 'cmich' => array (
-// 'rso' => array (
-// '' => 'rso.cmich.edu',
-// ),
-// ),
-// ),
-//);
-//array_shrinkbranch_leaves($a, '.', TRUE);
-//var_export($a);
-
-//$a = array (
-// '4' => array (
-// '5' => array (
-// '6' => array (
-// '' => '7.8.9',
-// ),
-// ),
-// ),
-//);
-//array_shrinkbranch_leaves($a, '.', TRUE);
-//var_export($a);
-
-
-
-// Check responsibility-root of the FQDN
-// 'foobar.example.co.jp' => 'example.co.jp' (.co.jp seems public)
-// 'foobar.example.act.edu.au' => 'example.act.edu.au' (.act.edu.au seems public)
-// 'foobar.example.com' => 'example.com' (.com seems public)
-function domain_responsibility($fqdn = 'fqdn.foo.bar.example.com', $implicit = TRUE)
-{
- // Domains who have 2nd and/or 3rd level domains
- static $domain = array(
-
- // ccTLD Australia http://www.auda.org.au/ http://www.aunic.net/ http://www.ausregistry.com.au/
- 'au' => array(
- // ".au Second Level Domains" http://www.auda.org.au/domains/
- 'asn' => TRUE,
- 'com' => TRUE,
- 'conf' => TRUE,
- 'csiro' => TRUE,
- 'edu' => array( // http://www.domainname.edu.au/
- 'act' => TRUE,
- 'nt' => TRUE,
- 'nsw' => TRUE,
- 'qld' => TRUE,
- 'sa' => TRUE,
- 'tas' => TRUE,
- 'vic' => TRUE,
- 'wa' => TRUE,
- ),
- 'gov' => array(
- 'act' => TRUE, // Australian Capital Territory
- 'nt' => TRUE, // Northern Territory
- 'nsw' => TRUE, // New South Wales
- 'qld' => TRUE, // Queensland
- 'sa' => TRUE, // South Australia
- 'tas' => TRUE, // Tasmania
- 'vic' => TRUE, // Victoria
- 'wa' => TRUE, // Western Australia
- ),
- 'id' => TRUE,
- 'net' => TRUE,
- 'org' => TRUE,
- 'info' => TRUE,
- ),
-
- // ccTLD Japan http://jprs.co.jp/en/ http://whois.jprs.jp/en/
- 'jp' => array(
- // http://jprs.co.jp/en/jpdomain.html
-
- // Organizational
- 'ac' => TRUE,
- 'ad' => TRUE,
- 'co' => TRUE,
- 'go' => TRUE,
- 'gr' => TRUE,
- 'lg' => TRUE,
- 'ne' => TRUE,
- 'or' => TRUE,
-
- // Geographic
- //
- // Examples for 3rd level domains
- //'kumamoto' => array(
- // // http://www.pref.kumamoto.jp/link/list.asp#4
- // 'amakusa' => TRUE,
- // 'hitoyoshi' => TRUE,
- // 'jonan' => TRUE,
- // 'kumamoto' => TRUE,
- // ...
- //),
- 'aichi' => TRUE,
- 'akita' => TRUE,
- 'aomori' => TRUE,
- 'chiba' => TRUE,
- 'ehime' => TRUE,
- 'fukui' => TRUE,
- 'fukuoka' => TRUE,
- 'fukushima' => TRUE,
- 'gifu' => TRUE,
- 'gunma' => TRUE,
- 'hiroshima' => TRUE,
- 'hokkaido' => TRUE,
- 'hyogo' => TRUE,
- 'ibaraki' => TRUE,
- 'ishikawa' => TRUE,
- 'iwate' => TRUE,
- 'kagawa' => TRUE,
- 'kagoshima' => TRUE,
- 'kanagawa' => TRUE,
- 'kawasaki' => TRUE,
- 'kitakyushu'=> TRUE,
- 'kobe' => TRUE,
- 'kochi' => TRUE,
- 'kumamoto' => TRUE,
- 'kyoto' => TRUE,
- 'mie' => TRUE,
- 'miyagi' => TRUE,
- 'miyazaki' => TRUE,
- 'nagano' => TRUE,
- 'nagasaki' => TRUE,
- 'nagoya' => TRUE,
- 'nara' => TRUE,
- 'niigata' => TRUE,
- 'oita' => TRUE,
- 'okayama' => TRUE,
- 'okinawa' => TRUE,
- 'osaka' => TRUE,
- 'saga' => TRUE,
- 'saitama' => TRUE,
- 'sapporo' => TRUE,
- 'sendai' => TRUE,
- 'shiga' => TRUE,
- 'shimane' => TRUE,
- 'shizuoka' => TRUE,
- 'tochigi' => TRUE,
- 'tokushima' => TRUE,
- 'tokyo' => TRUE,
- 'tottori' => TRUE,
- 'toyama' => TRUE,
- 'wakayama' => TRUE,
- 'yamagata' => TRUE,
- 'yamaguchi' => TRUE,
- 'yamanashi' => TRUE,
- 'yokohama' => TRUE,
- ),
-
- // ccTLD Ukraine http://www.nic.net.ua/ http://whois.com.ua/
- 'ua' => array(
- 'cherkassy' => TRUE, // www.cherkassy.ua
- 'chernigov' => TRUE,
- 'chernovtsy' => TRUE,
- 'ck' => TRUE,
- 'cn' => TRUE,
- 'com' => TRUE,
- 'crimea' => TRUE,
- 'cv' => TRUE,
- 'dn' => TRUE,
- 'dnepropetrovsk' => TRUE,
- 'donetsk' => TRUE,
- 'dp' => TRUE,
- 'edu' => TRUE,
- 'gov' => TRUE,
- 'if' => TRUE,
- 'ivano-frankivsk' => TRUE,
- 'kh' => TRUE,
- 'kharkov' => TRUE,
- 'kherson' => TRUE,
- 'kiev' => TRUE,
- 'kirovograd' => TRUE,
- 'km' => TRUE,
- 'kr' => TRUE,
- 'ks' => TRUE,
- 'lg' => TRUE,
- 'lugansk' => TRUE,
- 'lutsk' => TRUE,
- 'lviv' => TRUE,
- 'mk' => TRUE,
- 'net' => TRUE,
- 'nikolaev' => TRUE,
- 'od' => TRUE,
- 'odessa' => TRUE,
- 'org' => TRUE,
- 'pl' => TRUE,
- 'poltava' => TRUE,
- 'rovno' => TRUE,
- 'rv' => TRUE,
- 'sebastopol' => TRUE,
- 'sumy' => TRUE,
- 'te' => TRUE,
- 'ternopil' => TRUE,
- 'uz' => TRUE,
- 'uzhgorod' => TRUE,
- 'vinnica' => TRUE,
- 'vn' => TRUE,
- 'zaporizhzhe' => TRUE,
- 'zhitomir' => TRUE,
- 'zp' => TRUE,
- 'zt' => TRUE,
- ),
-
- // ccTLD United Kingdom http://www.nic.uk/
- 'uk' => array(
- // http://www.nominet.org.uk/registrants/faq/#available
- 'co' => TRUE,
- 'ltd' => TRUE,
- 'me' => TRUE,
- 'net' => TRUE,
- 'nic' => TRUE,
- 'org' => TRUE,
- 'plc' => TRUE,
- 'sch' => TRUE,
-
- // "Delegated Second Level Domains" http://www.nominet.org.uk/registrants/aboutdomainnames/sld/delegated/
- 'ac' => TRUE,
- 'gov' => TRUE,
- 'mil' => TRUE,
- 'mod' => TRUE,
- 'nhs' => TRUE,
- 'police' => TRUE,
- ),
-
- );
-
- if (! is_string($fqdn)) return '';
-
- $result = array();
- $dcursor = & $domain;
- $array = array_reverse(explode('.', $fqdn));
- $i = 0;
- while(TRUE) {
- $acursor = $array[$i];
- if (is_array($dcursor) && isset($dcursor[$acursor])) {
- $result[] = & $array[$i];
- $dcursor = & $dcursor[$acursor];
+ ksort_by_domain($trie);
+ foreach(array_keys($trie) as $key) {
+ ksort_by_domain($trie[$key]);
+ if (count($trie[$key]) == 1 && key($trie[$key]) == '') {
+ // Just one 'responsibility.example.com'
+ $result[] = ' \'' . $key . '\',';
} else {
- if (isset($acursor)) {
- $result[] = & $array[$i]; // Whois servers must know this subdomain
+ // One subdomain-or-host, or several ones
+ $subs = array();
+ foreach(array_keys($trie[$key]) as $sub) {
+ if ($sub == '') {
+ $subs[] = $key; // 'example.com'
+ } else {
+ $subs[] = $sub . '. '; // 'A.foo.bar. '
+ }
}
- break;
+ $result[] = ' \'' . $key . '\' => \'' . implode(', ', $subs) . '\',';
}
- ++$i;
- }
-
- // Implicit responsibility: Top-Level-Domains must not be yours
- // 'bar.foo.something' => 'foo.something'
- if ($implicit && count($result) == 1 && count($array) > 1) {
- $result[] = & $array[1];
+ unset($trie[$key]);
}
-
- return $result ? implode('.', array_reverse($result)) : '';
+ return
+ 'array (' . "\n" .
+ implode("\n", $result) . "\n" .
+ ')';
}
@@ -1880,21 +649,22 @@ function domain_responsibility($fqdn = 'fqdn.foo.bar.example.com', $implicit = T
function spam_dispose()
{
get_blocklist(NULL);
+ whois_responsibility(NULL);
}
// Common bahavior for blocking
// NOTE: Call this function from various blocking feature, to disgueise the reason 'why blocked'
function spam_exit($mode = '', $data = array())
{
-
$exit = TRUE;
+
switch ($mode) {
case '':
echo("\n");
break;
case 'dump':
echo('' . "\n");
- echo htmlspecialchars(var_export($data, TRUE));
+ echo htmlsc(var_export($data, TRUE));
echo('
' . "\n");
break;
};
@@ -1915,7 +685,14 @@ function pkwk_spamfilter($action, $page, $target = array('title' => ''), $method
if (empty($progress['is_spam'])) {
spam_dispose();
} else {
- $target = string($target, 0); // Removing "\0" etc
+
+// TODO: detect encoding from $target for mbstring functions
+// $tmp = array();
+// foreach(array_keys($target) as $key) {
+// $tmp[strings($key, 0, FALSE, TRUE)] = strings($target[$key], 0, FALSE, TRUE); // Removing "\0" etc
+// }
+// $target = & $tmp;
+
pkwk_spamnotify($action, $page, $target, $progress, $method);
spam_exit($exitmode, $progress);
}