(Separate good/bad hosts from $hosts)
+
+ if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
+ $list = get_blocklist('pre');
+ $blocked = blocklist_distiller($hosts, array_keys($list), $asap);
+ foreach($list as $key => $type){
+ if (! $type) unset($blocked[$key]); // Ignore goodhost etc
+ }
+ unset($list);
+ if (! empty($blocked)) $is_spam['badhost'] = TRUE;
+ }
+
+ // Return if ...
+ if ($asap && $is_spam) return $progress;
+
+ // Remove blocked from $pickups
+ foreach(array_keys($pickups) as $key) {
+ if (! isset($hosts[$key])) {
+ unset($pickups[$key]);
+ }
+ }
+
+ // ----------------------------------------
// URI: Check quantity
+
$sum['quantity'] += count($pickups);
// URI quantity
if ((! $asap || ! $is_spam) && isset($method['quantity']) &&
@@ -1317,7 +433,9 @@ function check_uri_spam($target = '', $method = array())
$is_spam['quantity'] = TRUE;
}
+ // ----------------------------------------
// URI: used inside HTML anchor tag pair
+
if ((! $asap || ! $is_spam) && isset($method['uri_anchor'])) {
$key = 'uri_anchor';
foreach($pickups as $pickup) {
@@ -1333,7 +451,9 @@ function check_uri_spam($target = '', $method = array())
}
}
+ // ----------------------------------------
// URI: used inside 'BBCode' pair
+
if ((! $asap || ! $is_spam) && isset($method['uri_bbcode'])) {
$key = 'uri_bbcode';
foreach($pickups as $pickup) {
@@ -1349,7 +469,9 @@ function check_uri_spam($target = '', $method = array())
}
}
+ // ----------------------------------------
// URI: Uniqueness (and removing non-uniques)
+
if ((! $asap || ! $is_spam) && isset($method['non_uniquri'])) {
$uris = array();
@@ -1374,10 +496,12 @@ function check_uri_spam($target = '', $method = array())
// Return if ...
if ($asap && $is_spam) return $progress;
+ // ----------------------------------------
// Host: Uniqueness (uniq / non-uniq)
- foreach ($pickups as $pickup) $hosts[] = & $pickup['host'];
+
$hosts = array_unique($hosts);
- $sum['uniqhost'] += count($hosts);
+
+ if (isset($sum['uniqhost'])) $sum['uniqhost'] += count($hosts);
if ((! $asap || ! $is_spam) && isset($method['non_uniqhost'])) {
$sum['non_uniqhost'] = $sum['quantity'] - $sum['uniqhost'];
if ($sum['non_uniqhost'] > $method['non_uniqhost']) {
@@ -1388,79 +512,32 @@ function check_uri_spam($target = '', $method = array())
// Return if ...
if ($asap && $is_spam) return $progress;
+ // ----------------------------------------
// URI: Bad host (Separate good/bad hosts from $hosts)
- if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
- // is_badhost()
- $list = get_blocklist('list');
- $blocked = blocklist_distiller($hosts, array_keys($list), $asap);
+ if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
+ $list = get_blocklist('list');
+ $blocked = array_merge_leaves(
+ $blocked,
+ blocklist_distiller($hosts, array_keys($list), $asap),
+ FALSE
+ );
foreach($list as $key=>$type){
if (! $type) unset($blocked[$key]); // Ignore goodhost etc
}
unset($list);
-
if (! empty($blocked)) $is_spam['badhost'] = TRUE;
}
- return $progress;
-}
-
-// Count leaves (A leaf = value that is not an array, or an empty array)
-function array_count_leaves($array = array(), $count_empty = FALSE)
-{
- if (! is_array($array) || (empty($array) && $count_empty)) return 1;
-
- // Recurse
- $count = 0;
- foreach ($array as $part) {
- $count += array_count_leaves($part, $count_empty);
- }
- return $count;
-}
-
-// An array-leaves to a flat array
-function array_flat_leaves($array, $unique = TRUE)
-{
- if (! is_array($array)) return $array;
-
- $tmp = array();
- foreach(array_keys($array) as $key) {
- if (is_array($array[$key])) {
- // Recurse
- foreach(array_flat_leaves($array[$key]) as $_value) {
- $tmp[] = $_value;
- }
- } else {
- $tmp[] = & $array[$key];
- }
- }
-
- return $unique ? array_values(array_unique($tmp)) : $tmp;
-}
+ // Return if ...
+ //if ($asap && $is_spam) return $progress;
-// An array() to an array leaf
-function array_leaf($array = array('A', 'B', 'C.D'), $stem = FALSE, $edge = TRUE)
-{
- if (! is_array($array)) return $array;
-
- $leaf = array();
- $tmp = & $leaf;
- foreach($array as $arg) {
- if (! is_string($arg) && ! is_int($arg)) continue;
- $tmp[$arg] = array();
- $parent = & $tmp;
- $tmp = & $tmp[$arg];
- }
- if ($stem) {
- $parent[key($parent)] = & $edge;
- } else {
- $parent = key($parent);
- }
+ // ----------------------------------------
+ // End
- return $leaf; // array('A' => array('B' => 'C.D'))
+ return $progress;
}
-
// ---------------------
// Reporting
@@ -1524,528 +601,46 @@ function summarize_detail_newtral($progress = array())
// Generate a responsible $trie
$trie = array();
foreach($progress['hosts'] as $value) {
-
// 'A.foo.bar.example.com'
$resp = whois_responsibility($value); // 'example.com'
- $rest = rtrim(substr($value, 0, - strlen($resp)), '.') . '.'; // 'A.foo.bar.'
-
- $trie = array_merge_recursive(
- $trie,
- array($resp => array($rest => NULL))
- );
+ if (empty($resp)) {
+ // One or more test, or do nothing here
+ $resp = strval($value);
+ $rest = '';
+ } else {
+ $rest = rtrim(substr($value, 0, - strlen($resp)), '.'); // 'A.foo.bar'
+ }
+ $trie = array_merge_leaves($trie, array($resp => array($rest => NULL)), FALSE);
}
- ksort_by_domain($trie);
+ // Format: var_export_shrink() -like output
$result = array();
+ ksort_by_domain($trie);
foreach(array_keys($trie) as $key) {
-
- // Sort and flatten -- 'A.foo.bar., B.foo.bar.'
ksort_by_domain($trie[$key]);
- $trie[$key] = implode(', ', array_keys($trie[$key]));
-
- // Format: var_export_shrink() -like output
- if ($trie[$key] == '.') {
+ if (count($trie[$key]) == 1 && key($trie[$key]) == '') {
// Just one 'responsibility.example.com'
$result[] = ' \'' . $key . '\',';
} else {
// One subdomain-or-host, or several ones
- $result[] = ' \'' . $key . '\' => \'' . $trie[$key] . '\',';
+ $subs = array();
+ foreach(array_keys($trie[$key]) as $sub) {
+ if ($sub == '') {
+ $subs[] = $key; // 'example.com'
+ } else {
+ $subs[] = $sub . '. '; // 'A.foo.bar. '
+ }
+ }
+ $result[] = ' \'' . $key . '\' => \'' . implode(', ', $subs) . '\',';
}
-
unset($trie[$key]);
}
-
return
'array (' . "\n" .
implode("\n", $result) . "\n" .
')';
}
-// ksort() by domain
-function ksort_by_domain(& $array)
-{
- $sort = array();
- foreach(array_keys($array) as $key) {
- $sort[delimiter_reverse($key)] = $key;
- }
- ksort($sort, SORT_STRING);
- $result = array();
- foreach($sort as $key) {
- $result[$key] = & $array[$key];
- }
- $array = $result;
-}
-
-// Check responsibility-root of the FQDN
-// 'foo.bar.example.com' => 'example.com' (.com has the last whois for it)
-// 'foo.bar.example.au' => 'example.au' (.au has the last whois for it)
-// 'foo.bar.example.edu.au' => 'example.edu.au' (.edu.au has the last whois for it)
-// 'foo.bar.example.act.edu.au' => 'example.act.edu.au' (.act.edu.au has the last whois for it)
-function whois_responsibility($fqdn = 'foo.bar.example.com', $parent = FALSE, $implicit = TRUE)
-{
- // Domains who have 2nd and/or 3rd level domains
- static $domain = array(
-
- // ccTLD: Australia
- // http://www.auda.org.au/
- // NIC : http://www.aunic.net/
- // Whois: http://www.ausregistry.com.au/
- 'au' => array(
- // .au Second Level Domains
- // http://www.auda.org.au/domains/
- 'asn' => TRUE,
- 'com' => TRUE,
- 'conf' => TRUE,
- 'csiro' => TRUE,
- 'edu' => array( // http://www.domainname.edu.au/
- // Geographic
- 'act' => TRUE,
- 'nt' => TRUE,
- 'nsw' => TRUE,
- 'qld' => TRUE,
- 'sa' => TRUE,
- 'tas' => TRUE,
- 'vic' => TRUE,
- 'wa' => TRUE,
- ),
- 'gov' => array(
- // Geographic
- 'act' => TRUE, // Australian Capital Territory
- 'nt' => TRUE, // Northern Territory
- 'nsw' => TRUE, // New South Wales
- 'qld' => TRUE, // Queensland
- 'sa' => TRUE, // South Australia
- 'tas' => TRUE, // Tasmania
- 'vic' => TRUE, // Victoria
- 'wa' => TRUE, // Western Australia
- ),
- 'id' => TRUE,
- 'net' => TRUE,
- 'org' => TRUE,
- 'info' => TRUE,
- ),
-
- // ccTLD: China
- // NIC : http://www.cnnic.net.cn/en/index/
- // Whois: http://ewhois.cnnic.cn/
- 'cn' => array(
- // Provisional Administrative Rules for Registration of Domain Names in China
- // http://www.cnnic.net.cn/html/Dir/2003/11/27/1520.htm
-
- // Organizational
- 'ac' => TRUE,
- 'com' => TRUE,
- 'edu' => TRUE,
- 'gov' => TRUE,
- 'net' => TRUE,
- 'org' => TRUE,
-
- // Geographic
- 'ah' => TRUE,
- 'bj' => TRUE,
- 'cq' => TRUE,
- 'fj' => TRUE,
- 'gd' => TRUE,
- 'gs' => TRUE,
- 'gx' => TRUE,
- 'gz' => TRUE,
- 'ha' => TRUE,
- 'hb' => TRUE,
- 'he' => TRUE,
- 'hi' => TRUE,
- 'hk' => TRUE,
- 'hl' => TRUE,
- 'hn' => TRUE,
- 'jl' => TRUE,
- 'js' => TRUE,
- 'jx' => TRUE,
- 'ln' => TRUE,
- 'mo' => TRUE,
- 'nm' => TRUE,
- 'nx' => TRUE,
- 'qh' => TRUE,
- 'sc' => TRUE,
- 'sd' => TRUE,
- 'sh' => TRUE,
- 'sn' => TRUE,
- 'sx' => TRUE,
- 'tj' => TRUE,
- 'tw' => TRUE,
- 'xj' => TRUE,
- 'xz' => TRUE,
- 'yn' => TRUE,
- 'zj' => TRUE,
- ),
-
- // ccTLD: India
- // NIC : http://www.inregistry.in/
- // Whois: http://www.inregistry.in/whois_search/
- 'in' => array(
- // Policies http://www.inregistry.in/policies/
- 'ac' => TRUE,
- 'co' => TRUE,
- 'firm' => TRUE,
- 'gen' => TRUE,
- 'gov' => TRUE,
- 'ind' => TRUE,
- 'mil' => TRUE,
- 'net' => TRUE,
- 'org' => TRUE,
- 'res' => TRUE,
- // Reserved Names by the government (for the 2nd level)
- // http://www.inregistry.in/policies/reserved_names
- ),
-
- // ccTLD: South Korea
- // NIC : http://www.nic.or.kr/english/
- // Whois: http://whois.nida.or.kr/english/
- 'kr' => array(
- // .kr domain policy [appendix 1] : Qualifications for Second Level Domains
- // http://domain.nida.or.kr/eng/policy.jsp
-
- // Organizational
- 'co' => TRUE,
- 'ne ' => TRUE,
- 'or ' => TRUE,
- 're ' => TRUE,
- 'pe' => TRUE,
- 'go ' => TRUE,
- 'mil' => TRUE,
- 'ac' => TRUE,
- 'hs' => TRUE,
- 'ms' => TRUE,
- 'es' => TRUE,
- 'sc' => TRUE,
- 'kg' => TRUE,
-
- // Geographic
- 'seoul' => TRUE,
- 'busan' => TRUE,
- 'daegu' => TRUE,
- 'incheon' => TRUE,
- 'gwangju' => TRUE,
- 'daejeon' => TRUE,
- 'ulsan' => TRUE,
- 'gyeonggi' => TRUE,
- 'gangwon' => TRUE,
- 'chungbuk' => TRUE,
- 'chungnam' => TRUE,
- 'jeonbuk' => TRUE,
- 'jeonnam' => TRUE,
- 'gyeongbuk' => TRUE,
- 'gyeongnam' => TRUE,
- 'jeju' => TRUE,
- ),
-
- // ccTLD: Japan
- // NIC : http://jprs.co.jp/en/
- // Whois: http://whois.jprs.jp/en/
- 'jp' => array(
- // Guide to JP Domain Name
- // http://jprs.co.jp/en/jpdomain.html
-
- // Organizational
- 'ac' => TRUE,
- 'ad' => TRUE,
- 'co' => TRUE,
- 'go' => TRUE,
- 'gr' => TRUE,
- 'lg' => TRUE,
- 'ne' => TRUE,
- 'or' => TRUE,
-
- // Geographic
- //
- // Examples for 3rd level domains
- //'kumamoto' => array(
- // // http://www.pref.kumamoto.jp/link/list.asp#4
- // 'amakusa' => TRUE,
- // 'hitoyoshi' => TRUE,
- // 'jonan' => TRUE,
- // 'kumamoto' => TRUE,
- // ...
- //),
- 'aichi' => TRUE,
- 'akita' => TRUE,
- 'aomori' => TRUE,
- 'chiba' => TRUE,
- 'ehime' => TRUE,
- 'fukui' => TRUE,
- 'fukuoka' => TRUE,
- 'fukushima' => TRUE,
- 'gifu' => TRUE,
- 'gunma' => TRUE,
- 'hiroshima' => TRUE,
- 'hokkaido' => TRUE,
- 'hyogo' => TRUE,
- 'ibaraki' => TRUE,
- 'ishikawa' => TRUE,
- 'iwate' => TRUE,
- 'kagawa' => TRUE,
- 'kagoshima' => TRUE,
- 'kanagawa' => TRUE,
- 'kawasaki' => TRUE,
- 'kitakyushu'=> TRUE,
- 'kobe' => TRUE,
- 'kochi' => TRUE,
- 'kumamoto' => TRUE,
- 'kyoto' => TRUE,
- 'mie' => TRUE,
- 'miyagi' => TRUE,
- 'miyazaki' => TRUE,
- 'nagano' => TRUE,
- 'nagasaki' => TRUE,
- 'nagoya' => TRUE,
- 'nara' => TRUE,
- 'niigata' => TRUE,
- 'oita' => TRUE,
- 'okayama' => TRUE,
- 'okinawa' => TRUE,
- 'osaka' => TRUE,
- 'saga' => TRUE,
- 'saitama' => TRUE,
- 'sapporo' => TRUE,
- 'sendai' => TRUE,
- 'shiga' => TRUE,
- 'shimane' => TRUE,
- 'shizuoka' => TRUE,
- 'tochigi' => TRUE,
- 'tokushima' => TRUE,
- 'tokyo' => TRUE,
- 'tottori' => TRUE,
- 'toyama' => TRUE,
- 'wakayama' => TRUE,
- 'yamagata' => TRUE,
- 'yamaguchi' => TRUE,
- 'yamanashi' => TRUE,
- 'yokohama' => TRUE,
- ),
-
- // ccTLD: Mexico
- // NIC : http://www.nic.mx/
- // Whois: http://www.nic.mx/es/Busqueda.Who_Is
- 'mx' => array(
- // Politicas Generales de Nombres de Dominio
- // http://www.nic.mx/es/Politicas?CATEGORY=INDICE
- 'com' => TRUE,
- 'edu' => TRUE,
- 'gob' => TRUE,
- 'net' => TRUE,
- 'org' => TRUE,
- ),
-
- // ccTLD: Taiwan
- // NIC : http://www.twnic.net.tw/
- // Whois: http://www.twnic.net.tw/
- 'tw' => array(
- // Guidelines for Administration of Domain Name Registration
- // http://www.twnic.net.tw/english/dn/dn_02.htm
- // II. Types of TWNIC Domain Names and Application Requirements
- // http://www.twnic.net.tw/english/dn/dn_02_b.htm
- 'club' => TRUE,
- 'com' => TRUE,
- 'ebiz' => TRUE,
- 'edu' => TRUE,
- 'game' => TRUE,
- 'gov' => TRUE,
- 'idv' => TRUE,
- 'mil' => TRUE,
- 'net' => TRUE,
- 'org' => TRUE,
- // Reserved words for the 2nd level
- // http://mydn.twnic.net.tw/en/dn02/INDEX.htm
- ),
-
- // ccTLD: Ukraine
- // NIC : http://www.nic.net.ua/
- // Whois: http://whois.com.ua/
- 'ua' => array(
- // policy for alternative 2nd level domain names (a2ld)
- // http://www.nic.net.ua/doc/a2ld
- // http://whois.com.ua/
- 'cherkassy' => TRUE,
- 'chernigov' => TRUE,
- 'chernovtsy' => TRUE,
- 'ck' => TRUE,
- 'cn' => TRUE,
- 'com' => TRUE,
- 'crimea' => TRUE,
- 'cv' => TRUE,
- 'dn' => TRUE,
- 'dnepropetrovsk' => TRUE,
- 'donetsk' => TRUE,
- 'dp' => TRUE,
- 'edu' => TRUE,
- 'gov' => TRUE,
- 'if' => TRUE,
- 'ivano-frankivsk' => TRUE,
- 'kh' => TRUE,
- 'kharkov' => TRUE,
- 'kherson' => TRUE,
- 'kiev' => TRUE,
- 'kirovograd' => TRUE,
- 'km' => TRUE,
- 'kr' => TRUE,
- 'ks' => TRUE,
- 'lg' => TRUE,
- 'lugansk' => TRUE,
- 'lutsk' => TRUE,
- 'lviv' => TRUE,
- 'mk' => TRUE,
- 'net' => TRUE,
- 'nikolaev' => TRUE,
- 'od' => TRUE,
- 'odessa' => TRUE,
- 'org' => TRUE,
- 'pl' => TRUE,
- 'poltava' => TRUE,
- 'rovno' => TRUE,
- 'rv' => TRUE,
- 'sebastopol' => TRUE,
- 'sumy' => TRUE,
- 'te' => TRUE,
- 'ternopil' => TRUE,
- 'uz' => TRUE,
- 'uzhgorod' => TRUE,
- 'vinnica' => TRUE,
- 'vn' => TRUE,
- 'zaporizhzhe' => TRUE,
- 'zhitomir' => TRUE,
- 'zp' => TRUE,
- 'zt' => TRUE,
- ),
-
- // ccTLD: United Kingdom
- // NIC : http://www.nic.uk/
- 'uk' => array(
- // Second Level Domains
- // http://www.nic.uk/registrants/aboutdomainnames/sld/
- 'co' => TRUE,
- 'ltd' => TRUE,
- 'me' => TRUE,
- 'net' => TRUE,
- 'nic' => TRUE,
- 'org' => TRUE,
- 'plc' => TRUE,
- 'sch' => TRUE,
-
- // Delegated Second Level Domains
- // http://www.nic.uk/registrants/aboutdomainnames/sld/delegated/
- 'ac' => TRUE,
- 'gov' => TRUE,
- 'mil' => TRUE,
- 'mod' => TRUE,
- 'nhs' => TRUE,
- 'police' => TRUE,
- ),
-
- // ccTLD: United States of America
- // NIC : http://nic.us/
- // Whois: http://whois.us/
- 'us' => array(
- // See RFC1480
-
- // Organizational
- 'dni',
- 'fed',
- 'isa',
- 'kids',
- 'nsn',
-
- // Geographical
- // United States Postal Service: State abbreviations (for postal codes)
- // http://www.usps.com/ncsc/lookups/abbreviations.html
- 'ak' => TRUE, // Alaska
- 'al' => TRUE, // Alabama
- 'ar' => TRUE, // Arkansas
- 'as' => TRUE, // American samoa
- 'az' => TRUE, // Arizona
- 'ca' => TRUE, // California
- 'co' => TRUE, // Colorado
- 'ct' => TRUE, // Connecticut
- 'dc' => TRUE, // District of Columbia
- 'de' => TRUE, // Delaware
- 'fl' => TRUE, // Florida
- 'fm' => TRUE, // Federated states of Micronesia
- 'ga' => TRUE, // Georgia
- 'gu' => TRUE, // Guam
- 'hi' => TRUE, // Hawaii
- 'ia' => TRUE, // Iowa
- 'id' => TRUE, // Idaho
- 'il' => TRUE, // Illinois
- 'in' => TRUE, // Indiana
- 'ks' => TRUE, // Kansas
- 'ky' => TRUE, // Kentucky
- 'la' => TRUE, // Louisiana
- 'ma' => TRUE, // Massachusetts
- 'md' => TRUE, // Maryland
- 'me' => TRUE, // Maine
- 'mh' => TRUE, // Marshall Islands
- 'mi' => TRUE, // Michigan
- 'mn' => TRUE, // Minnesota
- 'mo' => TRUE, // Missouri
- 'mp' => TRUE, // Northern mariana islands
- 'ms' => TRUE, // Mississippi
- 'mt' => TRUE, // Montana
- 'nc' => TRUE, // North Carolina
- 'nd' => TRUE, // North Dakota
- 'ne' => TRUE, // Nebraska
- 'nh' => TRUE, // New Hampshire
- 'nj' => TRUE, // New Jersey
- 'nm' => TRUE, // New Mexico
- 'nv' => TRUE, // Nevada
- 'ny' => TRUE, // New York
- 'oh' => TRUE, // Ohio
- 'ok' => TRUE, // Oklahoma
- 'or' => TRUE, // Oregon
- 'pa' => TRUE, // Pennsylvania
- 'pr' => TRUE, // Puerto Rico
- 'pw' => TRUE, // Palau
- 'ri' => TRUE, // Rhode Island
- 'sc' => TRUE, // South Carolina
- 'sd' => TRUE, // South Dakota
- 'tn' => TRUE, // Tennessee
- 'tx' => TRUE, // Texas
- 'ut' => TRUE, // Utah
- 'va' => TRUE, // Virginia
- 'vi' => TRUE, // Virgin Islands
- 'vt' => TRUE, // Vermont
- 'wa' => TRUE, // Washington
- 'wi' => TRUE, // Wisconsin
- 'wv' => TRUE, // West Virginia
- 'wy' => TRUE, // Wyoming
- ),
- );
-
- if (! is_string($fqdn)) return '';
-
- $result = array();
- $dcursor = & $domain;
- $array = array_reverse(explode('.', $fqdn));
- $i = 0;
- while(TRUE) {
- $acursor = $array[$i];
- if (is_array($dcursor) && isset($dcursor[$acursor])) {
- $result[] = & $array[$i];
- $dcursor = & $dcursor[$acursor];
- } else {
- if (! $parent && isset($acursor)) {
- $result[] = & $array[$i]; // Whois servers must know this subdomain
- }
- break;
- }
- ++$i;
- }
-
- // Implicit responsibility: Top-Level-Domains must not be yours
- // 'bar.foo.something' => 'foo.something'
- if ($implicit && count($result) == 1 && count($array) > 1) {
- $result[] = & $array[1];
- }
-
- return $result ? implode('.', array_reverse($result)) : '';
-}
-
// ---------------------
// Exit
@@ -2054,6 +649,7 @@ function whois_responsibility($fqdn = 'foo.bar.example.com', $parent = FALSE, $i
function spam_dispose()
{
get_blocklist(NULL);
+ whois_responsibility(NULL);
}
// Common bahavior for blocking
@@ -2068,7 +664,7 @@ function spam_exit($mode = '', $data = array())
break;
case 'dump':
echo('' . "\n");
- echo htmlspecialchars(var_export($data, TRUE));
+ echo htmlsc(var_export($data, TRUE));
echo('' . "\n");
break;
};