2 // $Id: spam.php,v 1.220 2011/01/24 14:46:45 henoheno Exp $
3 // Copyright (C) 2006-2009 PukiWiki Developers Team
4 // License: GPL v2 or (at your option) any later version
6 // Functions for Concept-work of spam-uri metrics
8 // (PHP 4 >= 4.3.0): preg_match_all(PREG_OFFSET_CAPTURE): $method['uri_XXX'] related feature
11 if (! defined('LIB_DIR')) define('LIB_DIR', './');
12 require(LIB_DIR . 'spam_pickup.php');
13 require(LIB_DIR . 'spam_util.php');
15 if (! defined('SPAM_INI_FILE')) define('SPAM_INI_FILE', 'spam.ini.php');
18 // ---------------------
21 // Rough implementation of globbing
23 // USAGE: $regex = '/^' . generate_glob_regex('*.txt', '/') . '$/i';
25 function generate_glob_regex($string = '', $divider = '/')
30 // 22 => '[', // Maybe cause regex compilation error (e.g. '[]')
46 if (! is_string($string)) return '';
48 $string = str_replace($from, $mid, $string); // Hide
49 $string = preg_quote($string, $divider);
50 $string = str_replace($mid, $to, $string); // Unhide
55 // Generate host (FQDN, IPv4, ...) regex
56 // 'localhost' : Matches with 'localhost' only
57 // 'example.org' : Matches with 'example.org' only (See host_normalize() about 'www')
58 // '.example.org' : Matches with ALL FQDN ended with '.example.org'
59 // '*.example.org' : Almost the same of '.example.org' except 'www.example.org'
60 // '10.20.30.40' : Matches with IPv4 address '10.20.30.40' only
61 // [TODO] '192.' : Matches with all IPv4 hosts started with '192.'
62 // TODO: IPv4, CIDR?, IPv6
63 function generate_host_regex($string = '', $divider = '/')
65 if (! is_string($string)) return '';
67 if (mb_strpos($string, '.') === FALSE || is_ip($string)) {
68 // "localhost", IPv4, etc
69 return generate_glob_regex($string, $divider);
73 $part = explode('.', $string, 2);
76 $part[0] = '(?:.*\.)?';
77 } else if ($part[0] == '*') {
82 return generate_glob_regex($string, $divider);
85 $part[1] = generate_glob_regex($part[1], $divider);
87 return implode('', $part);
91 // ---------------------
94 // Load SPAM_INI_FILE and return parsed one
95 function get_blocklist($list = '')
100 $regexes = NULL; // Unset
104 if (! isset($regexes)) {
106 if (file_exists(SPAM_INI_FILE)) {
107 $blocklist = array();
109 include(SPAM_INI_FILE);
110 // $blocklist['list'] = array(
111 // //'goodhost' => FALSE;
112 // 'badhost' => TRUE;
114 // $blocklist['badhost'] = array(
115 // '*.blogspot.com', // Blog services's subdomains (only)
116 // 'IANA-examples' => '#^(?:.*\.)?example\.(?:com|net|org)$#',
124 if (! isset($blocklist[$special])) continue;
126 $regexes[$special] = $blocklist[$special];
128 foreach(array_keys($blocklist[$special]) as $_list) {
129 if (! isset($blocklist[$_list])) continue;
131 foreach ($blocklist[$_list] as $key => $value) {
132 if (is_array($value)) {
133 $regexes[$_list][$key] = array();
134 foreach($value as $_key => $_value) {
135 get_blocklist_add($regexes[$_list][$key], $_key, $_value);
138 get_blocklist_add($regexes[$_list], $key, $value);
142 unset($blocklist[$_list]);
149 return $regexes; // ALL of
150 } else if (isset($regexes[$list])) {
151 return $regexes[$list]; // A part of
153 return array(); // Found nothing
157 // Subroutine of get_blocklist(): Add new regex to the $array
158 function get_blocklist_add(& $array, $key = 0, $value = '*.example.org/path/to/file.html')
160 if (is_string($key)) {
161 $array[$key] = & $value; // Treat $value as a regex for FQDN(host)s
163 $regex = generate_host_regex($value, '#');
164 if (! empty($regex)) {
165 $array[$value] = '#^' . $regex . '$#i';
170 // Blocklist metrics: Separate $host, to $blocked and not blocked
171 function blocklist_distiller(& $hosts, $keys = array('goodhost', 'badhost'), $asap = FALSE)
173 if (! is_array($hosts)) $hosts = array($hosts);
174 if (! is_array($keys)) $keys = array($keys);
176 $list = get_blocklist('list');
179 foreach($keys as $key){
180 foreach (get_blocklist($key) as $label => $regex) {
181 if (is_array($regex)) {
182 foreach($regex as $_label => $_regex) {
183 $group = preg_grep($_regex, $hosts);
185 $hosts = array_diff($hosts, $group);
186 $blocked[$key][$label][$_label] = $group;
187 if ($asap && $list[$key]) break;
191 $group = preg_grep($regex, $hosts);
193 $hosts = array_diff($hosts, $group);
194 $blocked[$key][$label] = $group;
195 if ($asap && $list[$key]) break;
205 // ---------------------
208 // Default (enabled) methods and thresholds (for content insertion)
209 function check_uri_spam_method($times = 1, $t_area = 0, $rule = TRUE)
211 $times = intval($times);
212 $t_area = intval($t_area);
216 'quantity' => 8 * $times, // Allow N URIs
217 'non_uniqhost' => 3 * $times, // Allow N duped (and normalized) Hosts
218 //'non_uniquri'=> 3 * $times, // Allow N duped (and normalized) URIs
221 'area_anchor' => $t_area, // Using <a href> HTML tag
222 'area_bbcode' => $t_area, // Using [url] or [link] BBCode
223 //'uri_anchor' => $t_area, // URI inside <a href> HTML tag
224 //'uri_bbcode' => $t_area, // URI inside [url] or [link] BBCode
229 //'asap' => TRUE, // Quit or return As Soon As Possible
230 'uniqhost' => TRUE, // Show uniq host (at block notification mail)
231 'badhost' => TRUE, // Check badhost
237 // Remove non-$positive values
238 foreach (array_keys($positive) as $key) {
239 if ($positive[$key] < 0) unset($positive[$key]);
242 return $positive + $bool;
245 // Simple/fast spam check
246 function check_uri_spam($target = '', $method = array())
251 // Theme to do => Dummy, optional value, or optional array()
253 //'uniqhost' => TRUE,
254 //'non_uniqhost'=> 3,
255 //'non_uniquri' => 3,
257 //'area_anchor' => 0,
258 //'area_bbcode' => 0,
263 // Theme => Volume found (int)
266 // Flag. If someting defined here,
267 // one or more spam will be included
272 //'category' => array(
281 // ----------------------------------------
284 $sum = & $progress['sum'];
285 $is_spam = & $progress['is_spam'];
286 $progress['method'] = & $method; // Argument
287 $blocked = & $progress['blocked'];
288 $hosts = & $progress['hosts'];
289 $asap = isset($method['asap']);
291 // ----------------------------------------
294 if (! is_array($method) || empty($method)) {
295 $method = check_uri_spam_method();
297 foreach(array_keys($method) as $key) {
298 if (! isset($sum[$key])) $sum[$key] = 0;
300 if (! isset($sum['quantity'])) $sum['quantity'] = 0;
302 // ----------------------------------------
305 if (is_array($target)) {
306 foreach($target as $str) {
307 if (! is_string($str)) continue;
309 $_progress = check_uri_spam($str, $method); // Recurse
312 $_sum = & $_progress['sum'];
313 foreach (array_keys($_sum) as $key) {
314 if (! isset($sum[$key])) {
315 $sum[$key] = & $_sum[$key];
317 $sum[$key] += $_sum[$key];
322 $_is_spam = & $_progress['is_spam'];
323 foreach (array_keys($_is_spam) as $key) {
324 $is_spam[$key] = TRUE;
327 if ($asap && $is_spam) break;
330 $blocked = array_merge_leaves($blocked, $_progress['blocked'], FALSE);
331 $hosts = array_merge_leaves($hosts, $_progress['hosts'], FALSE);
335 $blocked = array_unique_recursive($blocked);
336 $hosts = array_unique_recursive($hosts);
338 // Recount $sum['badhost']
339 $sum['badhost'] = array_count_leaves($blocked);
344 // ----------------------------------------
347 if (! $asap || ! $is_spam) {
352 'area_anchor', // There's HTML anchor tag
353 'area_bbcode', // There's 'BBCode' linking tag
355 if (isset($method[$key])) $_method[$key] = TRUE;
359 $_asap = isset($method['asap']) ? array('asap' => TRUE) : array();
360 $_result = area_pickup($target, $_method + $_asap);
367 foreach(array_keys($_method) as $key) {
368 if (isset($_result[$key])) {
369 $sum[$key] = $_result[$key];
370 if (isset($method[$key]) && $sum[$key] > $method[$key]) {
371 $is_spam[$key] = TRUE;
377 unset($_asap, $_method, $_result);
381 if ($asap && $is_spam) return $progress;
383 // ----------------------------------------
386 $pickups = spam_uri_pickup($target, $method);
390 if (empty($pickups)) return $progress;
393 $pickups = uri_pickup_normalize($pickups);
395 // ----------------------------------------
396 // Pickup some part of URI
399 foreach ($pickups as $key => $pickup) {
400 $hosts[$key] = & $pickup['host'];
403 // ----------------------------------------
404 // URI: Bad host <pre-filter> (Separate good/bad hosts from $hosts)
406 if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
407 $list = get_blocklist('pre');
408 $blocked = blocklist_distiller($hosts, array_keys($list), $asap);
409 foreach($list as $key => $type){
410 if (! $type) unset($blocked[$key]); // Ignore goodhost etc
413 if (! empty($blocked)) $is_spam['badhost'] = TRUE;
417 if ($asap && $is_spam) return $progress;
419 // Remove blocked from $pickups
420 foreach(array_keys($pickups) as $key) {
421 if (! isset($hosts[$key])) {
422 unset($pickups[$key]);
426 // ----------------------------------------
427 // URI: Check quantity
429 $sum['quantity'] += count($pickups);
431 if ((! $asap || ! $is_spam) && isset($method['quantity']) &&
432 $sum['quantity'] > $method['quantity']) {
433 $is_spam['quantity'] = TRUE;
436 // ----------------------------------------
437 // URI: used inside HTML anchor tag pair
439 if ((! $asap || ! $is_spam) && isset($method['uri_anchor'])) {
441 foreach($pickups as $pickup) {
442 if (isset($pickup['area'][$key])) {
443 $sum[$key] += $pickup['area'][$key];
444 if(isset($method[$key]) &&
445 $sum[$key] > $method[$key]) {
446 $is_spam[$key] = TRUE;
447 if ($asap && $is_spam) break;
449 if ($asap && $is_spam) break;
454 // ----------------------------------------
455 // URI: used inside 'BBCode' pair
457 if ((! $asap || ! $is_spam) && isset($method['uri_bbcode'])) {
459 foreach($pickups as $pickup) {
460 if (isset($pickup['area'][$key])) {
461 $sum[$key] += $pickup['area'][$key];
462 if(isset($method[$key]) &&
463 $sum[$key] > $method[$key]) {
464 $is_spam[$key] = TRUE;
465 if ($asap && $is_spam) break;
467 if ($asap && $is_spam) break;
472 // ----------------------------------------
473 // URI: Uniqueness (and removing non-uniques)
475 if ((! $asap || ! $is_spam) && isset($method['non_uniquri'])) {
478 foreach (array_keys($pickups) as $key) {
479 $uris[$key] = uri_pickup_implode($pickups[$key]);
481 $count = count($uris);
482 $uris = array_unique($uris);
483 $sum['non_uniquri'] += $count - count($uris);
484 if ($sum['non_uniquri'] > $method['non_uniquri']) {
485 $is_spam['non_uniquri'] = TRUE;
487 if (! $asap || ! $is_spam) {
488 foreach (array_diff(array_keys($pickups),
489 array_keys($uris)) as $remove) {
490 unset($pickups[$remove]);
497 if ($asap && $is_spam) return $progress;
499 // ----------------------------------------
500 // Host: Uniqueness (uniq / non-uniq)
502 $hosts = array_unique($hosts);
504 if (isset($sum['uniqhost'])) $sum['uniqhost'] += count($hosts);
505 if ((! $asap || ! $is_spam) && isset($method['non_uniqhost'])) {
506 $sum['non_uniqhost'] = $sum['quantity'] - $sum['uniqhost'];
507 if ($sum['non_uniqhost'] > $method['non_uniqhost']) {
508 $is_spam['non_uniqhost'] = TRUE;
513 if ($asap && $is_spam) return $progress;
515 // ----------------------------------------
516 // URI: Bad host (Separate good/bad hosts from $hosts)
518 if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
519 $list = get_blocklist('list');
520 $blocked = array_merge_leaves(
522 blocklist_distiller($hosts, array_keys($list), $asap),
525 foreach($list as $key=>$type){
526 if (! $type) unset($blocked[$key]); // Ignore goodhost etc
529 if (! empty($blocked)) $is_spam['badhost'] = TRUE;
533 //if ($asap && $is_spam) return $progress;
535 // ----------------------------------------
541 // ---------------------
544 // Summarize $progress (blocked only)
545 function summarize_spam_progress($progress = array(), $blockedonly = FALSE)
548 $tmp = array_keys($progress['is_spam']);
551 $method = & $progress['method'];
552 if (isset($progress['sum'])) {
553 foreach ($progress['sum'] as $key => $value) {
554 if (isset($method[$key]) && $value) {
555 $tmp[] = $key . '(' . $value . ')';
561 return implode(', ', $tmp);
564 function summarize_detail_badhost($progress = array())
566 if (! isset($progress['blocked']) || empty($progress['blocked'])) return '';
570 foreach($progress['blocked'] as $list => $lvalue) {
571 foreach($lvalue as $group => $gvalue) {
572 $flat = implode(', ', array_flat_leaves($gvalue));
573 if ($flat === $group) {
574 $blocked[$list][] = $flat;
576 $blocked[$list][$group] = $flat;
582 // From: 'A-1' => array('ie.to')
583 // To: 'A-1' => 'ie.to'
584 foreach($blocked as $list => $lvalue) {
585 if (is_array($lvalue) &&
586 count($lvalue) == 1 &&
587 is_numeric(key($lvalue))) {
588 $blocked[$list] = current($lvalue);
592 return var_export_shrink($blocked, TRUE, TRUE);
595 function summarize_detail_newtral($progress = array())
597 if (! isset($progress['hosts']) ||
598 ! is_array($progress['hosts']) ||
599 empty($progress['hosts'])) return '';
601 // Generate a responsible $trie
603 foreach($progress['hosts'] as $value) {
604 // 'A.foo.bar.example.com'
605 $resp = whois_responsibility($value); // 'example.com'
607 // One or more test, or do nothing here
608 $resp = strval($value);
611 $rest = rtrim(substr($value, 0, - strlen($resp)), '.'); // 'A.foo.bar'
613 $trie = array_merge_leaves($trie, array($resp => array($rest => NULL)), FALSE);
616 // Format: var_export_shrink() -like output
618 ksort_by_domain($trie);
619 foreach(array_keys($trie) as $key) {
620 ksort_by_domain($trie[$key]);
621 if (count($trie[$key]) == 1 && key($trie[$key]) == '') {
622 // Just one 'responsibility.example.com'
623 $result[] = ' \'' . $key . '\',';
625 // One subdomain-or-host, or several ones
627 foreach(array_keys($trie[$key]) as $sub) {
629 $subs[] = $key; // 'example.com'
631 $subs[] = $sub . '. '; // 'A.foo.bar. '
634 $result[] = ' \'' . $key . '\' => \'' . implode(', ', $subs) . '\',';
640 implode("\n", $result) . "\n" .
645 // ---------------------
649 function spam_dispose()
652 whois_responsibility(NULL);
655 // Common bahavior for blocking
656 // NOTE: Call this function from various blocking feature, to disgueise the reason 'why blocked'
657 function spam_exit($mode = '', $data = array())
666 echo('<pre>' . "\n");
667 echo htmlsc(var_export($data, TRUE));
668 echo('</pre>' . "\n");
672 if ($exit) exit; // Force exit
676 // ---------------------
680 // Simple/fast spam filter ($target: 'a string' or an array())
681 function pkwk_spamfilter($action, $page, $target = array('title' => ''), $method = array(), $exitmode = '')
683 $progress = check_uri_spam($target, $method);
685 if (empty($progress['is_spam'])) {
689 // TODO: detect encoding from $target for mbstring functions
691 // foreach(array_keys($target) as $key) {
692 // $tmp[strings($key, 0, FALSE, TRUE)] = strings($target[$key], 0, FALSE, TRUE); // Removing "\0" etc
696 pkwk_spamnotify($action, $page, $target, $progress, $method);
697 spam_exit($exitmode, $progress);
701 // ---------------------
704 // Mail to administrator(s)
705 function pkwk_spamnotify($action, $page, $target = array('title' => ''), $progress = array(), $method = array())
707 global $notify, $notify_subject;
709 if (! $notify) return;
711 $asap = isset($method['asap']);
713 $summary['ACTION'] = 'Blocked by: ' . summarize_spam_progress($progress, TRUE);
715 $summary['METRICS'] = summarize_spam_progress($progress);
718 $tmp = summarize_detail_badhost($progress);
719 if ($tmp != '') $summary['DETAIL_BADHOST'] = $tmp;
721 $tmp = summarize_detail_newtral($progress);
722 if (! $asap && $tmp != '') $summary['DETAIL_NEUTRAL_HOST'] = $tmp;
724 $summary['COMMENT'] = $action;
725 $summary['PAGE'] = '[blocked] ' . (is_pagename($page) ? $page : '');
726 $summary['URI'] = get_script_uri() . '?' . rawurlencode($page);
727 $summary['USER_AGENT'] = TRUE;
728 $summary['REMOTE_ADDR'] = TRUE;
729 pkwk_mail_notify($notify_subject, var_export($target, TRUE), $summary, TRUE);