2 // $Id: spam.php,v 1.217 2009/01/02 11:55:45 henoheno Exp $
3 // Copyright (C) 2006-2007 PukiWiki Developers Team
4 // License: GPL v2 or (at your option) any later version
6 // Functions for Concept-work of spam-uri metrics
8 // (PHP 4 >= 4.3.0): preg_match_all(PREG_OFFSET_CAPTURE): $method['uri_XXX'] related feature
11 if (! defined('LIB_DIR')) define('LIB_DIR', './');
12 require(LIB_DIR . 'spam_pickup.php');
13 require(LIB_DIR . 'spam_util.php');
15 if (! defined('SPAM_INI_FILE')) define('SPAM_INI_FILE', 'spam.ini.php');
18 // ---------------------
21 // Rough implementation of globbing
23 // USAGE: $regex = '/^' . generate_glob_regex('*.txt', '/') . '$/i';
25 function generate_glob_regex($string = '', $divider = '/')
30 // 22 => '[', // Maybe cause regex compilation error (e.g. '[]')
46 if (! is_string($string)) return '';
48 $string = str_replace($from, $mid, $string); // Hide
49 $string = preg_quote($string, $divider);
50 $string = str_replace($mid, $to, $string); // Unhide
55 // Generate host (FQDN, IPv4, ...) regex
56 // 'localhost' : Matches with 'localhost' only
57 // 'example.org' : Matches with 'example.org' only (See host_normalize() about 'www')
58 // '.example.org' : Matches with ALL FQDN ended with '.example.org'
59 // '*.example.org' : Almost the same of '.example.org' except 'www.example.org'
60 // '10.20.30.40' : Matches with IPv4 address '10.20.30.40' only
61 // [TODO] '192.' : Matches with all IPv4 hosts started with '192.'
62 // TODO: IPv4, CIDR?, IPv6
63 function generate_host_regex($string = '', $divider = '/')
65 if (! is_string($string)) return '';
67 if (mb_strpos($string, '.') === FALSE || is_ip($string)) {
68 // "localhost", IPv4, etc
69 return generate_glob_regex($string, $divider);
73 $part = explode('.', $string, 2);
76 $part[0] = '(?:.*\.)?';
77 } else if ($part[0] == '*') {
82 return generate_glob_regex($string, $divider);
85 $part[1] = generate_glob_regex($part[1], $divider);
87 return implode('', $part);
90 // Load SPAM_INI_FILE and return parsed one
91 function get_blocklist($list = '')
96 $regexes = NULL; // Unset
100 if (! isset($regexes)) {
102 if (file_exists(SPAM_INI_FILE)) {
103 $blocklist = array();
105 include(SPAM_INI_FILE);
106 // $blocklist['list'] = array(
107 // //'goodhost' => FALSE;
108 // 'badhost' => TRUE;
110 // $blocklist['badhost'] = array(
111 // '*.blogspot.com', // Blog services's subdomains (only)
112 // 'IANA-examples' => '#^(?:.*\.)?example\.(?:com|net|org)$#',
120 if (! isset($blocklist[$special])) continue;
122 $regexes[$special] = $blocklist[$special];
124 foreach(array_keys($blocklist[$special]) as $_list) {
125 if (! isset($blocklist[$_list])) continue;
127 foreach ($blocklist[$_list] as $key => $value) {
128 if (is_array($value)) {
129 $regexes[$_list][$key] = array();
130 foreach($value as $_key => $_value) {
131 get_blocklist_add($regexes[$_list][$key], $_key, $_value);
134 get_blocklist_add($regexes[$_list], $key, $value);
138 unset($blocklist[$_list]);
145 return $regexes; // ALL of
146 } else if (isset($regexes[$list])) {
147 return $regexes[$list]; // A part of
149 return array(); // Found nothing
153 // Subroutine of get_blocklist(): Add new regex to the $array
154 function get_blocklist_add(& $array, $key = 0, $value = '*.example.org/path/to/file.html')
156 if (is_string($key)) {
157 $array[$key] = & $value; // Treat $value as a regex for FQDN(host)s
159 $regex = generate_host_regex($value, '#');
160 if (! empty($regex)) {
161 $array[$value] = '#^' . $regex . '$#i';
166 // Blocklist metrics: Separate $host, to $blocked and not blocked
167 function blocklist_distiller(& $hosts, $keys = array('goodhost', 'badhost'), $asap = FALSE)
169 if (! is_array($hosts)) $hosts = array($hosts);
170 if (! is_array($keys)) $keys = array($keys);
172 $list = get_blocklist('list');
175 foreach($keys as $key){
176 foreach (get_blocklist($key) as $label => $regex) {
177 if (is_array($regex)) {
178 foreach($regex as $_label => $_regex) {
179 $group = preg_grep($_regex, $hosts);
181 $hosts = array_diff($hosts, $group);
182 $blocked[$key][$label][$_label] = $group;
183 if ($asap && $list[$key]) break;
187 $group = preg_grep($regex, $hosts);
189 $hosts = array_diff($hosts, $group);
190 $blocked[$key][$label] = $group;
191 if ($asap && $list[$key]) break;
201 // ---------------------
204 // Default (enabled) methods and thresholds (for content insertion)
205 function check_uri_spam_method($times = 1, $t_area = 0, $rule = TRUE)
207 $times = intval($times);
208 $t_area = intval($t_area);
212 'quantity' => 8 * $times, // Allow N URIs
213 'non_uniqhost' => 3 * $times, // Allow N duped (and normalized) Hosts
214 //'non_uniquri'=> 3 * $times, // Allow N duped (and normalized) URIs
217 'area_anchor' => $t_area, // Using <a href> HTML tag
218 'area_bbcode' => $t_area, // Using [url] or [link] BBCode
219 //'uri_anchor' => $t_area, // URI inside <a href> HTML tag
220 //'uri_bbcode' => $t_area, // URI inside [url] or [link] BBCode
225 //'asap' => TRUE, // Quit or return As Soon As Possible
226 'uniqhost' => TRUE, // Show uniq host (at block notification mail)
227 'badhost' => TRUE, // Check badhost
233 // Remove non-$positive values
234 foreach (array_keys($positive) as $key) {
235 if ($positive[$key] < 0) unset($positive[$key]);
238 return $positive + $bool;
241 // Simple/fast spam check
242 function check_uri_spam($target = '', $method = array())
247 // Theme to do => Dummy, optional value, or optional array()
249 //'uniqhost' => TRUE,
250 //'non_uniqhost'=> 3,
251 //'non_uniquri' => 3,
253 //'area_anchor' => 0,
254 //'area_bbcode' => 0,
259 // Theme => Volume found (int)
262 // Flag. If someting defined here,
263 // one or more spam will be included
268 //'category' => array(
277 // ----------------------------------------
280 $sum = & $progress['sum'];
281 $is_spam = & $progress['is_spam'];
282 $progress['method'] = & $method; // Argument
283 $blocked = & $progress['blocked'];
284 $hosts = & $progress['hosts'];
285 $asap = isset($method['asap']);
287 // ----------------------------------------
290 if (! is_array($method) || empty($method)) {
291 $method = check_uri_spam_method();
293 foreach(array_keys($method) as $key) {
294 if (! isset($sum[$key])) $sum[$key] = 0;
296 if (! isset($sum['quantity'])) $sum['quantity'] = 0;
298 // ----------------------------------------
301 if (is_array($target)) {
302 foreach($target as $str) {
303 if (! is_string($str)) continue;
305 $_progress = check_uri_spam($str, $method); // Recurse
308 $_sum = & $_progress['sum'];
309 foreach (array_keys($_sum) as $key) {
310 if (! isset($sum[$key])) {
311 $sum[$key] = & $_sum[$key];
313 $sum[$key] += $_sum[$key];
318 $_is_spam = & $_progress['is_spam'];
319 foreach (array_keys($_is_spam) as $key) {
320 $is_spam[$key] = TRUE;
323 if ($asap && $is_spam) break;
326 $blocked = array_merge_leaves($blocked, $_progress['blocked'], FALSE);
327 $hosts = array_merge_leaves($hosts, $_progress['hosts'], FALSE);
331 $blocked = array_unique_recursive($blocked);
332 $hosts = array_unique_recursive($hosts);
334 // Recount $sum['badhost']
335 $sum['badhost'] = array_count_leaves($blocked);
340 // ----------------------------------------
343 if (! $asap || ! $is_spam) {
348 'area_anchor', // There's HTML anchor tag
349 'area_bbcode', // There's 'BBCode' linking tag
351 if (isset($method[$key])) $_method[$key] = TRUE;
355 $_asap = isset($method['asap']) ? array('asap' => TRUE) : array();
356 $_result = area_pickup($target, $_method + $_asap);
363 foreach(array_keys($_method) as $key) {
364 if (isset($_result[$key])) {
365 $sum[$key] = $_result[$key];
366 if (isset($method[$key]) && $sum[$key] > $method[$key]) {
367 $is_spam[$key] = TRUE;
373 unset($_asap, $_method, $_result);
377 if ($asap && $is_spam) return $progress;
379 // ----------------------------------------
382 $pickups = spam_uri_pickup($target, $method);
386 if (empty($pickups)) return $progress;
389 $pickups = uri_pickup_normalize($pickups);
391 // ----------------------------------------
392 // Pickup some part of URI
395 foreach ($pickups as $key => $pickup) {
396 $hosts[$key] = & $pickup['host'];
399 // ----------------------------------------
400 // URI: Bad host <pre-filter> (Separate good/bad hosts from $hosts)
402 if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
403 $list = get_blocklist('pre');
404 $blocked = blocklist_distiller($hosts, array_keys($list), $asap);
405 foreach($list as $key => $type){
406 if (! $type) unset($blocked[$key]); // Ignore goodhost etc
409 if (! empty($blocked)) $is_spam['badhost'] = TRUE;
413 if ($asap && $is_spam) return $progress;
415 // Remove blocked from $pickups
416 foreach(array_keys($pickups) as $key) {
417 if (! isset($hosts[$key])) {
418 unset($pickups[$key]);
422 // ----------------------------------------
423 // URI: Check quantity
425 $sum['quantity'] += count($pickups);
427 if ((! $asap || ! $is_spam) && isset($method['quantity']) &&
428 $sum['quantity'] > $method['quantity']) {
429 $is_spam['quantity'] = TRUE;
432 // ----------------------------------------
433 // URI: used inside HTML anchor tag pair
435 if ((! $asap || ! $is_spam) && isset($method['uri_anchor'])) {
437 foreach($pickups as $pickup) {
438 if (isset($pickup['area'][$key])) {
439 $sum[$key] += $pickup['area'][$key];
440 if(isset($method[$key]) &&
441 $sum[$key] > $method[$key]) {
442 $is_spam[$key] = TRUE;
443 if ($asap && $is_spam) break;
445 if ($asap && $is_spam) break;
450 // ----------------------------------------
451 // URI: used inside 'BBCode' pair
453 if ((! $asap || ! $is_spam) && isset($method['uri_bbcode'])) {
455 foreach($pickups as $pickup) {
456 if (isset($pickup['area'][$key])) {
457 $sum[$key] += $pickup['area'][$key];
458 if(isset($method[$key]) &&
459 $sum[$key] > $method[$key]) {
460 $is_spam[$key] = TRUE;
461 if ($asap && $is_spam) break;
463 if ($asap && $is_spam) break;
468 // ----------------------------------------
469 // URI: Uniqueness (and removing non-uniques)
471 if ((! $asap || ! $is_spam) && isset($method['non_uniquri'])) {
474 foreach (array_keys($pickups) as $key) {
475 $uris[$key] = uri_pickup_implode($pickups[$key]);
477 $count = count($uris);
478 $uris = array_unique($uris);
479 $sum['non_uniquri'] += $count - count($uris);
480 if ($sum['non_uniquri'] > $method['non_uniquri']) {
481 $is_spam['non_uniquri'] = TRUE;
483 if (! $asap || ! $is_spam) {
484 foreach (array_diff(array_keys($pickups),
485 array_keys($uris)) as $remove) {
486 unset($pickups[$remove]);
493 if ($asap && $is_spam) return $progress;
495 // ----------------------------------------
496 // Host: Uniqueness (uniq / non-uniq)
498 $hosts = array_unique($hosts);
500 if (isset($sum['uniqhost'])) $sum['uniqhost'] += count($hosts);
501 if ((! $asap || ! $is_spam) && isset($method['non_uniqhost'])) {
502 $sum['non_uniqhost'] = $sum['quantity'] - $sum['uniqhost'];
503 if ($sum['non_uniqhost'] > $method['non_uniqhost']) {
504 $is_spam['non_uniqhost'] = TRUE;
509 if ($asap && $is_spam) return $progress;
511 // ----------------------------------------
512 // URI: Bad host (Separate good/bad hosts from $hosts)
514 if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
515 $list = get_blocklist('list');
516 $blocked = array_merge_leaves(
518 blocklist_distiller($hosts, array_keys($list), $asap),
521 foreach($list as $key=>$type){
522 if (! $type) unset($blocked[$key]); // Ignore goodhost etc
525 if (! empty($blocked)) $is_spam['badhost'] = TRUE;
529 //if ($asap && $is_spam) return $progress;
531 // ----------------------------------------
537 // ---------------------
540 // Summarize $progress (blocked only)
541 function summarize_spam_progress($progress = array(), $blockedonly = FALSE)
544 $tmp = array_keys($progress['is_spam']);
547 $method = & $progress['method'];
548 if (isset($progress['sum'])) {
549 foreach ($progress['sum'] as $key => $value) {
550 if (isset($method[$key]) && $value) {
551 $tmp[] = $key . '(' . $value . ')';
557 return implode(', ', $tmp);
560 function summarize_detail_badhost($progress = array())
562 if (! isset($progress['blocked']) || empty($progress['blocked'])) return '';
566 foreach($progress['blocked'] as $list => $lvalue) {
567 foreach($lvalue as $group => $gvalue) {
568 $flat = implode(', ', array_flat_leaves($gvalue));
569 if ($flat === $group) {
570 $blocked[$list][] = $flat;
572 $blocked[$list][$group] = $flat;
578 // From: 'A-1' => array('ie.to')
579 // To: 'A-1' => 'ie.to'
580 foreach($blocked as $list => $lvalue) {
581 if (is_array($lvalue) &&
582 count($lvalue) == 1 &&
583 is_numeric(key($lvalue))) {
584 $blocked[$list] = current($lvalue);
588 return var_export_shrink($blocked, TRUE, TRUE);
591 function summarize_detail_newtral($progress = array())
593 if (! isset($progress['hosts']) ||
594 ! is_array($progress['hosts']) ||
595 empty($progress['hosts'])) return '';
597 // Generate a responsible $trie
599 foreach($progress['hosts'] as $value) {
600 // 'A.foo.bar.example.com'
601 $resp = whois_responsibility($value); // 'example.com'
603 // One or more test, or do nothing here
604 $resp = strval($value);
607 $rest = rtrim(substr($value, 0, - strlen($resp)), '.'); // 'A.foo.bar'
609 $trie = array_merge_leaves($trie, array($resp => array($rest => NULL)), FALSE);
612 // Format: var_export_shrink() -like output
614 ksort_by_domain($trie);
615 foreach(array_keys($trie) as $key) {
616 ksort_by_domain($trie[$key]);
617 if (count($trie[$key]) == 1 && key($trie[$key]) == '') {
618 // Just one 'responsibility.example.com'
619 $result[] = ' \'' . $key . '\',';
621 // One subdomain-or-host, or several ones
623 foreach(array_keys($trie[$key]) as $sub) {
625 $subs[] = $key; // 'example.com'
627 $subs[] = $sub . '. '; // 'A.foo.bar. '
630 $result[] = ' \'' . $key . '\' => \'' . implode(', ', $subs) . '\',';
636 implode("\n", $result) . "\n" .
641 // ---------------------
645 function spam_dispose()
648 whois_responsibility(NULL);
651 // Common bahavior for blocking
652 // NOTE: Call this function from various blocking feature, to disgueise the reason 'why blocked'
653 function spam_exit($mode = '', $data = array())
662 echo('<pre>' . "\n");
663 echo htmlspecialchars(var_export($data, TRUE));
664 echo('</pre>' . "\n");
668 if ($exit) exit; // Force exit
672 // ---------------------
676 // Simple/fast spam filter ($target: 'a string' or an array())
677 function pkwk_spamfilter($action, $page, $target = array('title' => ''), $method = array(), $exitmode = '')
679 $progress = check_uri_spam($target, $method);
681 if (empty($progress['is_spam'])) {
685 // TODO: detect encoding from $target for mbstring functions
687 // foreach(array_keys($target) as $key) {
688 // $tmp[strings($key, 0, FALSE, TRUE)] = strings($target[$key], 0, FALSE, TRUE); // Removing "\0" etc
692 pkwk_spamnotify($action, $page, $target, $progress, $method);
693 spam_exit($exitmode, $progress);
697 // ---------------------
700 // Mail to administrator(s)
701 function pkwk_spamnotify($action, $page, $target = array('title' => ''), $progress = array(), $method = array())
703 global $notify, $notify_subject;
705 if (! $notify) return;
707 $asap = isset($method['asap']);
709 $summary['ACTION'] = 'Blocked by: ' . summarize_spam_progress($progress, TRUE);
711 $summary['METRICS'] = summarize_spam_progress($progress);
714 $tmp = summarize_detail_badhost($progress);
715 if ($tmp != '') $summary['DETAIL_BADHOST'] = $tmp;
717 $tmp = summarize_detail_newtral($progress);
718 if (! $asap && $tmp != '') $summary['DETAIL_NEUTRAL_HOST'] = $tmp;
720 $summary['COMMENT'] = $action;
721 $summary['PAGE'] = '[blocked] ' . (is_pagename($page) ? $page : '');
722 $summary['URI'] = get_script_uri() . '?' . rawurlencode($page);
723 $summary['USER_AGENT'] = TRUE;
724 $summary['REMOTE_ADDR'] = TRUE;
725 pkwk_mail_notify($notify_subject, var_export($target, TRUE), $summary, TRUE);