2 // $Id: spam.php,v 1.215 2009/01/02 10:37:47 henoheno Exp $
3 // Copyright (C) 2006-2007 PukiWiki Developers Team
4 // License: GPL v2 or (at your option) any later version
6 // Functions for Concept-work of spam-uri metrics
8 // (PHP 4 >= 4.3.0): preg_match_all(PREG_OFFSET_CAPTURE): $method['uri_XXX'] related feature
10 if (! defined('SPAM_INI_FILE')) define('SPAM_INI_FILE', 'spam.ini.php');
12 // ---------------------
15 // (PHP 4 >= 4.2.0): var_export(): mail-reporting and dump related
16 if (! function_exists('var_export')) {
17 function var_export() {
18 return 'var_export() is not found on this server' . "\n";
22 // (PHP 4 >= 4.2.0): preg_grep() enables invert option
23 function preg_grep_invert($pattern = '//', $input = array())
26 if (! isset($invert)) $invert = defined('PREG_GREP_INVERT');
29 return preg_grep($pattern, $input, PREG_GREP_INVERT);
31 $result = preg_grep($pattern, $input);
33 return array_diff($input, preg_grep($pattern, $input));
41 // ---------------------
44 // Very roughly, shrink the lines of var_export()
45 // NOTE: If the same data exists, it must be corrupted.
46 function var_export_shrink($expression, $return = FALSE, $ignore_numeric_keys = FALSE)
48 $result = var_export($expression, TRUE);
50 $result = preg_replace(
51 // Remove a newline and spaces
52 '# => \n *array \(#', ' => array (',
56 if ($ignore_numeric_keys) {
57 $result =preg_replace(
58 // Remove numeric keys
59 '#^( *)[0-9]+ => #m', '$1',
72 // Data structure: Create an array they _refer_only_one_ value
73 function one_value_array($num = 0, $value = NULL)
75 $num = max(0, intval($num));
78 for ($i = 0; $i < $num; $i++) {
85 // Reverse $string with specified delimiter
86 function delimiter_reverse($string = 'foo.bar.example.com', $from_delim = '.', $to_delim = NULL)
88 $to_null = ($to_delim === NULL);
90 if (! is_string($from_delim) || (! $to_null && ! is_string($to_delim))) {
93 if (is_array($string)) {
95 $count = count($string);
96 $from = one_value_array($count, $from_delim);
98 // Note: array_map() vanishes all keys
99 return array_map('delimiter_reverse', $string, $from);
101 $to = one_value_array($count, $to_delim);
102 // Note: array_map() vanishes all keys
103 return array_map('delimiter_reverse', $string, $from, $to);
106 if (! is_string($string)) {
110 // Returns com.example.bar.foo
111 if ($to_null) $to_delim = & $from_delim;
112 return implode($to_delim, array_reverse(explode($from_delim, $string)));
116 function ksort_by_domain(& $array)
119 foreach(array_keys($array) as $key) {
120 $reversed = delimiter_reverse($key);
121 if ($reversed !== FALSE) {
122 $sort[$reversed] = $key;
125 ksort($sort, SORT_STRING);
128 foreach($sort as $key) {
129 $result[$key] = & $array[$key];
135 // Roughly strings(1) using PCRE
136 // This function is useful to:
137 // * Reduce the size of data, from removing unprintable binary data
138 // * Detect _bare_strings_ from binary data
140 // http://www.freebsd.org/cgi/man.cgi?query=strings (Man-page of GNU strings)
141 // http://www.pcre.org/pcre.txt
142 // Note: mb_ereg_replace() is one of mbstring extension's functions
143 // and need to init its encoding.
144 function strings($binary = '', $min_len = 4, $ignore_space = FALSE, $multibyte = FALSE)
147 $binary = (is_array($binary) || $binary === TRUE) ? '' : strval($binary);
149 $regex = $ignore_space ?
150 '[^[:graph:] \t\n]+' : // Remove "\0" etc, and readable spaces
151 '[^[:graph:][:space:]]+'; // Preserve readable spaces if possible
153 $binary = $multibyte ?
154 mb_ereg_replace($regex, "\n", $binary) :
155 preg_replace('/' . $regex . '/s', "\n", $binary);
158 $binary = preg_replace(
173 // The last character seems "\n" or not
174 $br = (! empty($binary) && $binary[strlen($binary) - 1] == "\n") ? "\n" : '';
176 $min_len = min(1024, intval($min_len));
177 $regex = '/^.{' . $min_len . ',}/S';
178 $binary = implode("\n", preg_grep($regex, explode("\n", $binary))) . $br;
185 // ---------------------
188 // Count leaves (A leaf = value that is not an array, or an empty array)
189 function array_count_leaves($array = array(), $count_empty = FALSE)
191 if (! is_array($array) || (empty($array) && $count_empty)) return 1;
195 foreach ($array as $part) {
196 $count += array_count_leaves($part, $count_empty);
202 // Similar to PHP array_merge_leaves(), except strictly preserving keys as string
203 function array_merge_leaves($array1, $array2, $sort_keys = TRUE)
206 $is_array1 = is_array($array1);
207 $is_array2 = is_array($array2);
214 } else if ($is_array2) {
217 return $array2; // Not array ($array1 is overwritten)
220 $keys_all = array_merge(array_keys($array1), array_keys($array2));
221 if ($sort_keys) sort($keys_all, SORT_STRING);
224 foreach($keys_all as $key) {
225 $isset1 = isset($array1[$key]);
226 $isset2 = isset($array2[$key]);
227 if ($isset1 && $isset2) {
229 $result[$key] = array_merge_leaves($array1[$key], $array2[$key], $sort_keys);
230 } else if ($isset1) {
231 $result[$key] = & $array1[$key];
233 $result[$key] = & $array2[$key];
239 // An array-leaves to a flat array
240 function array_flat_leaves($array, $unique = TRUE)
242 if (! is_array($array)) return $array;
245 foreach(array_keys($array) as $key) {
246 if (is_array($array[$key])) {
248 foreach(array_flat_leaves($array[$key]) as $_value) {
252 $tmp[] = & $array[$key];
256 return $unique ? array_values(array_unique($tmp)) : $tmp;
259 // $array['something'] => $array['wanted']
260 function array_rename_keys(& $array, $keys = array('from' => 'to'), $force = FALSE, $default = '')
262 if (! is_array($array) || ! is_array($keys)) return FALSE;
264 // Nondestructive test
266 foreach(array_keys($keys) as $from) {
267 if (! isset($array[$from])) {
273 foreach($keys as $from => $to) {
274 if ($from === $to) continue;
275 if (! $force || isset($array[$from])) {
276 $array[$to] = & $array[$from];
277 unset($array[$from]);
279 $array[$to] = $default;
286 // Remove redundant values from array()
287 function array_unique_recursive($array = array())
289 if (! is_array($array)) return $array;
292 foreach($array as $key => $value){
293 if (is_array($value)) {
294 $array[$key] = array_unique_recursive($value);
296 if (isset($tmp[$value])) {
308 // ---------------------
309 // Part One : Checker
311 // Rough implementation of globbing
313 // USAGE: $regex = '/^' . generate_glob_regex('*.txt', '/') . '$/i';
315 function generate_glob_regex($string = '', $divider = '/')
317 static $from = array(
320 // 22 => '[', // Maybe cause regex compilation error (e.g. '[]')
336 if (! is_string($string)) return '';
338 $string = str_replace($from, $mid, $string); // Hide
339 $string = preg_quote($string, $divider);
340 $string = str_replace($mid, $to, $string); // Unhide
345 // Generate host (FQDN, IPv4, ...) regex
346 // 'localhost' : Matches with 'localhost' only
347 // 'example.org' : Matches with 'example.org' only (See host_normalize() about 'www')
348 // '.example.org' : Matches with ALL FQDN ended with '.example.org'
349 // '*.example.org' : Almost the same of '.example.org' except 'www.example.org'
350 // '10.20.30.40' : Matches with IPv4 address '10.20.30.40' only
351 // [TODO] '192.' : Matches with all IPv4 hosts started with '192.'
352 // TODO: IPv4, CIDR?, IPv6
353 function generate_host_regex($string = '', $divider = '/')
355 if (! is_string($string)) return '';
357 if (mb_strpos($string, '.') === FALSE || is_ip($string)) {
358 // "localhost", IPv4, etc
359 return generate_glob_regex($string, $divider);
363 $part = explode('.', $string, 2);
364 if ($part[0] == '') {
366 $part[0] = '(?:.*\.)?';
367 } else if ($part[0] == '*') {
372 return generate_glob_regex($string, $divider);
375 $part[1] = generate_glob_regex($part[1], $divider);
377 return implode('', $part);
380 // Load SPAM_INI_FILE and return parsed one
381 function get_blocklist($list = '')
385 if ($list === NULL) {
386 $regexes = NULL; // Unset
390 if (! isset($regexes)) {
392 if (file_exists(SPAM_INI_FILE)) {
393 $blocklist = array();
395 include(SPAM_INI_FILE);
396 // $blocklist['list'] = array(
397 // //'goodhost' => FALSE;
398 // 'badhost' => TRUE;
400 // $blocklist['badhost'] = array(
401 // '*.blogspot.com', // Blog services's subdomains (only)
402 // 'IANA-examples' => '#^(?:.*\.)?example\.(?:com|net|org)$#',
410 if (! isset($blocklist[$special])) continue;
412 $regexes[$special] = $blocklist[$special];
414 foreach(array_keys($blocklist[$special]) as $_list) {
415 if (! isset($blocklist[$_list])) continue;
417 foreach ($blocklist[$_list] as $key => $value) {
418 if (is_array($value)) {
419 $regexes[$_list][$key] = array();
420 foreach($value as $_key => $_value) {
421 get_blocklist_add($regexes[$_list][$key], $_key, $_value);
424 get_blocklist_add($regexes[$_list], $key, $value);
428 unset($blocklist[$_list]);
435 return $regexes; // ALL of
436 } else if (isset($regexes[$list])) {
437 return $regexes[$list]; // A part of
439 return array(); // Found nothing
443 // Subroutine of get_blocklist(): Add new regex to the $array
444 function get_blocklist_add(& $array, $key = 0, $value = '*.example.org/path/to/file.html')
446 if (is_string($key)) {
447 $array[$key] = & $value; // Treat $value as a regex for FQDN(host)s
449 $regex = generate_host_regex($value, '#');
450 if (! empty($regex)) {
451 $array[$value] = '#^' . $regex . '$#i';
456 // Blocklist metrics: Separate $host, to $blocked and not blocked
457 function blocklist_distiller(& $hosts, $keys = array('goodhost', 'badhost'), $asap = FALSE)
459 if (! is_array($hosts)) $hosts = array($hosts);
460 if (! is_array($keys)) $keys = array($keys);
462 $list = get_blocklist('list');
465 foreach($keys as $key){
466 foreach (get_blocklist($key) as $label => $regex) {
467 if (is_array($regex)) {
468 foreach($regex as $_label => $_regex) {
469 $group = preg_grep($_regex, $hosts);
471 $hosts = array_diff($hosts, $group);
472 $blocked[$key][$label][$_label] = $group;
473 if ($asap && $list[$key]) break;
477 $group = preg_grep($regex, $hosts);
479 $hosts = array_diff($hosts, $group);
480 $blocked[$key][$label] = $group;
481 if ($asap && $list[$key]) break;
491 // ---------------------
494 // Default (enabled) methods and thresholds (for content insertion)
495 function check_uri_spam_method($times = 1, $t_area = 0, $rule = TRUE)
497 $times = intval($times);
498 $t_area = intval($t_area);
502 'quantity' => 8 * $times, // Allow N URIs
503 'non_uniqhost' => 3 * $times, // Allow N duped (and normalized) Hosts
504 //'non_uniquri'=> 3 * $times, // Allow N duped (and normalized) URIs
507 'area_anchor' => $t_area, // Using <a href> HTML tag
508 'area_bbcode' => $t_area, // Using [url] or [link] BBCode
509 //'uri_anchor' => $t_area, // URI inside <a href> HTML tag
510 //'uri_bbcode' => $t_area, // URI inside [url] or [link] BBCode
515 //'asap' => TRUE, // Quit or return As Soon As Possible
516 'uniqhost' => TRUE, // Show uniq host (at block notification mail)
517 'badhost' => TRUE, // Check badhost
523 // Remove non-$positive values
524 foreach (array_keys($positive) as $key) {
525 if ($positive[$key] < 0) unset($positive[$key]);
528 return $positive + $bool;
531 // Simple/fast spam check
532 function check_uri_spam($target = '', $method = array())
537 // Theme to do => Dummy, optional value, or optional array()
539 //'uniqhost' => TRUE,
540 //'non_uniqhost'=> 3,
541 //'non_uniquri' => 3,
543 //'area_anchor' => 0,
544 //'area_bbcode' => 0,
549 // Theme => Volume found (int)
552 // Flag. If someting defined here,
553 // one or more spam will be included
558 //'category' => array(
567 // ----------------------------------------
570 $sum = & $progress['sum'];
571 $is_spam = & $progress['is_spam'];
572 $progress['method'] = & $method; // Argument
573 $blocked = & $progress['blocked'];
574 $hosts = & $progress['hosts'];
575 $asap = isset($method['asap']);
577 // ----------------------------------------
580 if (! is_array($method) || empty($method)) {
581 $method = check_uri_spam_method();
583 foreach(array_keys($method) as $key) {
584 if (! isset($sum[$key])) $sum[$key] = 0;
586 if (! isset($sum['quantity'])) $sum['quantity'] = 0;
588 // ----------------------------------------
591 if (is_array($target)) {
592 foreach($target as $str) {
593 if (! is_string($str)) continue;
595 $_progress = check_uri_spam($str, $method); // Recurse
598 $_sum = & $_progress['sum'];
599 foreach (array_keys($_sum) as $key) {
600 if (! isset($sum[$key])) {
601 $sum[$key] = & $_sum[$key];
603 $sum[$key] += $_sum[$key];
608 $_is_spam = & $_progress['is_spam'];
609 foreach (array_keys($_is_spam) as $key) {
610 $is_spam[$key] = TRUE;
613 if ($asap && $is_spam) break;
616 $blocked = array_merge_leaves($blocked, $_progress['blocked'], FALSE);
617 $hosts = array_merge_leaves($hosts, $_progress['hosts'], FALSE);
621 $blocked = array_unique_recursive($blocked);
622 $hosts = array_unique_recursive($hosts);
624 // Recount $sum['badhost']
625 $sum['badhost'] = array_count_leaves($blocked);
630 // ----------------------------------------
633 if (! $asap || ! $is_spam) {
638 'area_anchor', // There's HTML anchor tag
639 'area_bbcode', // There's 'BBCode' linking tag
641 if (isset($method[$key])) $_method[$key] = TRUE;
645 $_asap = isset($method['asap']) ? array('asap' => TRUE) : array();
646 $_result = area_pickup($target, $_method + $_asap);
653 foreach(array_keys($_method) as $key) {
654 if (isset($_result[$key])) {
655 $sum[$key] = $_result[$key];
656 if (isset($method[$key]) && $sum[$key] > $method[$key]) {
657 $is_spam[$key] = TRUE;
663 unset($_asap, $_method, $_result);
667 if ($asap && $is_spam) return $progress;
669 // ----------------------------------------
672 $pickups = spam_uri_pickup($target, $method);
676 if (empty($pickups)) return $progress;
679 $pickups = uri_pickup_normalize($pickups);
681 // ----------------------------------------
682 // Pickup some part of URI
685 foreach ($pickups as $key => $pickup) {
686 $hosts[$key] = & $pickup['host'];
689 // ----------------------------------------
690 // URI: Bad host <pre-filter> (Separate good/bad hosts from $hosts)
692 if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
693 $list = get_blocklist('pre');
694 $blocked = blocklist_distiller($hosts, array_keys($list), $asap);
695 foreach($list as $key => $type){
696 if (! $type) unset($blocked[$key]); // Ignore goodhost etc
699 if (! empty($blocked)) $is_spam['badhost'] = TRUE;
703 if ($asap && $is_spam) return $progress;
705 // Remove blocked from $pickups
706 foreach(array_keys($pickups) as $key) {
707 if (! isset($hosts[$key])) {
708 unset($pickups[$key]);
712 // ----------------------------------------
713 // URI: Check quantity
715 $sum['quantity'] += count($pickups);
717 if ((! $asap || ! $is_spam) && isset($method['quantity']) &&
718 $sum['quantity'] > $method['quantity']) {
719 $is_spam['quantity'] = TRUE;
722 // ----------------------------------------
723 // URI: used inside HTML anchor tag pair
725 if ((! $asap || ! $is_spam) && isset($method['uri_anchor'])) {
727 foreach($pickups as $pickup) {
728 if (isset($pickup['area'][$key])) {
729 $sum[$key] += $pickup['area'][$key];
730 if(isset($method[$key]) &&
731 $sum[$key] > $method[$key]) {
732 $is_spam[$key] = TRUE;
733 if ($asap && $is_spam) break;
735 if ($asap && $is_spam) break;
740 // ----------------------------------------
741 // URI: used inside 'BBCode' pair
743 if ((! $asap || ! $is_spam) && isset($method['uri_bbcode'])) {
745 foreach($pickups as $pickup) {
746 if (isset($pickup['area'][$key])) {
747 $sum[$key] += $pickup['area'][$key];
748 if(isset($method[$key]) &&
749 $sum[$key] > $method[$key]) {
750 $is_spam[$key] = TRUE;
751 if ($asap && $is_spam) break;
753 if ($asap && $is_spam) break;
758 // ----------------------------------------
759 // URI: Uniqueness (and removing non-uniques)
761 if ((! $asap || ! $is_spam) && isset($method['non_uniquri'])) {
764 foreach (array_keys($pickups) as $key) {
765 $uris[$key] = uri_pickup_implode($pickups[$key]);
767 $count = count($uris);
768 $uris = array_unique($uris);
769 $sum['non_uniquri'] += $count - count($uris);
770 if ($sum['non_uniquri'] > $method['non_uniquri']) {
771 $is_spam['non_uniquri'] = TRUE;
773 if (! $asap || ! $is_spam) {
774 foreach (array_diff(array_keys($pickups),
775 array_keys($uris)) as $remove) {
776 unset($pickups[$remove]);
783 if ($asap && $is_spam) return $progress;
785 // ----------------------------------------
786 // Host: Uniqueness (uniq / non-uniq)
788 $hosts = array_unique($hosts);
790 if (isset($sum['uniqhost'])) $sum['uniqhost'] += count($hosts);
791 if ((! $asap || ! $is_spam) && isset($method['non_uniqhost'])) {
792 $sum['non_uniqhost'] = $sum['quantity'] - $sum['uniqhost'];
793 if ($sum['non_uniqhost'] > $method['non_uniqhost']) {
794 $is_spam['non_uniqhost'] = TRUE;
799 if ($asap && $is_spam) return $progress;
801 // ----------------------------------------
802 // URI: Bad host (Separate good/bad hosts from $hosts)
804 if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
805 $list = get_blocklist('list');
806 $blocked = array_merge_leaves(
808 blocklist_distiller($hosts, array_keys($list), $asap),
811 foreach($list as $key=>$type){
812 if (! $type) unset($blocked[$key]); // Ignore goodhost etc
815 if (! empty($blocked)) $is_spam['badhost'] = TRUE;
819 //if ($asap && $is_spam) return $progress;
821 // ----------------------------------------
827 // ---------------------
830 // Summarize $progress (blocked only)
831 function summarize_spam_progress($progress = array(), $blockedonly = FALSE)
834 $tmp = array_keys($progress['is_spam']);
837 $method = & $progress['method'];
838 if (isset($progress['sum'])) {
839 foreach ($progress['sum'] as $key => $value) {
840 if (isset($method[$key]) && $value) {
841 $tmp[] = $key . '(' . $value . ')';
847 return implode(', ', $tmp);
850 function summarize_detail_badhost($progress = array())
852 if (! isset($progress['blocked']) || empty($progress['blocked'])) return '';
856 foreach($progress['blocked'] as $list => $lvalue) {
857 foreach($lvalue as $group => $gvalue) {
858 $flat = implode(', ', array_flat_leaves($gvalue));
859 if ($flat === $group) {
860 $blocked[$list][] = $flat;
862 $blocked[$list][$group] = $flat;
868 // From: 'A-1' => array('ie.to')
869 // To: 'A-1' => 'ie.to'
870 foreach($blocked as $list => $lvalue) {
871 if (is_array($lvalue) &&
872 count($lvalue) == 1 &&
873 is_numeric(key($lvalue))) {
874 $blocked[$list] = current($lvalue);
878 return var_export_shrink($blocked, TRUE, TRUE);
881 function summarize_detail_newtral($progress = array())
883 if (! isset($progress['hosts']) ||
884 ! is_array($progress['hosts']) ||
885 empty($progress['hosts'])) return '';
887 // Generate a responsible $trie
889 foreach($progress['hosts'] as $value) {
890 // 'A.foo.bar.example.com'
891 $resp = whois_responsibility($value); // 'example.com'
893 // One or more test, or do nothing here
894 $resp = strval($value);
897 $rest = rtrim(substr($value, 0, - strlen($resp)), '.'); // 'A.foo.bar'
899 $trie = array_merge_leaves($trie, array($resp => array($rest => NULL)), FALSE);
902 // Format: var_export_shrink() -like output
904 ksort_by_domain($trie);
905 foreach(array_keys($trie) as $key) {
906 ksort_by_domain($trie[$key]);
907 if (count($trie[$key]) == 1 && key($trie[$key]) == '') {
908 // Just one 'responsibility.example.com'
909 $result[] = ' \'' . $key . '\',';
911 // One subdomain-or-host, or several ones
913 foreach(array_keys($trie[$key]) as $sub) {
915 $subs[] = $key; // 'example.com'
917 $subs[] = $sub . '. '; // 'A.foo.bar. '
920 $result[] = ' \'' . $key . '\' => \'' . implode(', ', $subs) . '\',';
926 implode("\n", $result) . "\n" .
931 // ---------------------
935 function spam_dispose()
938 whois_responsibility(NULL);
941 // Common bahavior for blocking
942 // NOTE: Call this function from various blocking feature, to disgueise the reason 'why blocked'
943 function spam_exit($mode = '', $data = array())
952 echo('<pre>' . "\n");
953 echo htmlspecialchars(var_export($data, TRUE));
954 echo('</pre>' . "\n");
958 if ($exit) exit; // Force exit
962 // ---------------------
966 // Simple/fast spam filter ($target: 'a string' or an array())
967 function pkwk_spamfilter($action, $page, $target = array('title' => ''), $method = array(), $exitmode = '')
969 $progress = check_uri_spam($target, $method);
971 if (empty($progress['is_spam'])) {
975 // TODO: detect encoding from $target for mbstring functions
977 // foreach(array_keys($target) as $key) {
978 // $tmp[strings($key, 0, FALSE, TRUE)] = strings($target[$key], 0, FALSE, TRUE); // Removing "\0" etc
982 pkwk_spamnotify($action, $page, $target, $progress, $method);
983 spam_exit($exitmode, $progress);
987 // ---------------------
990 // Mail to administrator(s)
991 function pkwk_spamnotify($action, $page, $target = array('title' => ''), $progress = array(), $method = array())
993 global $notify, $notify_subject;
995 if (! $notify) return;
997 $asap = isset($method['asap']);
999 $summary['ACTION'] = 'Blocked by: ' . summarize_spam_progress($progress, TRUE);
1001 $summary['METRICS'] = summarize_spam_progress($progress);
1004 $tmp = summarize_detail_badhost($progress);
1005 if ($tmp != '') $summary['DETAIL_BADHOST'] = $tmp;
1007 $tmp = summarize_detail_newtral($progress);
1008 if (! $asap && $tmp != '') $summary['DETAIL_NEUTRAL_HOST'] = $tmp;
1010 $summary['COMMENT'] = $action;
1011 $summary['PAGE'] = '[blocked] ' . (is_pagename($page) ? $page : '');
1012 $summary['URI'] = get_script_uri() . '?' . rawurlencode($page);
1013 $summary['USER_AGENT'] = TRUE;
1014 $summary['REMOTE_ADDR'] = TRUE;
1015 pkwk_mail_notify($notify_subject, var_export($target, TRUE), $summary, TRUE);