<?php
-// $Id: spam.php,v 1.74 2006/12/16 04:47:07 henoheno Exp $
-// Copyright (C) 2006 PukiWiki Developers Team
+// $Id: spam.php,v 1.122 2007/03/04 03:59:37 henoheno Exp $
+// Copyright (C) 2006-2007 PukiWiki Developers Team
// License: GPL v2 or (at your option) any later version
-
+//
// Functions for Concept-work of spam-uri metrics
-// (PHP 4 >= 4.3.0): preg_match_all()
+//
+// (PHP 4 >= 4.3.0): preg_match_all(PREG_OFFSET_CAPTURE): $method['uri_XXX'] related feature
if (! defined('SPAM_INI_FILE')) define('SPAM_INI_FILE', 'spam.ini.php');
// ---------------------
+// Compat etc
+
+// (PHP 4 >= 4.2.0): var_export(): mail-reporting and dump related
+if (! function_exists('var_export')) {
+ function var_export() {
+ return 'var_export() is not found on this server' . "\n";
+ }
+}
+
+// (PHP 4 >= 4.2.0): preg_grep() enables invert option
+function preg_grep_invert($pattern = '//', $input = array())
+{
+ static $invert;
+ if (! isset($invert)) $invert = defined('PREG_GREP_INVERT');
+
+ if ($invert) {
+ return preg_grep($pattern, $input, PREG_GREP_INVERT);
+ } else {
+ $result = preg_grep($pattern, $input);
+ if ($result) {
+ return array_diff($input, preg_grep($pattern, $input));
+ } else {
+ return $input;
+ }
+ }
+}
+
+// ---------------------
// URI pickup
// Return an array of URIs in the $string
preg_match_all(
// scheme://userinfo@host:port/path/or/pathinfo/maybefile.and?query=string#fragment
// Refer RFC3986 (Regex below is not strict)
- '#(\b[a-z][a-z0-9.+-]{1,8})://' . // 1: Scheme
+ '#(\b[a-z][a-z0-9.+-]{1,8}):/+' . // 1: Scheme
'(?:' .
'([^\s<>"\'\[\]/\#?@]*)' . // 2: Userinfo (Username)
'@)?' .
// 3: Host
'\[[0-9a-f:.]+\]' . '|' . // IPv6([colon-hex and dot]): RFC2732
'(?:[0-9]{1-3}\.){3}[0-9]{1-3}' . '|' . // IPv4(dot-decimal): 001.22.3.44
- '[^\s<>"\'\[\]:/\#?]+' . // FQDN: foo.example.org
+ '[a-z0-9.-]+' . // hostname(FQDN) : foo.example.org
')' .
'(?::([0-9]*))?' . // 4: Port
'((?:/+[^\s<>"\'\[\]/\#]+)*/+)?' . // 5: Directory path or path-info
'#i',
$string, $array, PREG_SET_ORDER | PREG_OFFSET_CAPTURE
);
- //var_dump(recursive_map('htmlspecialchars', $array));
// Shrink $array
static $parts = array(
function area_pickup($string = '', $method = array())
{
$area = array();
+ if (empty($method)) return $area;
// Anchor tag pair by preg_match and preg_match_all()
// [OK] <a href></a>
// [NG] <a href="http://ng.example.com">visit http://ng.example.com _not_ended_
$regex = '#<a\b[^>]*\bhref\b[^>]*>.*?</a\b[^>]*(>)#i';
if (isset($method['area_anchor'])) {
- $key = 'area_anchor';
- if ($method[$key]) {
- if (preg_match($regex, $string)) $area[$key] = 1;
- } else {
- $areas = array();
- $res = preg_match_all($regex, $string, $areas, PREG_SET_ORDER);
- if (! empty($areas)) $area[$key] = $res;
- }
+ $areas = array();
+ $count = isset($method['asap']) ?
+ preg_match($regex, $string) :
+ preg_match_all($regex, $string, $areas);
+ if (! empty($count)) $area['area_anchor'] = $count;
}
if (isset($method['uri_anchor'])) {
$areas = array();
// [OK] [link http://nasty.example.com/]buy something[/link]
$regex = '#\[(url|link)\b[^\]]*\].*?\[/\1\b[^\]]*(\])#i';
if (isset($method['area_bbcode'])) {
- $key = 'area_bbcode';
- if ($method[$key]) {
- if (preg_match($regex, $string)) $area[$key] = 1;
- } else {
- $areas = array();
- $res = preg_match_all($regex, $string, $areas, PREG_SET_ORDER);
- if (! empty($areas)) $area[$key] = $res;
- }
+ $areas = array();
+ $count = isset($method['asap']) ?
+ preg_match($regex, $string) :
+ preg_match_all($regex, $string, $areas, PREG_SET_ORDER);
+ if (! empty($count)) $area['area_bbcode'] = $count;
}
if (isset($method['uri_bbcode'])) {
$areas = array();
// NOTE: It's maybe danger to var_dump(result). [e.g. 'javascript:']
// [OK] http://victim.example.org/go?http%3A%2F%2Fnasty.example.org
// [OK] http://victim.example.org/http://nasty.example.org
+// TODO: link.toolbot.com, urlx.org
function spam_uri_pickup_preprocess($string = '')
{
if (! is_string($string)) return '';
// Domain exposure (See _preg_replace_callback_domain_exposure())
$string = preg_replace_callback(
array(
- // Something Google: http://www.google.com/supported_domains
- '#(http)://([a-z0-9.]+\.google\.[a-z]{2,3}(?:\.[a-z]{2})?)/' .
- '([a-z0-9?=&.%_+-]+)' . // ?query=foo+
- '\bsite:([a-z0-9.%_-]+)' . // site:nasty.example.com
+ '#(http)://' .
+ '(' .
+ // Something Google: http://www.google.com/supported_domains
+ '(?:[a-z0-9.]+\.)?google\.[a-z]{2,3}(?:\.[a-z]{2})?' .
+ '|' .
+ // AltaVista
+ '(?:[a-z0-9.]+\.)?altavista.com' .
+
+ ')' .
+ '/' .
+ '([a-z0-9?=&.%_/\'\\\+-]+)' . // path/?query=foo+bar+
+ '\bsite:([a-z0-9.%_-]+\.[a-z0-9.%_-]+)' . // site:nasty.example.com
//'()' . // Preserve or remove?
'#i',
),
$array = uri_pickup($string);
- // Area elevation for '(especially external)link' intension
+ // Area elevation of URIs, for '(especially external)link' intension
if (! empty($array)) {
$_method = array();
if (isset($method['uri_anchor'])) $_method['uri_anchor'] = & $method['uri_anchor'];
if (isset($method['uri_bbcode'])) $_method['uri_bbcode'] = & $method['uri_bbcode'];
-
$areas = area_pickup($string, $_method, TRUE);
if (! empty($areas)) {
$area_shadow = array();
// ---------------------
// Part One : Checker
+// Rough implementation of globbing
+//
+// USAGE: $regex = '/^' . generate_glob_regex('*.txt', '/') . '$/i';
+//
function generate_glob_regex($string = '', $divider = '/')
{
static $from = array(
}
}
-// TODO: Ignore list
-// TODO: preg_grep() ?
-// TODO: Multi list
-function is_badhost($hosts = '', $asap = TRUE)
+function get_blocklist($list = '')
{
- static $regex;
-
- if (! isset($regex)) {
- $regex = array();
- $regex['badhost'] = array();
-
- // Sample
- if (TRUE) {
- $blocklist['badhost'] = array(
- //'*', // Deny all uri
- //'10.20.*.*', // 10.20.example.com also matches
- //'*.blogspot.com', // Blog services subdomains
- //array('blogspot.com', '*.blogspot.com')
- );
- foreach ($blocklist['badhost'] as $part) {
- if (is_array($part)) $part = implode(', ', $part);
- $regex['badhost'][$part] = '/^' . generate_glob_regex($part) . '$/i';
- }
- }
+ static $regexs;
- // Load
+ if (! isset($regexs)) {
+ $regexs = array();
if (file_exists(SPAM_INI_FILE)) {
$blocklist = array();
- require(SPAM_INI_FILE);
- foreach ($blocklist['badhost'] as $part) {
- if (is_array($part)) $part = implode(', ', $part);
- $regex['badhost'][$part] = '/^' . generate_glob_regex($part) . '$/i';
+ include(SPAM_INI_FILE);
+ // $blocklist['badhost'] = array(
+ // '*.blogspot.com', // Blog services's subdomains (only)
+ // 'IANA-examples' => '#^(?:.*\.)?example\.(?:com|net|org)$#',
+ // );
+ foreach(array('goodhost', 'badhost') as $_list) {
+ if (! isset($blocklist[$list])) continue;
+ foreach ($blocklist[$_list] as $key => $value) {
+ if (is_string($key)) {
+ $regexs[$_list][$key] = $value;
+ } else {
+ $regexs[$_list][$value] =
+ '/^(?:www\.)?' . generate_glob_regex($value, '/') . '$/i';
+ }
+ }
}
}
}
- //var_dump($regex);
+ if ($list == '') {
+ return $regexs; // ALL
+ } else if (isset($regexs[$list])) {
+ return $regexs[$list];
+ } else {
+ return array();
+ }
+}
+
+function is_badhost($hosts = array(), $asap = TRUE, & $remains)
+{
$result = array();
if (! is_array($hosts)) $hosts = array($hosts);
+ foreach(array_keys($hosts) as $key) {
+ if (! is_string($hosts[$key])) unset($hosts[$key]);
+ }
+ if (empty($hosts)) return $result;
- foreach($hosts as $host) {
- if (! is_string($host)) $host = '';
- foreach ($regex['badhost'] as $part => $_regex) {
- if (preg_match($_regex, $host)) {
- if (! isset($result[$part])) $result[$part] = array();
- $result[$part][] = $host;
- if ($asap) {
- return $result;
- } else {
- break;
- }
- }
+ foreach (get_blocklist('goodhost') as $regex) {
+ $hosts = preg_grep_invert($regex, $hosts);
+ }
+ if (empty($hosts)) return $result;
+
+ $tmp = array();
+ foreach (get_blocklist('badhost') as $label => $regex) {
+ $result[$label] = preg_grep($regex, $hosts);
+ if (empty($result[$label])) {
+ unset($result[$label]);
+ } else {
+ $hosts = array_diff($hosts, $result[$label]);
+ if ($asap) break;
}
}
+ $remains = $hosts;
+
return $result;
}
-// Default (enabled) methods and thresholds
+// Default (enabled) methods and thresholds (for content insertion)
function check_uri_spam_method($times = 1, $t_area = 0, $rule = TRUE)
{
$times = intval($times);
$positive = array(
// Thresholds
- 'quantity' => 8 * $times, // Allow N URIs
- 'non_uniq' => 3 * $times, // Allow N duped (and normalized) URIs
+ 'quantity' => 8 * $times, // Allow N URIs
+ 'non_uniqhost' => 3 * $times, // Allow N duped (and normalized) Hosts
+ //'non_uniquri'=> 3 * $times, // Allow N duped (and normalized) URIs
// Areas
- 'area_anchor'
=> $t_area, // Using <a href> HTML tag
- 'area_bbcode' => $t_area, // Using [url] or [link] BBCode
- //'uri_anchor'
=> $t_area, // URI inside <a href> HTML tag
- //'uri_bbcode' => $t_area, // URI inside [url] or [link] BBCode
+ 'area_anchor'
=> $t_area, // Using <a href> HTML tag
+ 'area_bbcode' => $t_area, // Using [url] or [link] BBCode
+ //'uri_anchor'
=> $t_area, // URI inside <a href> HTML tag
+ //'uri_bbcode' => $t_area, // URI inside [url] or [link] BBCode
);
if ($rule) {
$bool = array(
// Rules
- //'asap' => TRUE, // Quit As Soon As Possible
- 'uniqhost' => TRUE, // Show uniq host (at block notification mail)
- 'badhost' => TRUE, // Check badhost
+ //'asap' => TRUE, // Quit or return As Soon As Possible
+ 'uniqhost' => TRUE, // Show uniq host (at block notification mail)
+ 'badhost' => TRUE, // Check badhost
);
} else {
$bool = array();
'sum' => array(
'quantity' => 0,
'uniqhost' => 0,
- 'non_uniq' => 0,
+ 'non_uniqhost'=> 0,
+ 'non_uniquri' => 0,
'badhost' => 0,
'area_anchor' => 0,
'area_bbcode' => 0,
),
'is_spam' => array(),
'method' => & $method,
+ 'remains' => array(),
);
$sum = & $progress['sum'];
$is_spam = & $progress['is_spam'];
+ $remains = & $progress['remains'];
$asap = isset($method['asap']);
- // Return if ...
+ // Recurse
if (is_array($target)) {
foreach($target as $str) {
// Recurse
$_progress = check_uri_spam($str, $method);
- foreach (array_keys($_progress['sum']) as $key) {
- $sum[$key] += $_progress['sum'][$key];
+ $_sum = & $_progress['sum'];
+ $_is_spam = & $_progress['is_spam'];
+ $_remains = & $_progress['remains'];
+ foreach (array_keys($_sum) as $key) {
+ $sum[$key] += $_sum[$key];
}
- foreach(array_keys($_progress['is_spam']) as $key) {
- $is_spam[$key] = TRUE;
+ foreach (array_keys($_is_spam) as $key) {
+ if (is_array($_is_spam[$key])) {
+ // Marge keys (badhost)
+ foreach(array_keys($_is_spam[$key]) as $_key) {
+ if (! isset($is_spam[$key][$_key])) {
+ $is_spam[$key][$_key] = $_is_spam[$key][$_key];
+ } else {
+ $is_spam[$key][$_key] += $_is_spam[$key][$_key];
+ }
+ }
+ } else {
+ $is_spam[$key] = TRUE;
+ }
+ }
+ foreach ($_remains as $key=>$value) {
+ foreach ($value as $_key=>$_value) {
+ if (is_int($_key)) {
+ $remains[$key][] = $_value;
+ } else {
+ $remains[$key][$_key] = $_value;
+ }
+ }
}
if ($asap && $is_spam) break;
}
// Area: There's HTML anchor tag
if ((! $asap || ! $is_spam) && isset($method['area_anchor'])) {
$key = 'area_anchor';
- $result = area_pickup($target, array($key => $asap));
+ $_asap = isset($method['asap']) ? array('asap' => TRUE) : array();
+ $result = area_pickup($target, array($key => TRUE) + $_asap);
if ($result) {
- $sum[$key] += $result[$key];
- $is_spam[$key] = TRUE;
+ $sum[$key] = $result[$key];
+ if (isset($method[$key]) && $sum[$key] > $method[$key]) {
+ $is_spam[$key] = TRUE;
+ }
}
}
// Area: There's 'BBCode' linking tag
if ((! $asap || ! $is_spam) && isset($method['area_bbcode'])) {
$key = 'area_bbcode';
- $result = area_pickup($target, array($key => $asap));
+ $_asap = isset($method['asap']) ? array('asap' => TRUE) : array();
+ $result = area_pickup($target, array($key => TRUE) + $_asap);
if ($result) {
- $sum[$key] += $result[$key];
- $is_spam[$key] = TRUE;
+ $sum[$key] = $result[$key];
+ if (isset($method[$key]) && $sum[$key] > $method[$key]) {
+ $is_spam[$key] = TRUE;
+ }
}
}
- // URI Init
+ // Return if ...
+ if ($asap && $is_spam) return $progress;
+
+ // URI: Pickup
$pickups = spam_uri_pickup($target, $method);
- if (empty($pickups)) {
- return $progress;
- }
+ //$remains['uri_pickup'] = & $pickups;
+
+ // Return if ...
+ if (empty($pickups)) return $progress;
// URI: Check quantity
$sum['quantity'] += count($pickups);
}
// URI: Uniqueness (and removing non-uniques)
- if ((! $asap || ! $is_spam) && isset($method['non_uniq'])) {
+ if ((! $asap || ! $is_spam) && isset($method['non_uniquri'])) {
// Destructive normalize of URIs
uri_array_normalize($pickups);
}
$count = count($uris);
$uris = array_unique($uris);
- $sum['non_uniq'] += $count - count($uris);
- if ($sum['non_uniq'] > $method['non_uniq']) {
- $is_spam['non_uniq'] = TRUE;
+ $sum['non_uniquri'] += $count - count($uris);
+ if ($sum['non_uniquri'] > $method['non_uniquri']) {
+ $is_spam['non_uniquri'] = TRUE;
}
if (! $asap || ! $is_spam) {
foreach (array_diff(array_keys($pickups),
unset($uris);
}
- // URI: Unique host
+ // Return if ...
+ if ($asap && $is_spam) return $progress;
+
+ // Host: Uniqueness (uniq / non-uniq)
$hosts = array();
foreach ($pickups as $pickup) $hosts[] = & $pickup['host'];
$hosts = array_unique($hosts);
+ //$remains['uniqhost'] = & $hosts;
$sum['uniqhost'] += count($hosts);
+ if ((! $asap || ! $is_spam) && isset($method['non_uniqhost'])) {
+ $sum['non_uniqhost'] = $sum['quantity'] - $sum['uniqhost'];
+ if ($sum['non_uniqhost'] > $method['non_uniqhost']) {
+ $is_spam['non_uniqhost'] = TRUE;
+ }
+ }
+
+ // Return if ...
+ if ($asap && $is_spam) return $progress;
// URI: Bad host
if ((! $asap || ! $is_spam) && isset($method['badhost'])) {
- $count = array_count_leaves(is_badhost($hosts, $asap));
- $sum['badhost'] += $count;
- if ($count != 0) $is_spam['badhost'] = TRUE;
+ $__remains = array();
+ if ($asap) {
+ $badhost = is_badhost($hosts, $asap, $__remains);
+ } else {
+ $badhost = is_badhost($hosts, $asap, $__remains);
+ if ($__remains) {
+ $remains['badhost'] = array();
+ foreach ($__remains as $value) {
+ $remains['badhost'][$value] = TRUE;
+ }
+ }
+ }
+ unset($__remains);
+ if (! empty($badhost)) {
+ $sum['badhost'] += array_count_leaves($badhost);
+ foreach(array_keys($badhost) as $keys) {
+ $is_spam['badhost'][$keys] =
+ array_count_leaves($badhost[$keys]);
+ }
+ unset($badhost);
+ }
}
return $progress;
// Common bahavior for blocking
// NOTE: Call this function from various blocking feature, to disgueise the reason 'why blocked'
-function spam_exit()
+function spam_exit($mode = '', $data = array())
{
- die("\n");
+ switch ($mode) {
+ case '': echo("\n"); break;
+ case 'dump':
+ echo('<pre>' . "\n");
+ echo htmlspecialchars(var_export($data, TRUE));
+ echo('</pre>' . "\n");
+ break;
+ };
+
+ // Force exit
+ exit;
}
// TODO: Record them
// Simple/fast spam filter ($target: 'a string' or an array())
-function pkwk_spamfilter($action, $page, $target = array('title' => ''), $method = array())
+function pkwk_spamfilter($action, $page, $target = array('title' => ''), $method = array(), $exitmode = '')
{
- global $notify;
-
$progress = check_uri_spam($target, $method);
if (! empty($progress['is_spam'])) {
// Mail to administrator(s)
- if ($notify) pkwk_spamnotify($action, $page, $target, $progress, $method);
- // End
- spam_exit();
+ pkwk_spamnotify($action, $page, $target, $progress, $method);
+
+ // Exit
+ spam_exit($exitmode, $progress);
}
}
// Mail to administrator(s)
function pkwk_spamnotify($action, $page, $target = array('title' => ''), $progress = array(), $method = array())
{
- global $notify_subject;
+ global $notify, $notify_subject;
- $asap = isset($method['asap']) ? $method['asap'] : TRUE;
+ if (! $notify) return;
- $footer['ACTION'] = 'Blocked by: ' . summarize_spam_progress($progress, TRUE);
+ $asap = isset($method['asap']);
+ $summary['ACTION'] = 'Blocked by: ' . summarize_spam_progress($progress, TRUE);
if (! $asap) {
- $footer['METRICS'] = summarize_spam_progress($progress);
- }
- $footer['COMMENT'] = $action;
- $footer['PAGE'] = '[blocked] ' . $page;
- $footer['URI'] = get_script_uri() . '?' . rawurlencode($page);
- $footer['USER_AGENT'] = TRUE;
- $footer['REMOTE_ADDR'] = TRUE;
- pkwk_mail_notify($notify_subject, var_export($target, TRUE), $footer);
+ $summary['METRICS'] = summarize_spam_progress($progress);
+ }
+ if (isset($progress['is_spam']['badhost'])) {
+ $badhost = array();
+ foreach($progress['is_spam']['badhost'] as $glob=>$number) {
+ $badhost[] = $glob . '(' . $number . ')';
+ }
+ $summary['DETAIL_BADHOST'] = implode(', ', $badhost);
+ }
+ if (! $asap && $progress['remains']['badhost']) {
+ $count = count($progress['remains']['badhost']);
+ $summary['DETAIL_NEUTRAL_HOST'] = $count .
+ ' (' .
+ preg_replace(
+ '/[^, a-z0-9.-]/i', '',
+ implode(', ', array_keys($progress['remains']['badhost']))
+ ) .
+ ')';
+ }
+ $summary['COMMENT'] = $action;
+ $summary['PAGE'] = '[blocked] ' . (is_pagename($page) ? $page : '');
+ $summary['URI'] = get_script_uri() . '?' . rawurlencode($page);
+ $summary['USER_AGENT'] = TRUE;
+ $summary['REMOTE_ADDR'] = TRUE;
+ pkwk_mail_notify($notify_subject, var_export($target, TRUE), $summary, TRUE);
}
?>
OSDN Git Service
