Apache-Nginx-PHP屏蔽垃圾UA爬虫实战指南

1
cd /usr/local/nginx/conf
2
vim agent_deny.conf

1
#禁止Scrapy等工具的抓取
2
if ($http_user_agent ~* (Scrapy|Curl|HttpClient)) {
3
     return 403;
4
}
5
#禁止指定UA及UA为空的访问
6
if ($http_user_agent ~* "FeedDemon|Indy Library|Alexa Toolbar|AskTbFXTV|AhrefsBot|CrawlDaddy|CoolpadWebkit|Java|Feedly|UniversalFeedParser|ApacheBench|Microsoft URL Control|Swiftbot|ZmEu|oBot|jaunty|Python-urllib|lightDeckReports Bot|YYSpider|DigExt|HttpClient|MJ12bot|heritrix|EasouSpider|DotBot|Ezooms|^$" ) {
7
     return 403;
8
}
9
#禁止非GET|HEAD|POST方式的抓取
10
if ($request_method !~ ^(GET|HEAD|POST)$) {
11
    return 403;
12
}

1
[marsge@Mars_Server ~]$ cat /usr/local/nginx/conf/gowah.conf
2
location / {
3
        try_files $uri $uri/ /index.php?$args;
4
        #这个位置新增1行：
5
        include agent_deny.conf;
6
        rewrite ^/sitemap_360_sp.txt$ /sitemap_360_sp.php last;
7
        rewrite ^/sitemap_baidu_sp.xml$ /sitemap_baidu_sp.php last;
8
        rewrite ^/sitemap_m.xml$ /sitemap_m.php last;

1
/usr/local/nginx/sbin/nginx -s reload

1
//获取UA信息
2
$ua = $_SERVER['HTTP_USER_AGENT'];
3
//将恶意USER_AGENT存入数组
4
$now_ua = array('FeedDemon ','BOT/0.1 (BOT for JCE)','CrawlDaddy ','Java','Feedly','UniversalFeedParser','ApacheBench','Swiftbot','ZmEu','Indy Library','oBot','jaunty','YandexBot','AhrefsBot','MJ12bot','WinHttp','EasouSpider','HttpClient','Microsoft URL Control','YYSpider','jaunty','Python-urllib','DotBot','lightDeckReports Bot');
5
//禁止空USER_AGENT，dedecms等主流采集程序都是空USER_AGENT，部分sql注入工具也是空USER_AGENT
6
if(!$ua) {
7
    header("Content-type: text/html; charset=utf-8");
8
    die('请勿采集本站，因为采集的站长木有小JJ！');
9
}else{
10
    foreach($now_ua as $value )
11
//判断是否是数组中存在的UA
12
    if(eregi($value,$ua)) {
13
        header("Content-type: text/html; charset=utf-8");
14
        die('请勿采集本站，因为采集的站长木有小JJ！');
15
    }
16
}

1
curl -I -A 'Swiftbot' moewah.com

1
curl -I -A '' moewah.com

1
curl -I -A 'Baiduspider' moewah.com

1
FeedDemon             内容采集
2
BOT/0.1 (BOT for JCE) sql注入
3
CrawlDaddy            sql注入
4
Java                  内容采集
5
Jullo                 内容采集
6
Feedly                内容采集
7
UniversalFeedParser   内容采集
8
ApacheBench           cc攻击器
9
Swiftbot              无用爬虫
10
YandexBot             无用爬虫
11
AhrefsBot             无用爬虫
12
YisouSpider           无用爬虫（已被UC神马搜索收购，此蜘蛛可以放开！）
13
MJ12bot               无用爬虫
14
ZmEu phpmyadmin       漏洞扫描
15
WinHttp               采集cc攻击
16
EasouSpider           无用爬虫
17
HttpClient            tcp攻击
18
Microsoft URL Control 扫描
19
YYSpider              无用爬虫
20
jaunty                wordpress爆破扫描器
21
oBot                  无用爬虫
22
Python-urllib         内容采集
23
Indy Library          扫描
24
FlightDeckReports Bot 无用爬虫
25
Linguee Bot           无用爬虫
26
DotBot                无用爬虫（国外SEO爬虫）