网站被爬虫爬得扛不住了。。想把这些爬虫都屏蔽了。。
nginx下配有多个站点。。nginx.conf里是没有server字段的。。只有各个站点的.conf文件里有。。
我按照网上的说法，在nginx.conf的server字段(nginx.conf的server字段也是我自己加的)中加入了以下内容：
if ($http_user_agent ~* "qihoobot|Baiduspider|Googlebot|Googlebot-Mobile|Googlebot-Image|Mediapartners-Google|Adsbot-Google|Feedfetcher-Google|Yahoo! Slurp|Yahoo! Slurp China|YoudaoBot|Sosospider|Sogou spider|Sogou web spider|MSNBot|ia_archiver|Tomato Bot") {
return 403;
}
但是用curl -I -A "Googlebot" www.XXX.com，仍然没有返回403。。
太不靠谱了。。
求高手指点。。
另外robots.txt也不好使。。那个东西全靠自觉。。我想要能主动禁止他们。。。因为有的流氓爬虫显然没法用那个解决
nginx.conf内容如下：
#user nobody;
worker_processes 2;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid文件的位置
pid nginx.pid;
events {
worker_connections 10240;
}
http {
include mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
# open(OUTFILE, ">>/home/wamdm/perl_learn/a");
# print OUTFILE ($r->uri,"\n");
# close (OUTFILE);
perl_set $fix_upper_lower_case '
use File::Basename;
sub {
my $r = shift;
my $uri = $r->uri;
my $filepath = $r->filename;
my $uri_prefix = substr($uri, 0, rindex($uri, "/") + 1);
my $dir = dirname($filepath);
my $filename = basename($filepath);
opendir(my $dh, $dir) || die ("~~fail to open dir $dir");
my @files = grep { /$filename/i && -f "$dir/$_" } readdir($dh);
closedir($dh);
if (@files > 0) {
return "$uri_prefix@files[0]";
}
return $r->uri;
}
';
server {
if ($http_user_agent ~* "MJ12bot|qihoobot|Baiduspider|Googlebot|Googlebot-Mobile|YandexBot|Googlebot-Image|Mediapartners-Google|Adsbot-Google|Feedfetcher-Google|Yahoo! Slurp|Yahoo! Slurp China|YoudaoBot|Sosospider|Sogou spider|Sogou web spider|MSNBot|ia_archiver|Tomato Bot")
{
return 403;
}
listen 80;
# server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
# location / {
# root html;
# index index.html index.htm;
# }
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
# error_page 500 502 503 504 /50x.html;
# location = /50x.html {
# root html;
# }
# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
# proxy_pass http://127.0.0.1;
#}
# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
# root html;
# fastcgi_pass 127.0.0.1:9000;
# fastcgi_index index.php;
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
}
# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
# listen 8000;
# listen somename:8080;
# server_name somename alias another.alias;
# location / {
# root html;
# index index.html index.htm;
# }
#}
# HTTPS server
#
#server {
# listen 443;
# server_name localhost;
# ssl on;
# ssl_certificate cert.pem;
# ssl_certificate_key cert.key;
# ssl_session_timeout 5m;
# ssl_protocols SSLv2 SSLv3 TLSv1;
# ssl_ciphers HIGH:!aNULL:!MD5;
# ssl_prefer_server_ciphers on;
# location / {
# root html;
# index index.html index.htm;
# }
#}
}
站点的conf文件如下：
server {
listen 80;
server_name computer.cdblp.cn;
access_log /home/wamdm/sites/logs/computer.access.log main;
error_log /home/wamdm/sites/logs/computer.error.log error;
root /home/wamdm/sites/searchscholar/computer;
index index.php index.html index.htm;
rewrite "^/conference/([^/]+)$" /con_detail.php?con_title=$1 last;
rewrite "^/conference/([^/]+)/$" /con_detail.php?con_title=$1 last;
if ($http_user_agent ~* "qihoobot|Baiduspider|Googlebot|Googlebot-Mobile|Googlebot-Image|Mediapartners-Google|Adsbot-Google|Feedfetcher-Google|Yahoo! Slurp|Yahoo! Slurp China|YoudaoBot|Sosospider|Sogou spider|Sogou web spider|MSNBot|ia_archiver|Tomato Bot") {
return 403;
}
#大小转换的补丁，处理从windows平台（大小写不敏感）迁移到ubuntu（大小写敏感）的站点
#对于需要url重写生效的请求失效
#if ( !-e $request_filename ) {
# rewrite ^(.*)$ $fix_upper_lower_case last;
#}
#location /{
# include agent_deny.conf;
# }
#favicon.ico不用打日志
location = /favicon.ico {
log_not_found off;
access_log off;
}
#不允许访问隐藏文件
location ~ /\. {
deny all;
access_log off;
log_not_found off;
}
#访问图片，flash文件等不用打日志
location ~ .*\.(gif|jpg|jpeg|png|bmp|swf)$ {
expires 7d; #文件返回的过期时间是7天
access_log off;
}
#访问js和css文件不用打日志
location ~ .*\.(js|css)?$ {
expires 1d; #文件返回的过期时间是1天
access_log off;
}
#设置php-cgi
location ~ [^/]\.php(/|$) {
fastcgi_split_path_info ^(.+?\.php)(/.*)$;
#拦截不存在的php页面请求
if (!-f $document_root$fastcgi_script_name) {
return 404;
}
}
}
