Robots.txt collector/scanner
PERL CODE:
use Socket;
@sites = qw( URL );
foreach $site (@sites) {
# resolve host name
$iaddr = inet_aton($site);
unless ($iaddr) {
print "Can't resolve $iaddr\n";
next;
}
$paddr = sockaddr_in(80,$iaddr);
$proto = getprotobyname("tcp");
socket (SOCK, PF_INET, SOCK_STREAM, $proto) ;
connect (SOCK, $paddr) ;
select SOCK;
$|=1;
print "GET /robots.txt HTTP/1.1\r\nhost: $site\r\n\r\n";
@response = <SOCK>;
$|=0;
select STDOUT;
print "Response from $site was ",@response+0," lines\n";
open (FH,">$site.robo");
print FH @response;
}
-
ericsouthga -
Thanks
{{ DiscussionBoard.errors[5071543].message }} -