=**************************************
    = Name: Wb Spider
    = Description:Webspider is a Perl script
    =     that, when given a start page, will "fol
    =     low" every link it finds, scanning the H
    =     TML code for the use of CGI's. After Web
    =     spider has gone over all available links
    =     it will report every CGI used on the web
    =     site.
    = By: nfs
    =
    =This code is copyrighted and has    = limited warranties.Please see http://w
    =     ww.Planet-Source-Code.com/vb/scripts/Sho
    =     wCode.asp?txtCodeId=309&lngWId;=6    =for details.    =**************************************
    
    # Hmmm, why on earth would we need a socket ?
    use Socket;
    sub preps() {
    if ($ARGV[2] eq '') { 
    print "\n\nUsage: perl webspider_1.1.pl <proxy server> <proxy port> <URL>\n";
    print "Example: perl webspider_1.1.pl proxy.pandora.be 8080 http://www.microsoft.com/\n";
    exit;
    }
    $proxy = $ARGV[0];
    $port = $ARGV[1];
    @currentlayer[0] = $ARGV[2];
    $layer = "10";
    $maxcurrentlayerteller = "100";
    $noname = "WebSpider 1.1";
    @currentlayer[$currentlayerteller] =~ s/http:\/\///g ;
    ($server, $dir, $file) = split(/\//, @currentlayer[$currentlayerteller]);
    $logfile = "WebSpider_Log.txt";
    @currentlayer[$currentlayerteller] = "http://@currentlayer[$currentlayerteller]";
    @dontignore[1] = ".html";
    @dontignore[2] = ".xml";
    @dontignore[3] = ".asp";
    @dontignore[4] = ".php";
    @dontignore[5] = ".htm";
    $prepsdontignoreteller = 0 ;
    while (@dontignore[$prepsdontignoreteller] ne '') { print "Don\'t Ignore: @dontignore[$prepsdontignoreteller]\n"; $prepsdontignoreteller++; }
    }
    sub LogToFile() {
    open(OUTF, ">>$logfile");
    print OUTF "$layerteller $currentlayerteller @foundcgi[$foundcgiteller] http://@currentlayer[$currentlayerteller]\n";
    close(OUTF);
    }
    sub CheckCGIHistory() {
    $cgihistoryteller = 0 ;
    $cgiwasinhistory = 0 ;
    while (@cgihistory[$cgihistoryteller] ne '') { if (@cgihistory[$cgihistoryteller] eq @foundcgi[$foundcgiteller]) { $cgiwasinhistory = 1; } $cgihistoryteller++; }
    if ($cgiwasinhistory != 0) { $foundcgiteller-- ; } else { @cgihistory[$cgihistoryteller] = @foundcgi[$foundcgiteller] ; print "$layerteller:$currentlayerteller @foundcgi[$foundcgiteller]\n"; LogToFile(); }
    }
    sub CheckHistory() {
    $historyteller = 0 ;
    $wasinhistory = 0 ;
    while (@history[$historyteller] ne '') { 
    if (@history[$historyteller] eq @nextlayer[$nextlayerteller]) { 
    $wasinhistory = 1; 
    $placeinhistory = $historyteller ;
    } 
    $historyteller++; 
    }
    if ($wasinhistory == 0) { 
    @history[$historyteller] = @nextlayer[$nextlayerteller] ;
    } else { 
    @nextlayer[$nextlayerteller] = "";
    $nextlayerteller-- ; 
    }
    }
    sub itcontainslocation() {
    ($temp, $link) = split(/ /, @response[$responseteller]);
    if ($link =~ /(.*)http:\/\/(.*)/) { @nextlayer[$nextlayerteller] = "$link"; } else { @nextlayer[$nextlayerteller] = "http://$server/"; if ($dir ne '') { @nextlayer[$nextlayerteller] = "@nextlayer[$nextlayerteller]$dir/"; } else { @nextlayer[$nextlayerteller] = "@nextlayer[$nextlayerteller]$link"; } }
    CheckHistory() ;
    $nextlayerteller++ ;
    }
    sub itcontainshref() {
    ($temp, $therest) = split(/href=\"/, @response[$responseteller]);
    ($link,$temp) = split(/\"/, $therest);
    if ($link =~ /(.*)http:\/\/(.*)/) { @nextlayer[$nextlayerteller] = "$link"; } else { @nextlayer[$nextlayerteller] = "http://$server/"; if ($dir ne '') { @nextlayer[$nextlayerteller] = "@nextlayer[$nextlayerteller]$dir/"; } else { @nextlayer[$nextlayerteller] = "@nextlayer[$nextlayerteller]$link"; } }
    CheckHistory() ;
    $nextlayerteller++ ;
    }
    sub itcontainsscr() {
    ($temp, $therest) = split(/scr=\"/, @response[$responseteller]);
    ($link,$temp) = split(/\"/, $therest);
    if ($link =~ /(.*)http:\/\/(.*)/) { @nextlayer[$nextlayerteller] = "$link"; } else { @nextlayer[$nextlayerteller] = "http://$server/"; if ($dir ne '') { @nextlayer[$nextlayerteller] = "@nextlayer[$nextlayerteller]$dir/"; } else { @nextlayer[$nextlayerteller] = "@nextlayer[$nextlayerteller]$link"; } }
    CheckHistory() ;
    $nextlayerteller++ ;
    }
    sub itcontainsaction() {
    ($temp, $therest) = split(/action=\"/, @response[$responseteller]);
    ($cgi,$temp) = split(/\"/, $therest);
    if ($cgi =~ /(.*)http:\/\/(.*)/) { $tempfoundcgi = "$cgi"; } else { $tempfoundcgi = "http://$server/"; if ($dir ne '') { $tempfoundcgi = "$tempfoundcgi$dir/$cgi"; } else { $tempfoundcgi = "$tempfoundcgi$cgi"; } }
    @foundcgi[$foundcgiteller] = $tempfoundcgi ;
    CheckCGIHistory() ;
    $foundcgiteller++ ;
    }
    sub parse() {
    $serverIP = inet_aton($proxy);
    $serverAddr = sockaddr_in($port, $serverIP);
    socket(SOCKET, PF_INET, SOCK_STREAM, getprotobyname('tcp')); 
    if (!connect(SOCKET, $serverAddr)) { print "Could not connect, try another proxy server.\n"; exit ; }
    # Send the URL 
    print "Sending: GET http://@currentlayer[$currentlayerteller] HTTP/1.0\n";
    send(SOCKET,"GET http://@currentlayer[$currentlayerteller] HTTP/1.0\n\n",0);
    @response=<SOCKET>;
    $responseteller = 0 ;
    while (@response[$responseteller] ne '') {
    chomp (@response[$responseteller]);
    # Convert everything to lowercase...
    @response[$responseteller] = "\L@response[$responseteller]\E";
    # If we get a 302...
    if (@response[$responseteller] =~ /(.*)Location:(.*)/) { itcontainslocation() ; }
    # If we get a 200...
    if (@response[$responseteller] =~ /(.*)href=(.*)/) { 
    $dontignoreteller = 0 ;
    $dontignoreit = 0 ;
    # If the link is not in the @dontignore-list, $dontignoreit stays 0
    while(@dontignore[$dontignoreteller] ne '') { if (@response[$responseteller] =~ /(.*)@dontignore[$dontignoreteller](.*)/) { $dontignoreit = 1 ; } $dontignoreteller++; }
    if ($dontignoreit == 0) { itcontainshref(); }
    }
    # Site has frames...
    if (@response[$responseteller] =~ /(.*)scr=(.*)/) { itcontainsscr() ; }
    # CGI found...
    if (@response[$responseteller] =~ /(.*)action=(.*)/) { itcontainsaction() ; }
    $responseteller++;
    }
    }
    ################
    # MAIN PROGGIE #
    ################
    print "\nPreparing...";
    preps();
    print "Done.\n";
    for ($layerteller=0;$layerteller<$layer;$layerteller++) {
    for ($currentlayerteller=0;$currentlayerteller<$maxcurrentlayerteller;$currentlayerteller++) {
    @currentlayer[$currentlayerteller] =~ s/http:\/\///g ;
    ($server, $dir, $file) = split(/\//, @currentlayer[$currentlayerteller]);
    if (@currentlayer[$currentlayerteller] ne '') { parse(); }
    }
    @currentlayer = @nextlayer ;
    $nextlayerteller = 0 ;
    }