Platforma de hosting cu software liber, gratuit, open source.

#!/usr/bin/perl
 
use strict;
use warnings;
 
use Getopt::Long;
use WWW::Mechanize;
 
my $tmpFile = "/tmp/crawl.$$";
my $delay   = 1;
my $hostname;
my $start;
 
GetOptions( "start=s" => \$start );
 
$hostname = &getHostname($start) or die("Invalid URL.\n");
 
open( LINKS,   "+>", $tmpFile );
die("Cannot create links file.\n") if ($@);
 
# Start crawling
&getLinks($start);
 
close(LINKS);
# unlink($tmpFile);
 
# Subrutines
# Get the hostname for the given URL
sub getHostname {
        my $url = shift;
        return 0 if ( $url !~ /^http:\/\/([^\/]+)\/?/ );
        return $1;
}
 
# Get links recursively
sub getLinks {
        my $url = shift;
 
        no warnings 'recursion';
 
        my $mech = WWW::Mechanize->new( agent => 'Pinochio 1.33.7' );
        eval { $mech->get($url); };
 
        next if ($@);
 
        my @links = $mech->links();
 
        foreach my $link (@links) {
                my $url = $link->url();
                $url = "http://$hostname" . $link->url() if ( $url =~ /^\// );
 
                # Blacklist
                next if( $url =~ /\.(avi|mp3|mpg|mpeg|swf|png|gif|jpg|jpeg|bmp|wav)/ );
                next if( $url !~ /^http:\/\/$hostname/ );
                next if( &saved( $url ) );
 
                &save( $url );
                &getLinks($url);
        }
 
        sleep($delay);
 
}
 
# Check to see if an insite URL is saved
sub saved {
        my $url = shift;
        chomp $url;
 
        seek( LINKS, 0, 0 );
        while (<links>) {
                chomp;
                return 1 if ( $url eq $_ );
        }
 
        return 0;
}
 
# Adds to LINKS file the URL passed as an argument
sub save {
        my $url = lc shift;
 
        seek( LINKS, 0, 2 );
        return 1 if ( print LINKS "$url\n" );
        return 0;
}
</links>

[root@lamp ~/crawl]# ./crawl.pl --start=http://www.regex.ro