# --------------------------------------------------------------------------------- 
# Author: Dean Stringer
# Description/Purpose:
#    Used to test for HTTP errors (404's etc). Parses a filename
#    specified on the command line and for each URL performs
#    an LWP fetch reporting on any non successful HTTP responses
# --------------------------------------------------------------------------------- 

use strict;
use LWP::UserAgent;
my $ua = LWP::UserAgent->new;
$ua->timeout(10);    # wait 10 seconds

my $debugMode = 0;
my %checkURLs;

my $inFile = $ARGV[0];
die "No such file $inFile" unless -e $inFile;

open(INFILE, "< $inFile");
while(my $line = <INFILE>) {
    my ($url, $extra) = split('\t', $line);
    if ($url =~ /^http:\/\/([\w\/\.\?\-\=_])+$/) {
        $checkURLs{$url} = chomp($extra);
    } else {
        if ($debugMode) { print "\nCant Handle $url"; }
    }
}
close INFILE;

foreach my $key (keys %checkURLs) {
    my $req = HTTP::Request->new( GET => $key );
    # perform a 'simple_request', normal 'request' follows redirects which 
    # is not what we want
    my $response = $ua->simple_request($req);
    if ($debugMode) { print "\n" . $response->status_line; }
    unless ($response->is_success) {    #  <--  see man page for HTTP::Response
        print $response->code . " : " . $key;
        if ($debugMode) {
            foreach (keys %{$response->headers()}) {
                print "\n$_ = " . $response->headers()->{$_};
            }
        }
    }
}