#!/usr/bin/perl -w

# Print http results for all urls

use strict;

my $url_list = "urls_unique.txt";

my %urls;
open(FILE, $url_list) || die("Can't open $url_list: $!");
while (my $line = <FILE>) {
	chomp($line);
	$urls{$line} = 1;
}
close FILE;

my @files = glob("http_responses/*.txt");

print "date\turl\thttp code\n";

foreach my $file (@files) {
	open(FILE, $file) || die("Can't open $file: $!\n");
	while (my $line = <FILE>) {
		chomp($line);
		if ($line !~ m|^http://www.dlib.org| &&
		    $line !~ m|^http://dx.doi.org/10.1045/|) {
			my ($url, $len, $code) = split(/ /, $line);

			# Some urls that were tested weren't legal
			next if (!defined $urls{$url});
			
			my ($date) = ($file =~ /(\d\d\d\d-\d\d-\d\d)/);

			$code = 200 if ($code == 304 || $code == 301 || $code == 302);

			if ($code eq '' || $len eq '') { 
				#print STDERR "error in [$line] from [$file]\n";
				$len = 9999;
			}

			$code = "200-0" if ($code == 200 && $len == 0);

			print "$date\t$url\t$code\n";
		}
	}
	close FILE;
}