#!/usr/bin/perl -w

# Print http results for all urls

use strict;

my $url_list = "urls_unique.txt";

my %urls;
open(FILE, $url_list) || die("Can't open $url_list: $!");
while (my $line = <FILE>) {
	chomp($line);
	$urls{$line} = 1;
}
close FILE;

my @files = glob("http_responses/*.txt");

my %seen;
my %http_codes;

foreach my $file (@files) {
	open(FILE, $file) || die("Can't open $file: $!\n");
	while (my $line = <FILE>) {
		chomp($line);
		if ($line !~ m|^http://www.dlib.org| &&
		    $line !~ m|^http://dx.doi.org/10.1045/|) {
			my ($url, $len, $code) = split(/ /, $line);

			# Some urls that were tested weren't legal
			next if (!defined $urls{$url});
		
			my ($date) = ($file =~ /(\d\d\d\d-\d\d-\d\d)/);

			$code = 200 if ($code == 304 || $code == 301 || $code == 302);
	
			$len = 9999 if ($len eq '');

			$code = "200-0" if ($code == 200 && $len == 0);

			$seen{$date}->{$code}++;
			$http_codes{$code} = 1;
		}
	}
	close FILE;
}

print "date\t";

foreach (sort keys %http_codes) {
	print "$_\t";
}
print "total_urls\n";

foreach my $date (sort keys %seen) {
	#my %codes = $seen{$date};
	print "$date\t";

	my $total_urls = 0;

	# We may want to change this
	#my $available = $seen{$date}->{'200'};
	#print "$available\t";

	foreach my $code (sort keys %http_codes) {
	#foreach my $code (sort keys %{ $seen{$date} }) {
		my $count = $seen{$date}->{$code};
		$count = 0 if (!defined $count);
		print "$count\t";	
		$total_urls += $count;
	}
	print "$total_urls\n";
}