#!/usr/bin/perl -w

# Print http results for all urls

use strict;

my $url_list = "urls_unique.txt";

my @files = glob("http_responses/*.txt");

my %seen;

foreach my $file (@files) {
	open(FILE, $file) || die("Can't open $file: $!\n");
	while (my $line = <FILE>) {
		chomp($line);
		if ($line !~ m|^http://www.dlib.org| &&
		    $line !~ m|^http://dx.doi.org/10.1045/|) {
			if (!defined $seen{$line} && $line !~ m| 200| && $line !~ m| 304|) {
				print "$file\t$line\n";
				$seen{$line} = 1;
			}

		}
	}
	close FILE;
}