#!/usr/bin/perl -w

# Find duplicates in url list

use strict;

open(URLS, "url_list.txt") || die($!);
my @lines = <URLS>;
close URLS;

my $line_count = @lines;

for (my $i = 0; $i < $line_count-1; $i++) {
	my $url = $lines[$i];
	chomp($url);
	
	for (my $j = $i+1; $j < $line_count; $j++) {
		my $url_comp = $lines[$j];
		chomp($url_comp);
		
		my $new_url = $url;
		$new_url .= "/";
			
		if ($url_comp eq $url) {
			print "$url\t$url_comp\n";
		}
		
		if ($url_comp eq $new_url) {
			print "$url\t$url_comp\n";
		}
		
		if ($url_comp eq $new_url . "index.html") {
			print "$url\t$url_comp\n";
		}
		
		if ($url_comp eq $new_url . "index.htm") {
			print "$url\t$url_comp\n";
		}
		
		if ($url_comp eq $new_url . "default.htm") {
			print "$url\t$url_comp\n";
		}
	}
}