# parking-lot-RED-PI-REM.tcl
#
# Based on rbp_demo.tcl by John Heidemann (johnh@isi.edu) and 
# parking_lot_topo.tcl by Dina Katabi (dk@mit.edu) 
#
# Michele Weigle (mcweigle@unc.edu)
# last updated: June 21, 2003
#

# useful globals
set NSFILE parking-lot
set DST 0
set SRC 1
set NODES_PER_Q 4;      # nodes per queue
set bindir /home2/mweigle/multi-cl/bin

Class MySim;                  # class declaration

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Usage
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

proc usage {} {
    global NSFILE
    puts stderr "usage: ns $NSFILE.tcl [variable value]"
    puts stderr {       # EXPERIMENT VARS
       ID             EXP      - experiment identifier (filename prefix)
       duration       1        - length of measured simulation (seconds)
       warmup         0        - warmup interval before tracing (seconds)
       run            1        - experiment number 
       substream      0        - pick substream or use run (0 = use run)
       stagger        0        - space between starting FTP flows (seconds)
       scheduler      Heap     - event scheduler (Calendar or Heap)
       pairs          0        - goal pairs

       # TOPOLOGY VARS
       queues         8        - number of intermediate queues
       flows          0        - FTP flows connected to each router
       rev-flows      0        - FTP flows connected to each router rev path
       PMrate         0        - PM rate at each node at each router
       rev-PMrate     0        - PM rate at each node at each router rev path
       PMnodes        5        - nodes per queue with PM
       HTTP-1.1       1        - use HTTP 1.1
       bw             10       - bandwidth of links (Mbps)
       delay          1        - delay per link (ms)
       qlen           0        - queue length in packets (0 = use qlen-factor)
       qlen-factor    2        - queue length = factor * max win size
       queue-method   DropTail - queue algorithm  {DropTail, RED, PI, REM}

       # AQM VARS
       gentle         1        - use gentle RED
       ared           1        - use Adaptive RED
       delaytarget    0.01     - target_delay for ARED (seconds)
       avgpktsz       1320     - avg pkt size for ARED (bytes)
       ecn            1        - use ECN
       qib            0        - keep track of queue in bytes (byte-mode)
       qref           20       - qref setting for PI and REM 

       # TCP VARS
       tcptype        Sack     - TCP type at each cloud/node {OTT, Reno, Sack}
       segsperack     2        - segments per ACK (2 = delayed ACKs)
       window         103      - max TCP window size (packets)
       segsize        1420     - default TCP segment size (bytes)
       ssthresh       0        - default ssthresh in packets (0 = use default)
       init-rtx       6        - initial rexmit TO (for SYN packets)

       # LOGGING VARS
       trace          0        - links to trace packets
                                 ex: "1 0 0 0 1" trace src and dst all hops
       PMdump         1        - save info about PackMime clouds
       tcpvars        0        - save info about TCP variables
       cwnd-fid       0        - list of fids to trace cwnd
       datadir        ../data  - directory to store output
       nam            0        - run nam
       out            0        - print debugging output
       debug          0        - PackMime debugging level
    }
    exit 1
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Command-Line Option Default Values
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

proc default_options {} {
    global opts

    set raw_opt_info {
	# to add a new command-line option, insert the option name and default 
	# value inside this structure
	ID EXP
	duration 1
	warmup 0
	run 1
	substream 0
	stagger 0
	scheduler Heap
	pairs 0
	# TOPOLOGY
	queues 8
	flows 0
	rev-flows 0
	PMrate 0
	rev-PMrate 0
	PMnodes 5
	HTTP-1.1 1
	bw 10
	delay 1
	qlen 0
	qlen-factor 2
	queue-method DropTail
	# AQM
	gentle 1
	ared 1
	delaytarget 0.01
	avgpktsz 1460
	ecn 1
	qib 0
        qref 20
	# TCP 
	tcptype Sack
	tcp Sack
	segsperack 2
	window 103
	segsize 1420
	ssthresh 0
	init-rtx 6
	# LOGGING
	trace 0
	cwnd-fid 0
	datadir ../data
	PMdump 1
	tcpvars 0
	nam 0
	out 0
	debug 0
    }

    # parses the default options and creates the $opts array
    while {$raw_opt_info != ""} {
	if {![regexp "^\[^\n\]*\n" $raw_opt_info line]} {
	    break
	}
	regsub "^\[^\n\]*\n" $raw_opt_info {} raw_opt_info
	set line [string trim $line]
	if {[regexp "^\[ \t\]*#" $line]} {
	    continue
	}
	if {$line == ""} {
	    continue
	} 
	set result [regexp {^([^ ]+)[ ]+([^\n]+)$} $line dummy key value]
	if {$result == 1} {
	    set opts($key) $value
	} else {
	    puts "Error processing raw_opt_info: $line"
	    exit
	}
    }
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Process Command Line Arguments 
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

proc process_args {} {
    global argc argv opts NSFILE

    exec touch TMP; exec rm TMP; 
    exec echo -n "ns $NSFILE.tcl " > TMP 

    # read in defaults
    default_options

    # process command line args
    for {set i 0} {$i < $argc} {incr i} {
	set key [lindex $argv $i]
	if {$key == "-?" || $key == "--help" || $key == "-help" \
		|| $key == "-h"} {
	    usage
	}
	regsub {^-} $key {} key
	if {![info exists opts($key)]} {
	    puts stderr "unknown option $key";
	    usage
	}
	exec printf "[lindex $argv $i] " >> TMP
	incr i
	set opts($key) [lindex $argv $i]
	exec printf "[lindex $argv $i] " >> TMP
    }

    # save command line args to $datadir/EXPS
    exec printf "\n" >> TMP
    exec mkdir -p $opts(datadir)
    exec touch $opts(datadir)/EXPS
    exec cat TMP >> $opts(datadir)/EXPS
    exec rm TMP
    set time [exec date "+%H:%M:%S (%m/%d/%y)"]
    exec echo -n "# START: $time  " >> $opts(datadir)/EXPS
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# MAIN
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

proc main {} {
    global argv opts
    process_args
    new MySim
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Helper function to create duplex links
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

proc my-duplex-link {ns n1 n2 bw delay queue_method queue_length} {
    $ns duplex-link $n1 $n2 $bw $delay $queue_method
    $ns queue-limit $n1 $n2 $queue_length
    $ns queue-limit $n2 $n1 $queue_length
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Helper function to create Full-TCP Agents
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

proc create-agent {ns id node role dst_tcp chan file tcptype} {
    global opts SRC DST

    set flows [lindex $opts(flows) 0]

    # create new Agent
    if {$tcptype != "OTT"} {
	set tcp [new Agent/TCP/FullTcp/$tcptype]
    } else {
	set tcp [new Agent/TCP/FullTcp/OTT]
    }
    $tcp set fid_ $id

    # set parameters
    if {$opts(tcpvars) && $chan != "" && $id < $flows} {
	$tcp trace cwnd_
	$tcp attach $chan
    }

    # attach agent to node
    $ns attach-agent $node $tcp

    if {$role == $SRC} {
	$ns connect $tcp $dst_tcp
    } else {
	$tcp listen
    }

    return $tcp
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Helper function to create PackMime node
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc create-PM {rate srcind dstind tcptype} {
    global opts SRC DST
    $self instvar ns_ PM_ src_ dst_ PMrng_ PMrv_ file_
    
    # create new PackMime
    set PM_($srcind) [new PackMimeHTTP]
	
    $PM_($srcind) set-server $src_($srcind)
    $PM_($srcind) set-client $dst_($dstind)

    # setup PackMime RNG and random variables
    set substream $opts(run)
    if {$opts(substream) > 0} {
	set substream $opts(substream)
    }
    for {set i 0} {$i < 3} {incr i} {
	set PMrng_($srcind,$i) [new RNG]
	for {set j 1} {$j < $substream} {incr j} {
	    # set to appropriate substream
	    $PMrng_($srcind,$i) next-substream
	}
    }
    # PM client file size (request size)
    set PMrv_($srcind,0) [new RandomVariable/PackMimeHTTPFileSize $rate $DST $PMrng_($srcind,0)]   
    # PM server file size (response size)
    set PMrv_($srcind,1) [new RandomVariable/PackMimeHTTPFileSize $rate $SRC $PMrng_($srcind,1)] 
    # PM flow starting interval
    set PMrv_($srcind,2) [new RandomVariable/PackMimeHTTPFlowArrive $rate $PMrng_($srcind,2)] 

    # set PackMime parameters
    $PM_($srcind) set-req_size $PMrv_($srcind,0)
    $PM_($srcind) set-rsp_size $PMrv_($srcind,1)
    $PM_($srcind) set-flow_arrive $PMrv_($srcind,2)
    $PM_($srcind) set-rate $rate
    # set global opts
    $PM_($srcind) set-ID $srcind
    $PM_($srcind) set-warmup $opts(warmup)
    $PM_($srcind) set-debug $opts(debug)
    if {$opts(HTTP-1.1) == 1} {
	$PM_($srcind) set-1.1
    }
    
    # pass $opts to PackMime
    $PM_($srcind) set-opts $srcind { $opts }

    # HARD-CODED
    if {$opts(PMdump) && $srcind < 4} {   
	# only dump the end-to-end forward path PM nodes
	$PM_($srcind) set-outfile "$file_-PM${srcind}.dat"
    }

    # setup DelayBox rules for this PM
    $self init_delaybox $rate $srcind $dstind
}

#
# Initialize Routers as DelayBox nodes
#
MySim instproc init_delaybox {rate srcind dstind} {
    global opts SRC DST 
    $self instvar ns_ end_ src_ dst_ q_ DBrng_ DBrv_ file_

    # Each router is responsible for delaying pckts to and from
    # the nodes connected to it.
    set trace_delays 0
    if {$srcind < 4} {
	set srcq 0
	set dstq 7
	set trace_delays 1
    } elseif {$srcind < 6} {
	set srcq 1
	set dstq 2
    } elseif {$srcind < 8} {
	set srcq 2
	set dstq 1
    } elseif {$srcind < 10} {
	set srcq 3
	set dstq 4
    } elseif {$srcind < 12} {
	set srcq 4
	set dstq 3
    } elseif {$srcind < 14} {
	set srcq 5
	set dstq 6
    } elseif {$srcind < 16} {
	set srcq 6
	set dstq 5
    } elseif {$srcind < 20} {
	set srcq 7
	set dstq 0
    }

#    puts stderr "srcind: $srcind  srcq: $srcq  dstq: $dstq"

    # setup DelayBox RNG and random variables
    set DBrng_($SRC,$srcind) [new RNG]
    set DBrng_($DST,$dstind) [new RNG]
    set substream $opts(run)
    if {$opts(substream) > 0} {
	set substream $opts(substream)
    }
    for {set j 1} {$j < $substream} {incr j} {
	# set to appropriate substream
	$DBrng_($SRC,$srcind) next-substream
	$DBrng_($DST,$dstind) next-substream
    }

    # setup RTT distribution
    set DBrv_($SRC,$srcind) [new RandomVariable/PackMimeHTTPXmit $rate $SRC $DBrng_($SRC,$srcind)]
    set DBrv_($DST,$dstind) [new RandomVariable/PackMimeHTTPXmit $rate $DST $DBrng_($DST,$dstind)]
    $q_($srcq) add-rule [$src_($srcind) id] [$dst_($dstind) id] $DBrv_($SRC,$srcind)	
    $q_($dstq) add-rule [$src_($srcind) id] [$dst_($dstind) id] $DBrv_($DST,$dstind)

    # setup file to save delays
    if {$trace_delays} {
	$q_($srcq) set-delay-file $file_-delays-$srcq.dat
	$q_($dstq) set-delay-file $file_-delays-$dstq.dat
	$ns_ at $end_ "$q_($srcq) close-delay-file"
	$ns_ at $end_ "$q_($dstq) close-delay-file"
    }
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Initialize vars for scheduling
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
MySim instproc init_schedule {} {
    global opts
    $self instvar end_ fin_

    # end simulation
    set end_ [expr $opts(warmup) + $opts(duration)]
    
    # send FIN (close FTP flows)
    set fin_ [expr $end_ - 0.5]
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
#  Initialize Network
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc init_network {} {
    global opts NODES_PER_Q
    $self instvar ns_ q_ src_ dst_ nsrc_ ndst_

    set qtype DropTail
    if {[llength $opts(queue-method)] == 3} {
	if {[lindex $opts(queue-method) 0] != "DropTail"} {
	    set qtype [lindex $opts(queue-method) 0]
	}
	if {[lindex $opts(queue-method) 1] != "DropTail"} {
	    set qtype [lindex $opts(queue-method) 1]
	}
	if {[lindex $opts(queue-method) 2] != "DropTail"} {
	    set qtype [lindex $opts(queue-method) 2]
	}
    }

    if {$qtype == "RED"} {
	# set global RED options
	# default parameters are automatic settings for RED
	if {$opts(ecn) == 1} {
	    Queue/RED set setbit_ true;  # use ECN marking
	}
	if {$opts(gentle) == 1} {
	    Queue/RED set gentle_ true;  # use gentle RED
	} else {
	    Queue/RED set gentle_ false; # don't use gentle RED
	}
	if {$opts(ared) == 1} {
	    Queue/RED set adaptive_ 1;   # use adaptive RED
	    Queue/RED set gentle_ true;  # use gentle RED
	}
	if {$opts(qib) == 1} {
	    # byte-mode
	    Queue/RED set bytes_ true  
	    Queue/RED set queue_in_bytes_ true
	}
	Queue/RED set targetdelay_ $opts(delaytarget)
	Queue/RED set mean_pktsize_ $opts(avgpktsz)
    }
    if {$qtype == "PI"} {
	# use default settings for all other PI parameters
	if {$opts(ecn) == 1} {
	    Queue/PI set setbit_ true;    # use ECN marking
	}
	if {$opts(qib) == 1} {
	    # byte-mode
	    Queue/PI set bytes_ true  
	    Queue/PI set queue_in_bytes_ true
	}
	Queue/PI set mean_pktsize_ $opts(avgpktsz)
	Queue/PI set qref_ $opts(qref)
    }
    if {$qtype == "REM"} { 
	# use default settings for all other REM parameters
	if {$opts(ecn) == 1} {
	    Queue/REM set markpkts_ true;  # use ECN marking
	}
	if {$opts(qib) == 1} {
	    # byte-mode
	    Queue/REM set qib_ true
	}
	Queue/REM set mean_pktsize_ $opts(avgpktsz)
	Queue/REM set pbo_ $opts(qref)
    }


    # create routers
    for {set i 0} {$i < $opts(queues)} {incr i} {
	set q_($i) [$ns_ DelayBox]
    }

    # create sources
    set nsrc_ [expr $opts(queues) * $NODES_PER_Q / 2]
    for {set i 0} {$i < $nsrc_} {incr i} {
	set src_($i) [$ns_ node]
    }
    # create destinations
    set ndst_ [expr $opts(queues) * $NODES_PER_Q / 2]
    for {set i 0} {$i < $ndst_} {incr i} {
	set dst_($i) [$ns_ node]
    }
    # create e2e rev path sources
    for {set i $nsrc_} {$i < [expr $nsrc_ + 4]} {incr i} {
	set src_($i) [$ns_ node]
    }
    
    # create e2e rev path destinations
    for {set i $ndst_} {$i < [expr $ndst_ + 4]} {incr i} {
	set dst_($i) [$ns_ node]
    }

    $self build-parking-lot
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
#  Parking Lot Topology
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
#           src1  src2         src3         src4
# src0                                                   dst0
#     - Q0 - Q1 - Q2 - ... - Q(queues-2) - Q(queues-1) -
# dst5                                                   src5
#           dst2  dst1/3       dst2/4      dst3
#
MySim instproc build-parking-lot {} {
    global opts
    $self instvar ns_ q_ src_ dst_ nsrc_ ndst_

    set qlen $opts(qlen)
    if {($opts(qlen) == 0) && ($opts(qlen-factor) != 0)} {
	set qlen [expr $opts(qlen-factor) * $opts(window)]
    }

    # link routers together
    set bw ${opts(bw)}Mbps
    set delay 1ms

    # link routers together
    set qtype DropTail
    if {[llength $opts(queue-method)] == 3} {
	if {[lindex $opts(queue-method) 0] != "DropTail"} {
	    set qtype [lindex $opts(queue-method) 0]
	}
	if {[lindex $opts(queue-method) 1] != "DropTail"} {
	    set qtype [lindex $opts(queue-method) 1]
	}
	if {[lindex $opts(queue-method) 2] != "DropTail"} {
	    set qtype [lindex $opts(queue-method) 2]
	}
	my-duplex-link $ns_ $q_(0) $q_(1) $bw $delay DropTail $qlen
	my-duplex-link $ns_ $q_(1) $q_(2) $bw $delay [lindex $opts(queue-method) 0] $qlen
	if {$opts(out)} {
	    puts stderr "Q1-Q2: [lindex $opts(queue-method) 0]"
	}
	my-duplex-link $ns_ $q_(2) $q_(3) $bw $delay $qtype $qlen
	my-duplex-link $ns_ $q_(3) $q_(4) $bw $delay [lindex $opts(queue-method) 1] $qlen
	if {$opts(out)} {
	    puts stderr "Q3-Q4: [lindex $opts(queue-method) 1]"
	}
	my-duplex-link $ns_ $q_(4) $q_(5) $bw $delay $qtype $qlen
	my-duplex-link $ns_ $q_(5) $q_(6) $bw $delay [lindex $opts(queue-method) 2] $qlen
	if {$opts(out)} {
	    puts stderr "Q5-Q6: [lindex $opts(queue-method) 2]"
	}
	my-duplex-link $ns_ $q_(6) $q_(7) $bw $delay DropTail $qlen
    } else {
	my-duplex-link $ns_ $q_(0) $q_(1) $bw $delay DropTail $qlen
	my-duplex-link $ns_ $q_(1) $q_(2) $bw $delay $opts(queue-method) $qlen
	my-duplex-link $ns_ $q_(2) $q_(3) $bw $delay $opts(queue-method) $qlen
	my-duplex-link $ns_ $q_(3) $q_(4) $bw $delay $opts(queue-method) $qlen
	my-duplex-link $ns_ $q_(4) $q_(5) $bw $delay $opts(queue-method) $qlen
	my-duplex-link $ns_ $q_(5) $q_(6) $bw $delay $opts(queue-method) $qlen
	my-duplex-link $ns_ $q_(6) $q_(7) $bw $delay DropTail $qlen
    }
    # each HTTP node has 4 Mbps (since only 2 sources at each queue, this 
    # won't overwhelm the router)
    set bw 4Mbps
    set qlen 300

    # link src and dst to appropriate router

    # Q  SRCS   DSTS  DIR
    # -  ----   ----  ---
    # 0  0,1,2,3      Fwd
    # --
    # 1  4,5    6,7   Fwd
    # 2  6,7    4,5   Rev
    # --
    # 3  8,9    10,11 Fwd
    # 4  10,11  8,9   Rev
    # --
    # 5  12,13  14,15 Fwd
    # 6  14,15  12,13 Rev
    # --
    # 7         0,1,2,3 Rev

    # Q(0)
    my-duplex-link $ns_ $src_(0) $q_(0) 1Mbps $delay DropTail $qlen; # FTP
    my-duplex-link $ns_ $src_(1) $q_(0) 1Mbps $delay DropTail $qlen; # FTP
    my-duplex-link $ns_ $src_(2) $q_(0) $bw $delay DropTail $qlen;   # HTTP
    my-duplex-link $ns_ $src_(3) $q_(0) $bw $delay DropTail $qlen;   # HTTP

    # reverse path
    my-duplex-link $ns_ $dst_(16) $q_(0) 1Mbps $delay DropTail $qlen; # HTTP
    my-duplex-link $ns_ $dst_(17) $q_(0) 1Mbps $delay DropTail $qlen; # HTTP
    my-duplex-link $ns_ $dst_(18) $q_(0) $bw $delay DropTail $qlen;   # FTP
    my-duplex-link $ns_ $dst_(19) $q_(0) $bw $delay DropTail $qlen;   # FTP

    # Q(1) - Q(6)
    set ind 4
    set qInd 1
    while {$qInd < 7} {
	set nextQInd [expr $qInd + 1]
	for {set i 0} {$i < 2} {incr i} {
	    my-duplex-link $ns_ $src_($ind) $q_($qInd) $bw $delay DropTail $qlen; # HTTP
	    my-duplex-link $ns_ $dst_($ind) $q_($nextQInd) $bw $delay DropTail $qlen; # HTTP
	    incr ind
	}
	for {set i 0} {$i < 2} {incr i} {
	    my-duplex-link $ns_ $dst_($ind) $q_($qInd) $bw $delay DropTail $qlen; # HTTP
	    my-duplex-link $ns_ $src_($ind) $q_($nextQInd) $bw $delay DropTail $qlen; # HTTP
	    incr ind
	}
	set qInd [expr $qInd + 2]
    }
 
    # Q(7)
    my-duplex-link $ns_ $dst_(0) $q_(7) 1Mbps $delay DropTail $qlen; # FTP
    my-duplex-link $ns_ $dst_(1) $q_(7) 1Mbps $delay DropTail $qlen; # FTP
    my-duplex-link $ns_ $dst_(2) $q_(7) $bw $delay DropTail $qlen;   # HTTP
    my-duplex-link $ns_ $dst_(3) $q_(7) $bw $delay DropTail $qlen;   # HTTP

    # reverse path
    my-duplex-link $ns_ $src_(16) $q_(7) $bw $delay DropTail $qlen;   # HTTP
    my-duplex-link $ns_ $src_(17) $q_(7) $bw $delay DropTail $qlen;   # HTTP
    my-duplex-link $ns_ $src_(18) $q_(7) 1Mbps $delay DropTail $qlen; # FTP
    my-duplex-link $ns_ $src_(19) $q_(7) 1Mbps $delay DropTail $qlen; # FTP
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
#  Initialize Traffic
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc init_connections {} {
    global opts

    # set global TCP options
    Agent/TCP/FullTcp set segsize_ $opts(segsize)
    Agent/TCP set window_ $opts(window)
    Agent/TCP set rtxcur_init_ $opts(init-rtx); # SYN timeout
    if {$opts(ssthresh) > 0} {
	Agent/TCP set default_ssthresh_ $opts(ssthresh)
    }    
    if {$opts(ecn)} {
	Agent/TCP set ecn_ 1
    }
    if {$opts(segsperack)} {
	Agent/TCP/FullTcp set segsperack_ $opts(segsperack)
    }

    if {[llength $opts(flows)] > 1 || [lindex $opts(flows) 0] > 0} {
	$self init_ftp_traffic
    } 
    if {[llength $opts(rev-flows)] > 1 || [lindex $opts(rev-flows) 0] > 0} {
	$self init_rev_ftp_traffic
    } 
    if {[llength $opts(PMrate)] > 1 || [lindex $opts(PMrate) 0] > 0} {
	$self init_http_traffic
	$self init_http_crosstraffic
    }
    if {[llength $opts(rev-PMrate)] > 1 || [lindex $opts(rev-PMrate) 0] > 0} {
	$self init_rev_http_traffic
    }
}   

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
#  Initialize FTP Traffic  (all hops)
#
# flows 0:n            forward path
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc init_ftp_traffic {} {
    global opts SRC DST defaultRNG
    $self instvar ns_ nsrc_ ndst_ src_ dst_ tcpd_ tcps_ file_ end_ fin_ tchan_

    # scheduling vars
    set time $opts(warmup)

    # forward path traffic
    set flows [lindex $opts(flows) 0]
    if {$flows == 0} {
	return
    }
    if {$opts(out)} {
	puts "FORWARD PATH E2E FTP:"
    }

    set ind 0
    for {set fid 0} {$fid < $flows} {incr fid} {
	set tcptype [lindex $opts(tcptype) 0]

	# TCP variable tracing
	set tchan_($fid) ""
	set tcpd_($fid) [create-agent $ns_ $fid $dst_($ind) $DST 0            $tchan_($fid) $file_ $tcptype]
	set tcps_($fid) [create-agent $ns_ $fid $src_($ind) $SRC $tcpd_($fid) $tchan_($fid) $file_ $tcptype]
	if {$opts(tcpvars)} {
	    set tchan_($fid) [open "|gzip > $file_-vars-f${fid}.tr.gz" w]
	}

	# schedule flow
	$ns_ at $time "$tcps_($fid) sendmsg -1 \" \""

	set gap 0
	if {$opts(stagger) > 0} {
	    # pick a random number between 0 and stagger for stagger
	    set gap [$defaultRNG uniform 0 $opts(stagger)]
	}
	set time [expr $time + $gap]
	$ns_ at $fin_ "$tcps_($fid) close"

	# output 
	if {$opts(out)} {
	    puts "  fid $fid: src$ind ([$src_($ind) id]) --> dst$ind ([$dst_($ind) id])"
	}

	# ind is either 0 or 1
	incr ind
	if {$ind == 2} {
	    set ind 0
	}
    }

    # HARD-CODED
    # setup delaybox rules for node 0 and 1
    $self init_delaybox 25 0 0 
    $self init_delaybox 25 1 1 

}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
#  Initialize Rev-path FTP Traffic  (all hops)
#
# flows 0:n            reverse path
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc init_rev_ftp_traffic {} {
    global opts SRC DST defaultRNG
    $self instvar ns_ nsrc_ ndst_ src_ dst_ tcpd_ tcps_ file_ end_ fin_ tchan_

    # scheduling vars
    set time $opts(warmup)

    # reverse path traffic
    set flows [lindex $opts(flows) 0]
    set lastq [expr $opts(queues) - 1]
    set rev_flows [lindex $opts(rev-flows) $lastq]
    if {$rev_flows == 0} {
	return
    }
    if {$opts(out)} {
	puts "REVERSE PATH E2E FTP:"
    }

    set ind 18;   # HARD CODED VALUE!
    for {set fid [expr $flows-1]} {$fid < [expr $flows + $rev_flows]} {incr fid} {
	set tcptype [lindex $opts(tcptype) 0]

	# TCP variable tracing
	set tchan_($fid) ""
	set tcpd_($fid) [create-agent $ns_ $fid $dst_($ind) $DST 0            $tchan_($fid) $file_ $tcptype]
	set tcps_($fid) [create-agent $ns_ $fid $src_($ind) $SRC $tcpd_($fid) $tchan_($fid) $file_ $tcptype]
	if {$opts(tcpvars)} {
	    set tchan_($fid) [open "|gzip > $file_-vars-f${fid}.tr.gz" w]
	}

	# schedule flow
	$ns_ at $time "$tcps_($fid) sendmsg -1 \" \""

	set gap 0
	if {$opts(stagger) > 0} {
	    # pick a random number between 0 and stagger for stagger
	    set gap [$defaultRNG uniform 0 $opts(stagger)]
	}
	set time [expr $time + $gap]
	$ns_ at $fin_ "$tcps_($fid) close"

	# output 
	if {$opts(out)} {
	    puts "  fid $fid: src$ind ([$src_($ind) id]) --> dst$ind ([$dst_($ind) id])"
	}

	# ind is either 18 or 19 - HARD-CODED
	incr ind
	if {$ind == 20} {
	    set ind 18
	}
    }

    # HARD-CODED
    # setup delaybox rules for node 18 and 19
    $self init_delaybox 25 18 18
    $self init_delaybox 25 19 19

}



#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
#  Initialize HTTP Traffic
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc init_http_traffic {} {
    global opts SRC DST 
    $self instvar ns_ nsrc_ ndst_ src_ dst_ file_ end_ PM_

    # scheduling vars
    set time 0.001

    # forward path
    if {$opts(out)} {
	puts "FORWARD PATH E2E HTTP:"
    }

    set rate [lindex $opts(PMrate) 0]
    set tcptype [lindex $opts(tcptype) 0]

    # ind starts at 2 (inds 0 and 1 are FTP)
    set ind 2
    while {$ind < 4} {
	if {$rate > 0} {
	    $self create-PM $rate $ind $ind $tcptype

	    # schedule flow
	    $ns_ at $time "$PM_($ind) start"
	    $ns_ at $end_ "$PM_($ind) stop"	    
	    
	    # output 
	    if {$opts(out)} {
		puts "  PM ($rate c/s): src$ind ([$src_($ind) id]) --> dst$ind ([$dst_($ind) id])"
	    }
	}
	incr ind
    }
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
#  Initialize HTTP Cross Traffic
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc init_http_crosstraffic {} {
    global opts SRC DST 
    $self instvar ns_ nsrc_ ndst_ src_ dst_ file_ end_ PM_

    # scheduling vars
    set time 0.001

    # all cross-traffic is of the same type
    set tcptype [lindex $opts(tcptype) 0]

    # forward path
    if {$opts(out)} {
	puts "FORWARD PATH CT:"
    }

    # Q  SRCS   DSTS  DIR
    # -  ----   ----  ---
    # 1  4,5    6,7   Fwd
    # 2  6,7    4,5   Rev
    # --
    # 3  8,9    10,11 Fwd
    # 4  10,11  8,9   Rev
    # --
    # 5  12,13  14,15 Fwd
    # 6  14,15  12,13 Rev

    set qInd 1
    set ind 4
    while {$qInd <= 5} {
	if {[llength $opts(PMrate)] == 1} {
	    set rate [lindex $opts(PMrate) 0]
	} else {
	    set rate [lindex $opts(PMrate) $qInd]
	}

	if {$rate > 0} {
	    for {set i 0} {$i < 2} {incr i} {
		$self create-PM $rate $ind $ind $tcptype
		$ns_ at $time "$PM_($ind) start"
		$ns_ at $end_ "$PM_($ind) stop"
		# output 
		if {$opts(out)} {
		    puts "  PM ($rate c/s): (Q $qInd) src$ind ([$src_($ind) id]) --> dst$ind ([$dst_($ind) id])"
		}
		incr ind
	    }
	}
	set ind [expr $ind + 2]
	set qInd [expr $qInd + 2]
    }


    # reverse path
    if {$opts(out)} {
	puts "REVERSE PATH CT:"
    }

    # Q  SRCS   DSTS  DIR
    # -  ----   ----  ---
    # 1  4,5    6,7   Fwd
    # 2  6,7    4,5   Rev
    # --
    # 3  8,9    10,11 Fwd
    # 4  10,11  8,9   Rev
    # --
    # 5  12,13  14,15 Fwd
    # 6  14,15  12,13 Rev

    set qInd 2
    set ind 6
    while {$qInd <= 6} {
	if {[llength $opts(rev-PMrate)] == 1} {
	    set rate [lindex $opts(rev-PMrate) 0]
	} else {
	    set rate [lindex $opts(rev-PMrate) $qInd]
	}

	if {$rate > 0} {
	    for {set i 0} {$i < 2} {incr i} {
		$self create-PM $rate $ind $ind $tcptype
		$ns_ at $time "$PM_($ind) start"
		$ns_ at $end_ "$PM_($ind) stop"
		# output 
		if {$opts(out)} {
		    puts "  PM ($rate c/s): (Q $qInd) src$ind ([$src_($ind) id]) --> dst$ind ([$dst_($ind) id])"
		}
		incr ind
	    }
	}
	set ind [expr $ind + 2]
	set qInd [expr $qInd + 2]
    }
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
#  Initialize reverse path HTTP Traffic
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc init_rev_http_traffic {} {
    global opts SRC DST 
    $self instvar ns_ nsrc_ ndst_ src_ dst_ file_ end_ PM_

    # scheduling vars
    set time 0.001

    # reverse path
    if {$opts(out)} {
	puts "REVERSE PATH E2E HTTP:"
    }

    set lastq [expr $opts(queues)-1]
    set rate [lindex $opts(rev-PMrate) $lastq]
    set tcptype [lindex $opts(tcptype) 0]

    # ind starts at 16 - HARD-CODED  (16 and 17)
    set ind 16
    while {$ind < 18} {
	if {$rate > 0} {
	    $self create-PM $rate $ind $ind $tcptype

	    # schedule flow
	    $ns_ at $time "$PM_($ind) start"
	    $ns_ at $end_ "$PM_($ind) stop"	    
	    
	    # output 
	    if {$opts(out)} {
		puts "  PM ($rate c/s): src$ind ([$src_($ind) id]) --> dst$ind ([$dst_($ind) id])"
	    }
	}
	incr ind
    }
}


#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Setup Tracing
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc trace {} {
    global opts 
    $self instvar ns_ q_ nsrc_ ndst_ src_ dst_ file_ end_ src_file_ ack_file_ dst_file_ drop_file_

    $ns_ at $end_ "$self finish"

    if {[llength $opts(trace)] == 1 && [lindex $opts(trace) 0] == 0} {
	# don't open any files if no tracing set
	return
    }

    Trace set show_tcphdr_ 1

    set drop_file_ [open "|grep \"d\" | gzip > $file_-drop.trq.gz" w]
    
    # Trace E2E Fwd Path Source Link
    if {[lindex $opts(trace) 0] == 1} {
	set src_file_ [open "|awk {{if (\$1==\"d\" || \$1==\"+\") print}} | gzip > $file_-src.trq.gz" w]
	set ack_file_ [open "|grep \"r \" | gzip > $file_-ack.trq.gz" w]
	# HARD-CODED
	for {set i 0} {$i < 4} {incr i} {
	    $ns_ trace-queue $src_($i) $q_(0) $src_file_
	    $ns_ trace-queue $q_(0) $src_($i) $ack_file_
	}
    }

    # Trace E2E Fwd Path Destination Link
    set lastind [expr [llength $opts(trace)] - 1]
    if {[llength $opts(trace)] == 1} {
	set trace [lindex $opts(trace) 0]
    } else {
	set trace [lindex $opts(trace) $lastind]
    }
    if {$trace == 1} {
	set dst_file_ [open "|grep \"r \" | gzip > $file_-dst.trq.gz" w]
	set lastq [expr $opts(queues) - 1]
	for {set i 0} {$i < 4} {incr i} {
	    $ns_ trace-queue $q_($lastq) $dst_($i) $dst_file_
	}
    }

    # Trace Intermediate Links
    for {set i 0} {$i < $lastq} {incr i} {
	set next [expr $i + 1]
	if {[llength $opts(trace)] == 1} {
	    set trace [lindex $opts(trace) 0]
	} else {
	    set trace [lindex $opts(trace) $next]
	}
	if {$trace  == 1} {
	    set trace_file_($i) [open "|gzip > $file_-$i$next.trq.gz" w]
	    $ns_ trace-queue $q_($i) $q_($next) $trace_file_($i)
	    $ns_ trace-queue $q_($next) $q_($i) $trace_file_($i)
	} 
	# trace drops
	$ns_ trace-queue $q_($i) $q_($next) $drop_file_
	$ns_ trace-queue $q_($next) $q_($i) $drop_file_
    }

    # Q  SRCS   DSTS  DIR
    # -  ----   ----  ---
    # 1  4,5    6,7   Fwd
    # 2  6,7    4,5   Rev
    # --
    # 3  8,9    10,11 Fwd
    # 4  10,11  8,9   Rev
    # --
    # 5  12,13  14,15 Fwd
    # 6  14,15  12,13 Rev

    # trace drops
    set ind 4
    set qInd 1
    while {$qInd < 7} {
	set nextQInd [expr $qInd + 1]
	for {set i 0} {$i < 2} {incr i} {
	    $ns_ trace-queue $q_($qInd) $src_($ind) $drop_file_
	    $ns_ trace-queue $src_($ind) $q_($qInd) $drop_file_
	    $ns_ trace-queue $q_($nextQInd) $dst_($ind) $drop_file_
	    $ns_ trace-queue $dst_($ind) $q_($nextQInd) $drop_file_
	    incr ind
	}
	for {set i 0} {$i < 2} {incr i} {
	    $ns_ trace-queue $q_($qInd) $dst_($ind) $drop_file_
	    $ns_ trace-queue $dst_($ind) $q_($qInd) $drop_file_
	    $ns_ trace-queue $q_($nextQInd) $src_($ind) $drop_file_
	    $ns_ trace-queue $src_($ind) $q_($nextQInd) $drop_file_
	    incr ind
	}
	set qInd [expr $qInd + 2]
    }
    for {set ind 0} {$ind < 4} {incr ind} {
	    $ns_ trace-queue $q_(0) $src_($ind) $drop_file_
	    $ns_ trace-queue $src_($ind) $q_(0) $drop_file_
	    $ns_ trace-queue $q_(7) $dst_($ind) $drop_file_
	    $ns_ trace-queue $dst_($ind) $q_(7) $drop_file_
    }
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Schedule Packet Tracing
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc schedule_tracing {} {
    global opts 
    $self instvar ns_ end_ tcps_ q_ file_ src_ dst_ tchan_

    # queue size monitors
    set end [expr $opts(queues) - 1]
    for {set i 0} {$i < $end} {incr i} {
	set next [expr $i + 1]
	if {[llength $opts(trace)] == 1} {
	    set trace [lindex $opts(trace) 0]
	} else {
	    set trace [lindex $opts(trace) $next]
	}
	if {$trace == 1} {
	    set qfile($i,$next) [open "|awk {{print \$3,\$9,\$11}} | gzip > $file_-$i$next-qsz.tr.gz" w]
	    set qmon($i,$next) [$ns_ monitor-queue $q_($i) $q_($next) ""]
	    $ns_ at 0 "$qmon($i,$next) trace $qfile($i,$next)"
	}
    }

    $ns_ at $opts(warmup) "$self trace"
}

#:::::::::::::::::::::
#
# Finish and Clean up
#
#::::::::::::::::::::
MySim instproc finish {} {
    global opts
    $self instvar ns_ nsrc_ src_file_ dst_file_ ack_file_ drop_file_ tchan_

    $ns_ flush-trace  

    if {[lindex $opts(trace) 0] == 1} {
	close $src_file_
	close $ack_file_ 
    }
    set lastq [expr $opts(queues) - 1]
    if {([llength $opts(trace)] == 1 && [lindex $opts(trace) 0] == 1) 
    || ([llength $opts(trace)] > 1 && [lindex $opts(trace) $lastq] == 1)} {
	close $dst_file_
    }

    # code from http://mini.net/tcl/1036
    if {[catch {close $drop_file_} standard_error] != 0} {
	global errorCode
	if {"CHILDSTATUS" == [lindex $errorCode 0]} {
	    set exit_status [lindex $errorCode 2]
	}
    }

    if {$opts(tcpvars)} {
	set flows [lindex $opts(flows) 0]
	for {set fid 0} {$fid < $flows} {incr fid} {
	    close $tchan_($fid)
	}
    }

    # post-processing
    $self process

    $ns_ halt

    exit 0
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Post-Process
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc process {} {
    global opts bindir
    $self instvar file_

    set time [exec date "+%H:%M:%S (%m/%d/%y)"]
    exec echo "   END: $time" >> $opts(datadir)/EXPS

    if {$opts(nam)} {
	exec nam $opts(datadir)/out.nam &
    }
}

#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Count Completed HTTP Request/Response Pairs
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc gather_pairs {} {
    global opts 
    $self instvar PM_ q_ tcps_ tcpd_ src_ dst_

    set pairs 0
    for {set i 2} {$i < 4} {incr i} {
	set pair($i) [$PM_($i) get-pairs]
	set pairs [expr $pairs + $pair($i)]
    }

    if {$pairs >= $opts(pairs)} {
	# stop forward path e2e flows
	set rate [lindex $opts(PMrate) 0]
	if {$rate > 0} {
	    if {$opts(out)} {
		puts "\nSTOPPING FORWARD PATH E2E HTTP:"
	    }
	    set ind 2
	    while {$ind < 4} {
		$PM_($ind) stop
		if {$opts(out)} {
		    puts "  PM ($rate c/s): src$ind ([$src_($ind) id]) --> dst$ind ([$dst_($ind) id])"
		}
		incr ind
	    }
	}
    
	# stop fwd path FTP flows
	set flows [lindex $opts(flows) 0]
	if {$flows > 0} {
	    if {$opts(out)} {
		puts "STOPPING FORWARD PATH E2E FTP:"
	    }
	    for {set fid 0} {$fid < $flows} {incr fid} {
		$tcps_($fid) close
		if {$opts(out)} {
		    puts "  fid $fid"
		}

	    }
	}

	# stop forward path cross-traffic
	if {$opts(out)} {
	    puts "STOPPING FORWARD PATH CT HTTP:"
	}
	set qInd 1
	set ind 4
	while {$qInd <= 5} {
	    if {[llength $opts(PMrate)] == 1} {
		set rate [lindex $opts(PMrate) 0]
	    } else {
		set rate [lindex $opts(PMrate) $qInd]
	    }
	    
	    if {$rate > 0} {
		for {set i 0} {$i < 2} {incr i} {
		    $PM_($ind) stop
		    # output 
		    if {$opts(out)} {
			puts "  PM ($rate c/s): (Q $qInd) src$ind ([$src_($ind) id]) --> dst$ind ([$dst_($ind) id])"
		    }
		    incr ind
		}
	    }
	    set qInd [expr $qInd + 2]
	    set ind [expr $ind + 2]
	}

	# stop reverse path cross-traffic
	if {$opts(out)} {
	    puts "\nSTOPPING REVERSE PATH CT HTTP:"
	}

	set qInd 2
	set ind 6
	while {$qInd <= 6} {
	    if {[llength $opts(rev-PMrate)] == 1} {
		set rate [lindex $opts(rev-PMrate) 0]
	    } else {
		set rate [lindex $opts(rev-PMrate) $qInd]
	    }
	    
	    if {$rate > 0} {
		for {set i 0} {$i < 2} {incr i} {
		    $PM_($ind) stop
		    # output 
		    if {$opts(out)} {
			puts "  PM ($rate c/s): (Q $qInd) src$ind ([$src_($ind) id]) --> dst$ind ([$dst_($ind) id])"
		    }
		    incr ind
		}
	    }
	    set qInd [expr $qInd + 2]
	    set ind [expr $ind + 2]
	}

	$self finish
    }
}


#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Initialize Simulation
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

MySim instproc init {} {
    global opts NSFILE bindir
    $self instvar ns_ file_ q_ src_ dst_ nsrc_ ndst_ PM_ end_

    # copy files over to datadir
    exec cp $NSFILE.tcl $opts(datadir)/$NSFILE.tcl
    
    # set data file name
    set flows [lindex $opts(flows) 0]
    set rate [lindex $opts(PMrate) 0]
    if {$flows > 0 && $rate == 0} {
	set file_ $opts(datadir)/${flows}FTP-$opts(ID)-$opts(duration)s
    } elseif {$flows == 0 && $rate > 0} {
	set file_ $opts(datadir)/${rate}cs-$opts(ID)-$opts(duration)s
    } else {
	set file_ $opts(datadir)/${rate}cs-${flows}FTP-$opts(ID)-$opts(duration)s
    }

    # setup simulator
    remove-all-packet-headers;            # removes all packet headers
    add-packet-header IP TCP;             # adds TCP/IP headers
    set ns_ [new Simulator];              # instantiate the simulator
    $ns_ use-scheduler $opts(scheduler)

    # nam tracing
    if {$opts(nam)} {
	$ns_ namtrace-all [open "$opts(datadir)/out.nam" w]
	# HARD-CODED
	for {set i 0} {$i < 4} {incr i} {
	    $ns_ color $i red
	    set revfid [expr $i + 2000]
	    $ns_ color $revfid green
	}
    }

    # setup network
    $self init_schedule
    $self init_network

    # make the topology pretty
    if {$opts(nam)} {
	set last [expr $opts(queues) - 1]
	for {set i 0} {$i < $last} {incr i} {
	    set next [expr $i+1]
	    $ns_ duplex-link-op $q_($i) $q_($next) orient right
	    $q_($i) shape square
	    $q_($next) shape square
	}
	# HARD-CODED
	for {set ind 0} {$ind<20} {incr ind} {
	    $src_($ind) shape hexagon
	    $src_($ind) color blue
	    $src_($ind) label "src($ind)"
	    $dst_($ind) color green
	    $dst_($ind) label "dst($ind)"
	}
    }

#    puts stderr "calling init_connections"
    $self init_connections
    $self schedule_tracing

    # setup time and memory usage stats
    set time_mem $file_-TIME-MEM
    exec touch $time_mem;     exec rm $time_mem;     exec touch $time_mem
    exec printf "#sim    real    mem\n" >> $time_mem
    for {set j 0} {$j <= $end_} {incr j} {
	$ns_ at $j "exec printf \"$j\" >> $time_mem; \
		exec $bindir/format-time-mem.pl >> $time_mem"
   }

    # stop after $opts(pairs) req/rsp pairs
    if {$opts(pairs) > 0} {
	if {[llength $opts(PMrate)] > 1 || [lindex $opts(PMrate) 0] > 0} {
	    for {set i $opts(warmup)} {$i <= $end_} {incr i} {
		$ns_ at $i "$self gather_pairs"
	    }
	}
    }

    $ns_ run
}

main