#!/usr/bin/perl
#
# file: $isip/util/demo/isip_comm_server_dialog_database/web_map.perl
#
#------------------------------------------------------------------------------
#
#  This perl script is used to extract distance and directions
#  information from the Mapquest.com web site.  The script input
#  arguments are as follows:
#
#
# ARGV[0] = number [1-4]
# 1 - equiv to web_location.perl
# 2 - equiv to web_location_1.perl
# 3 - equiv to web_address.perl
# 4 - equiv to web_map.perl

#==========================================
# web_location.perl [1]
#==========================================
#- inputs
#  business_type   = 1 ;;name or type or anything regarding what's being found
#  street_name     = 2 ;;start\search area
#  city            = 3
#  state           = 4
#  zip             = 5
#  number_for_out  = 6 ;;number for the outfile
#  path for out    = 7 ;; '/tmp/perl' or whatever
#
#- outputs
#  print DATA "name           $hotel     \n";
#  print DATA "address        $local_address     \n";
#  print DATA "city           $hotel     \n";
#  print DATA "telephone      $hotel     \n";
#  print DATA "startaddress   $street_name     \n";
#
#  The last "startaddress" is the data passed in on the command line.
#
#  This query looks for a certain place near an address and then prints
#  out all results. Uses the query website of http://yp.mapquest.com/main.adp.

#==========================================
# web_location_1.perl [2]
#==========================================
#- inputs
#  business_type   = 1 ;;name or type or anything regarding what's being found
#  start_add       = 2 ;;start\search area -- stands for 'start address'
#  city            = 3
#  state           = 4
#  zip             = 5
#  number_for_out  = 6 ;;number for the outfile
#  path for out    = 7 ;; '/tmp/perl' or whatever
#
#- outputs
#  print DATA "name           $placename     \n";
#  print DATA "address        $local_address     \n";
#  print DATA "city           $cityname     \n";
#  print DATA "state          $statename     \n";
#  print DATA "telephone      $placename     \n";
#  print DATA "miles          $mile     \n";
#
#
#  This query looks for a place near an address and then prints out all the 
#  results. Uses the query website of http://yp.mapquest.com/main.adp.
#
#  This has two different queries based on whether "start_add" is present
#  or not.

#==========================================
# web_address.perl [3]
#==========================================
#
#- inputs
#  business_name   = 1 ;; name search
#  business_spe    = 2 ;; business specification
#  business_type   = 3 ;; type
#  start_city      = 4 ;; 
#  start_state     = 5 ;;
#  start_zip       = 6 ;;
#  ref_name        = 7 ;; used only in output - as "refer_name"
#  ref_add         = 8 ;; used in the "near this address" part of the search
#  number_for_out  = 9 ;;number for the outfile
#  path for out    = 10 ;; '/tmp/perl' or whatever
#
#- outputs
#  print DATA "name           $location             \n";
#  print DATA "specification  $business_spe      \n";
#  print DATA "type           $business_type     \n";
#  print DATA "address        $local_address     \n";
#  print DATA "city           $cityname     \n";
#  print DATA "state          $statename     \n";
#  print DATA "telephone      $location     \n";
#  print DATA "miles          $mile     \n";
#  print DATA "refer_name     $ref_name   \n";
#  print DATA "refer_add      $ref_add    \n";
#
#  This query looks for a certain business type or name NEAR a given address.
#  The three inputs business_name\spe\type are all concat'd together so ONLY
#  PUT ONE of them.
#
#  If "ref_add" does not contain a digit as the first character, it puts
#  "100" on the front so that it is a true address
#  instead of just a street name
#
#==========================================
# web_map.perl [4]
#==========================================
#
#-inputs
#  start_name      = 1 ;;
#  start_add       = 2 ;;
#  start_interest  = 3 ;;
#  start_city      = 4 ;;
#  start_state     = 5 ;;
#  start_zip       = 6 ;;
#  end_name        = 7 ;;
#  end_add         = 8 ;;
#  end_interest    = 9 ;;
#  end_city        = 10 ;;
#  end_state       = 11 ;;
#  end_zip         = 12 ;;
#  number_for_out  = 13 ;;
#  path for out    = 14 ;; '/tmp/perl' or whatever
# #note: all inputs are self-explanatory
#
#-output
# print DATA "start_name          $start_name\n";
# print DATA "start_add           $start_add\n";
#
# print DATA "start_interest      $start_interest\n";
# print DATA "start_city          $start_city\n";
# print DATA "start_state         $start_state\n";
# print DATA "start_zip           $start_zip\n";
#      
# print DATA "end_name            $end_name\n";
# print DATA "end_add             $end_add\n";
# print DATA "end_interest        $end_interest\n";
# print DATA "end_city            $end_city\n";
# print DATA "end_state           $end_state\n";
# print DATA "end_zip             $end_zip\n";
#      
# print DATA "total_distance      $distance\n";
# print DATA "total_time          $time\n";
# print DATA "directions          $count\n";
# for ($count)
#   print DATA "$direction"
#   print DATA "$mile"
#
# This query attempts to find out the directions from one place to another.
# Only one set of directions is given per output file.
# The output file contains the standard 15 lines, then in pairs of lines,
# contains ($count) number of pairs.

#==========================================
# Sample executions:
#==========================================
#
# 1) web_location -- THESE DO NOT WORK BUT HERE IS A SAMPLE LAYOUT
#
# ./web_access.perl 1 "Walmart" "" "Starkville" "MS" "39759" 0 "/tmp/"
#
# 2) web_location_1 - find a place
#
# ./web_access.perl 2 "Walmart" "" "Starkville" "MS" "39759" 0 "/tmp/"
# ./web_access.perl 2 "Foodmax" "" "Starkville" "MS" "39759" 0 "/tmp/"
# ./web_access.perl 2 "Stromboli" "" "Starkville" "MS" "39759" 0 "/tmp/"
#
# 3) web_address.perl - find something near an address
#
# ./web_access.perl 3 "" "gas station" "" "Starkville" "MS" "39759" "" "118 Highway 12 W" 9 "/tmp/"
# ./web_access.perl 3 "" "movie rental" "" "Starkville" "MS" "39759" "" "118 Highway 12 W" 8 "/tmp/"
# ./web_access.perl 3 "" "food" "" "Starkville" "MS" "39759" "" "118 Hwy 82 W" "0" "/tmp/"
#
# 4) web_map.perl - directions
#
# foodmax - 118 Highway 12 W
# stromboli's - 408 University Dr
#
#./web_access.perl 4 "" "118 Highway 12 W" "" "Starkville" "MS" "39759" "" "408 University Dr" "" "Starkville" "MS" "39759" 1 "/tmp/"
#
#

############################
# variable names
# common to all perl scripts
############################

# value for end of flatmap_%i eg flatmap_0, flatmap_1
my $myval;

# path name for the temp\out directory
my $OUT_PATH;
my $TEMP_PATH;
my $COOKIE_FILENAME;

my $retry;
my $RETRY_TIME_OUT;
my $NUM_RETRIES;

# names for each thing in the searching, may or may not be used
# depending on which web query you are doing
my $business_name;
my $business_spe;  
my $business_type; 
my $state;
my $street_name;
my $city;
my $zip;

my $ref_add;
my $ref_name;

my $start_add;
my $start_city;
my $start_interest;
my $start_name;
my $start_state;
my $start_zip;

my $end_add;
my $end_city;
my $end_interest;
my $end_name;
my $end_state;
my $end_zip;

use LWP::UserAgent;
use Benchmark;
use URI::URL;
use HTTP::Cookies;

# function: status
#
# arguments:
#  string : input to print to screen
#
# return: void
#
# This function will print off the single string passed in
# to stderr
#
sub status {
    my $arg = $_[0];
    print STDERR " [$arg]\n";
}

# function: trim
#
# arguments:
#  string : input to remove whitespace
#
# return: a string without beginning and trailing whitespace
#
# This function will remove whitespace from before and after a string
# and return the altered string
#
sub trim {
    my($string) = @_;
    
    for ($string) {
	s/^\s+//;
	s/\s+$//;
	s/\n//;
    }
    
    return $string;
}

# function: check_num_params
#
# arguments:
#  int : the value of $#ARGV which is array size - 1
#  int : the actual number of params there should be
#
# return: 0 if array size + 1 != actual to find
#         1 if array size + 1 == actual to find
#
# This function returns yes\no depending on if there are the correct
# number of parameters
#
sub check_num_params {
    my ($my_argc, $num_to_find) = @_;
    
    # $#ARGV is total-1 so add one to it
    $my_argc++;
    if ($my_argc == $num_to_find) {
	return 1;
    }    
    return 0;
}

# function: print_usage1
#
# arguments:
#  none
#
# return: none
#
# This function prints out how to choose which perl to use and then dies
#
sub print_usage1 {
    print "******************\n";
    print "  ERROR: web_access.perl requires the first argument\n";
    print "  to be [1-4] to refer to:\n";
    print "    1) web_location.perl -- NOT WORKING\n";
    print "    2) web_location_1.perl\n";
    print "    3) web_address.perl\n";
    print "    4) web_map.perl\n";
    print "******************\n";
    die "Improper usage error!";
}	

# function: print_usage2
#
# arguments:
#  none
#
# return: none
#
# This function prints out a vague error message and then dies
#
sub print_usage2 {
    print "******************\n";
    print "  ERROR: wrong number of arguments into the query type\n";
    print "    open the perl script and look at the top of file\n";
    print "    for additional help with each option\n";
    print "******************\n";
    die "Wrong number of arguments error";
}	

# function: fix_path
#
# arguments:
#  string - a path
#
# return: string - the path as the same or with the trailing '/' missing
#         eg "/tmp/path"  => "/tmp/path" (NO CHANGE)
#         eg "/tmp/path/" => "/tmp/path" (REMOVED FINAL /)
#
# This function prints out a vague error message and then dies
#
sub fix_path {
    my $mypath = shift;
    
    $mypath =~ /(^.*)[\/]$/;
    if ($1 ne "") {
	#print "fixed it!\n";
	$mypath = $1;
    }
   
    return $mypath;
}
	
################################################
# for all you C++ programmers (me), begin main()
################################################

#find out how many parameters there are
print "Number of args found is '". ($#ARGV + 1) ."'\n";

# check to make sure argv is there
if (!defined $ARGV[0] || $ARGV[0] < 1 || $ARGV[0] > 4) {
    &print_usage1();
}

# populate and fix the $TEMP_PATH variable
$TEMP_PATH = $ENV{"ISIP_TMPDIR"};
if ($TEMP_PATH eq "") {
    print "Unable to locate environment $ISIP_TMPDIR. Using '/tmp'\n";
    $TEMP_PATH = "/tmp";
}
$TEMP_PATH = &fix_path($TEMP_PATH);

#populate the cookies filename
$COOKIE_FILENAME = "$TEMP_PATH/isip_db_perl_cookies.txt";
#print "the cookie file is [$COOKIE_FILENAME]\n";

while ($retry>-1) {
    # run the respective script
    # 1 web_location.perl 
    if ($ARGV[0] == 1) {
	if (&check_num_params($#ARGV, 8) == 0) {
	    &print_usage2();    
	}
	$myval = $ARGV[6];
	$OUT_PATH = &fix_path($ARGV[7]);
	print "====================================================\n";
	print "ERROR: web_location.perl script does not work.\n";
	print "use argument #2 (web_location_1.perl) for same purpose\n";
	print "====================================================\n";
	die "exiting due to web_location.perl(1) not working";
#	$num_returned = goto_web_location(
#					  $ARGV[1], #$business_type,
#					  $ARGV[2], #$street_name,
#					  $ARGV[3], #$city,
#					  $ARGV[4], #$state,
#					  $ARGV[5], #$zip,
#					  $ARGV[6]  #$myval
#					  );
    }
    
    # 2 web_location_1.perl
    if ($ARGV[0] == 2) {
	if (&check_num_params($#ARGV, 8) == 0) {
	    &print_usage2();    
	}
	$myval = $ARGV[6];
	$OUT_PATH = &fix_path($ARGV[7]);
	$num_returned = goto_web_location_1(
					    $ARGV[1], #$business_type,
					    $ARGV[2], #$start_add
					    $ARGV[3], #$city,
					    $ARGV[4], #$state,
					    $ARGV[5], #$zip,
					    $ARGV[6]  #$myval,
					    );
    }
    
    # 3 web_address.perl
    if ($ARGV[0] == 3) {
	if (&check_num_params($#ARGV, 11) == 0) {
	    &print_usage2();    
	}
	$myval = $ARGV[9];
	$OUT_PATH = &fix_path($ARGV[10]);
	$num_returned = goto_web_address(
					 $ARGV[1], #$business_name,
					 $ARGV[2], #$business_spe,
					 $ARGV[3], #$business_type,
					 $ARGV[4], #$start_city,
					 $ARGV[5], #$start_state,
					 $ARGV[6], #$start_zip,
					 $ARGV[7], #$ref_name,
					 $ARGV[8], #$ref_add,
					 $ARGV[9]  #$myval
					 );
    }
    
    # 4 web_map.perl
    if ($ARGV[0] == 4) {
	if (&check_num_params($#ARGV, 15) == 0) {
	    &print_usage2();    
	}
	$myval = $ARGV[13];
	$OUT_PATH = &fix_path($ARGV[14]);
	$num_returned = goto_web_map(
				     $ARGV[1], #start_name,
				     $ARGV[2], #$start_add,
				     $ARGV[3], #$start_interest,
				     $ARGV[4], #$start_city,
				     $ARGV[5], #$start_state,
				     $ARGV[6], #$start_zip,
				     $ARGV[7], #$end_name,
				     $ARGV[8], #$end_add,
				     $ARGV[9], #$end_interest,
				     $ARGV[10], #$end_city,
				     $ARGV[11], #$end_state,
				     $ARGV[12], #$end_zip,
				     $ARGV[13]  #$myval);
				     );
    }
    
    if ($num_returned < 0) {
	if ($retry) {
	    &status("System Busy.... ($retry)");
	    $retry--;
	    sleep($RETRY_TIME_OUT);
	} 
	else {
	    &status("skipping...");
	    $retry--;
	}
    } 
    else {
	$retry = -1;
	$outfile = "$OUT_PATH/flatmap_$myval.done";
	open(DATA, ">>$outfile") || die "ERROR: can't open $outfile.\n";
	print DATA "done\n";
	close DATA;
    }
}
################################################
# for all you C++ programmers (me), end main()
################################################

# pass in the local_file (eg, html) and it will return a non empty
# html page, or else the same page
#
# this while loop says
# if the phrase "Most popular matching category(ies) occur, find the first
# link after that phrase on the page and then download that file. This
# *should* be called only once, but it doesn't hurt to have it inside
# a while loop in case for some reason we go from
# 'gas' -> 'gas stations' -> 'convenient stores' or something like that
sub find_popular_matching {
    my ($local_file, $req, $ua) = @_;

    my ($return_string) = $local_file;

    while ($local_file =~ /Most Popular Matching Categ/) {
	print "=====================\n";		
	print "Found Most Popular Matching Categories\n";
	print "=====================\n";		
	#print "Business type we are looking at is [$business_type]\n";
	#print "=====================\n";		
        #  $distance =~ /Next Page(.*)Listings On a Map/gs;
	
	$local_file =~ /Most Popular Matching Cate(.*)/gs;
	$local_file = $1;
	#print "local_file is now [$local_file]\n";
	
	@second = $local_file =~ /href=\"map.adp[^<]*/g;
	# @second now contains all of the href til the next < character
	# $second[0] is the first one so that's the one we want to follow
	#
	#print "second[0] is [$second[0]]*******\n";
	$links = $second[0];
	#print "links is *****[$links]*****\n";

	# now to clean up and remove the href part
	#
	$links =~ /href=\"map.adp\?([^\"]+)/g;
	$links = $1;
	print "Inside Most Popular: new link is [$links]\n";

	# send request again using gained 'links'
	#
	$req->content($links);
	$res = $ua->request($req);
	
	#debug
	#print "printing result content would go here\n";
	#print "$res->content"; 
	
	$local_file = $res->content;
	
	# DEBUG
	$outfile = "$TEMP_PATH/isip_db_perl_actual_html_popular_matching.html";
	open(DEBUGDATA,">$outfile") || die "CANNOT OPEN $outfile";
	print DEBUGDATA $local_file;
	close DEBUGDATA;
	# END DEBUG
    }

    return $local_file;
}

sub find_next_page {
    my ($local_file, $req, $ua) = @_;
    
    my ($return_string) = "";

    if ($local_file =~ /Next<\/a>/) {
	print "Found Next_Page\n";
	
	# rather than Previous\Next do
	# <a href=.*Next</a>
	
	$local_file =~ /\<a href=\"([^\"]*)\">Next<\/a>/g;
	$links = $1;
	print "Inside Next Page: new link is [$links]\n";
	
	#print "found links as [$links]\n";
	# request the new page that has just been parsed out
	# if the link is not null
	#
	if ($links eq "") {
	    print "could not find next page, exiting\n";
	    last;
	}
	
	$req->content($links);
	$res = $ua->request($req);
	
	$return_string = $res->content;   
	
	# DEBUG
	$outfile = "$TEMP_PATH/isip_db_perl_actual_html_next.html";
	open(DEBUGDATA,">$outfile") || die "CANNOT OPEN $outfile";
	print DEBUGDATA $local_file;
	close DEBUGDATA;
	# END DEBUG
    }

    return $return_string;
}

sub find_more_specific_address {

    my ($local_file, $req, $ua) = @_;

    # if I find the phrase
    # "Your Search for XXX matched 2 locations. Please choose one."
    # then follow the first link on the page
    #
    if ($local_file =~
	/Your Search for(.*)matched(.*)locations. Please choose one./g) {

	print "Found a more specific address\n";

	$local_file =~ /\<li\>\<a href=([^>]*)>/g;
#/maps/map.adp?searchtype=search&country=US&cat=MCDONALD%27S&formtype=searchadv&addtohistory=&location=mOSStTBFOaw3Fu4rTSFgq23NtEgmZl5UO497IhDa63iqLnws2cZvmAsxBj8y0MThnk0YMSmjhf%2fBLQ9znlnnXdEFjBuRvuaB%2f6dKaheHAUmkIm4KNf0b7zwZusHHyrhoNOs0OKFsHXn0zeca4G8PdpjaZ6%2fI9vPZWcQMO75iC4U%3d">100
	my $entire_link = $1;
	#print "inside 'find_more_specific_address' entire_link".
	#"is [$entire_link]\n";
	$entire_link =~ /([^?]*)\?([^"]*)\"$/g; #"
	#print "parsed is [$1] [$2]\n";

	my $link = $2;		  
	if ($link eq "" || $entire_link eq "") {
	    print "Error finding link in 'find_more_specific_address'\n";
	    die "Error parsing page in 'find_more_specific_address'.";
	}
        #print"inside 'find_more_specific_address'".
				    #"going to request [$link]\n";

	$req->content($link);
	$res = $ua->request($req);
	
	$local_file = $res->content;   

	# DEBUG
	$outfile = "$TEMP_PATH/isip_db_perl_actual_html_matched_location.html";
	open(DEBUGDATA,">$outfile") || die "CANNOT OPEN $outfile";
	print DEBUGDATA $local_file;
	close DEBUGDATA;
	# END DEBUG

    }

    return $local_file;
}

    
    


############################
# web_address.perl
############################

sub goto_web_address {
    my ($business_name, $business_spe,
	$business_type, $city, $state,
	$zip, $ref_name, $ref_add, $myval) = @_;
    
    defined $zip ?
	print " zip is [$zip]\n"
	: die "zip is not defined!\n";
    
    defined $business_name ?
	print " business_name is [$business_name]\n"
	: die "business_name is not defined!\n";
    
    defined $business_spe ?
	print " business_spe is [$business_spe]\n"
	: die "business_spe is not defined!\n";
    
    defined $city ?
	print " city is [$city]\n"
	: die "city is not defined!\n";
    
    defined $state ?
	print " state is [$state]\n"
	: die "state is not defined!\n";
    
    defined $myval ?
	print " myval is [$myval]\n"
	: die "myval is not defined!\n";
    
    my $RETRY_TIME_OUT = 5;
    
    my $cookie_jar = HTTP::Cookies::Netscape->new(File => $COOKIE_FILENAME,
						  AutoSave => 1);
    
    my $ua = new LWP::UserAgent;
    $ua->cookie_jar($cookie_jar);
    $ua->agent("Mozilla/4.08 [en] (X11; U; Linux 2.0.32 i686)"); # . $ua->agent);
    
    #PRE-SB my $req = new HTTP::Request POST => 'http://yp.mapquest.com/main.adp';
    my $req = new HTTP::Request POST => 'http://www.mapquest.com/maps/map.adp';
    $req->content_type('application/x-www-form-urlencoded');  
    
    &status("Submiting address query");
    
    # The thinking behind putting the "100" on the ref_add is that if it doesnt
    # start with a \d(digit) then it needs a location so put the address as 100
    #
    if($ref_add eq "") {
	print " empty reference address: $ref_add\n";
	print " ================================\n";
    }
    else {
	if ($ref_add =~ /^\d.*/g) {
	    #no-op
	    #print STDERR "$ref_add\n";
	    #print STDERR	"***************************\n";
	}
	else {
	    $ref_add = "100 ".$ref_add;  
	    #print STDERR "$ref_add\n";
	    #print STDERR "++++++++++++++++++++++++++++\n";
	}
	print " ref_add is [$ref_add]\n";
    }
    
    #NEWNEW
    #http://www.mapquest.com/maps/map.adp?
    #formtype=searchadv&searchtype=search&country=US&addtohistory=&
    #cat=gas+station&address=118+Highway+12+W&city=Starkville&state=MS&
    #zipcode=39759
    
    # build web string to query the web site according to the
    # requirement of the web site
    #
    $sendString = "formtype=searchadv&searchtype=search&country=US&addtohistory=&cat=".$business_name." ".$business_spe." ".$business_type."&address=".$ref_add."&city=".$city."&state=".$state."&zipcode=".$zip;
    
   print "sendString is [$sendString]\n";
    
    # Make the request
    #
    $req->content($sendString);
    my $res = $ua->request($req);
    
    if ($res->is_success) {
	#do nothing
    } 
    else {
	print $res->error_as_HTML;
    }

    # get the resulting html page from the result
    #
    my ($local_file) = $res->content;
    
    #print "This is where the page would be printed [local_file]\n";
    
    # DEBUG
    $outfile = "$TEMP_PATH/isip_db_perl_actual_html.html";
    open(DEBUGDATA,">$outfile") || die "CANNOT OPEN $outfile";
    print DEBUGDATA $local_file;
    close DEBUGDATA;
    # END DEBUG
    
    my @second;
    my $links;
    
    # if the returned page include "Most Popular Matching Categories", go
    # to the first meaningful links in this page #

    $local_file = &find_more_specific_address($local_file, $req, $ua);

    #look for most popular
    $local_file = &find_popular_matching($local_file, $req, $ua);

    # DEBUG
    $outfile = "$TEMP_PATH/isip_db_perl_actual_html_returned.html";
    open(DEBUGDATA,">$outfile") || die "CANNOT OPEN $outfile";
    print DEBUGDATA $local_file;
    close DEBUGDATA;
    # END DEBUG
    print "Returned local_file\n";
    
    # the page downloaded right now is in $res->content as basically a string
    # this function returns the number of things found on the page as valid
    # addresses
    
    print "entering my web_address_page_new\n";
    $count = 0;
    # if the returned page include "Next Page", go to the first 
    # meaningful links in this page
    #
    while (1) {
	$count += &parse_web_address_page_new($local_file, $business_name,
					     $business_spe, $business_type,
					     $ref_name, $ref_add, $myval);
	$local_file = &find_next_page($local_file, $req, $ua);
	if ($local_file eq "") {
	    last;
	}
    }

    return $count;
}

# ----------------------------------------------------------------------------
#
#  Routine which parses the returned web page from mapquest.com for
#  location information The output is written to a file called
#  "flatmap_*". 
# ----------------------------------------------------------------------------
sub parse_web_address_page_new {
    
    my ($file, $business_name,  $business_spe, 
	$business_type, $ref_name, $ref_add, $myval) = @_;
    
    # Declare some variables
    my $num = 0;
    my $mile;
    
    # save only the part of the html that has the table that
    # contains the information about the locations
    # it just happens to be that directly before there is a javascript func
    # named 'selectSortBy' and afterwards there is a 'openGoogle'. If these
    # change on the Mapquest page then they need to be changed here
    #
    my $html = $file;
    $html =~ /function selectSortBy(.*)function openGoogle()/gs;
    $html = $1;
    
    #print STDERR "html value is [$html]\n";
    
    #DEBUG--
    # This is sample code for matching in this html. 
    #my $direction = $html;
    #@miles = $direction =~ /([0-9\.]*) miles away/g;
    #print "miles is ".@miles."\n";
    #my $num = 0;
    #foreach my $unused_var (@miles) {
    #my $mile = $miles[$num];
    #$num++;
    #print STDERR "num : $num: [$unused_var] $mile\n";
    #}
    #print STDERR "total num : $num\n";
    #END DEBUG---------
    
    # if output file is open, fileopen is 1. it will be opened
    # the first time there is output to the file
    my $fileopen = 0;
    my $outfile1 = 0;
    
    my $found = 0;
    my $linenum = 0;
    @lines = split(/\n+/, $html);
    foreach my $line (@lines) {
	
	@entries = $line =~ /<td class=boldText><b>([^<]*)<\/b><\/td><\/tr><tr><td><\/td><td>&nbsp;<\/td><td class=size12>([^&]*)&nbsp;\(([0-9.]*) miles away\)<tr><td><\/td><td>\&nbsp;<\/td><td class=size12>([^<]*)<\/td>/g;
	
	#there are 4 entries per line.
	#example as such:
	#0: Highway 82 Texaco & Tobacco
	#1: 101 Martin Luther King Jr Dr W, Starkville, MS
	#2: 1.30           <<- hint: This is "miles away"
	#3: 662-323-1222
	
	if (@entries == 4) {
	    
	    # open the output file if it hasn't been opened yet
	    if ($fileopen == 0) {
		$outfile1="$OUT_PATH/flatmap_$myval";
		open(DATA, ">>$outfile1") || 
		    die "ERROR: can't open $outfile.\n";
		$fileopen = 1;
	    }
	    
	    #split out the address, city, state
	    my ($name, $address, $city, $state, $miles, $phone) =
		(@entries[0], "", "", "", @entries[2], @entries[3]);
	    
	    #populate the address, city, state from the parsing of the
	    #entries[1] splitting
	    #
	    #print "looking at entry 1 which is[".@entries[1]."]\n";
	    if (@entries[1] =~ /([^,]*),([^,]*),(.*)/g) {
		$address = trim($1);
		$city = trim($2);
		$state = trim($3);
	    }
	    
	    #print "on line $linenum there are some entries!\n";
	    print "Found as [$name] [$address] [$city] ".
		"[$state] [$miles] [$phone]\n";
	    
	    #DEBUG output everything in @entriesdebug output
	    #my $entnum = 0;
	    #foreach (@entries) {
		#print "$entnum: $_\n";
		#$entnum++;
	    #}
	    #END DEBUG -----
	    print DATA "name           $name             \n";
	    print DATA "specification  $business_spe      \n";
	    print DATA "type           $business_type     \n";
	    print DATA "address        $address     \n";
	    print DATA "city           $city     \n";
	    print DATA "state          $state     \n";
	    print DATA "telephone      $phone     \n";
	    print DATA "miles          $miles     \n";
	    print DATA "refer_name     $ref_name   \n";
	    print DATA "refer_add      $ref_add    \n";
	    $found++;
	}
	$linenum++;
    }
    
    # if the output file was opened, close it
    if ($fileopen == 1) {
	close $outfile1;
    }

    return $found;
}

#sub parse_web_address_page {
##    
##    my ($file, $business_name,  $business_spe, 
##	$business_type, $ref_name, $ref_add, $myval) = @_;
##    
##    my $num = 0;
##    my $mile;
##    
##    # remove the html tag
##    #
##    foreach $rt ($file) {
##	$rt=~ s/<(?:[^>'"]*|(['"]).*?\1)*>/\n/gs; # '"])>
##	#so that emacs will color the rest of the file corrctly
##    }
##    my $html = $file;
##    
##    # pick up the listing stuff
##    #
##    # /gs options are g=global + s=treat as single line
##    $html =~ /Next Page(.*)Listings On a Map/gs;
##    
##    # The line below effectively does nothing because from ^ to $ is not null.
##    # and even if it is, it will be erased in the line below.
##    # mwe-this is the way I read it and experimentation shows.
##    # see "test.perl" in elliott directory.
##    #
##    #$distance =~ s/^(\s)*$//g;
##    
##    $html = $1;
##    
##    @directions = $html =~ /\w.*/g;
##    
##    my $count = 0;
##    my $count_num = 0;
##    
##    $outfile = "$OUT_PATH/flatmap_$myval";
##    open(DATA, ">$outfile") || die "ERROR: can't open $outfile.\n";
##    
##    my $local_count = 9;	
##    
##    # parse the content and
##    # output to a file using a specified format
##    #
##    foreach $r (@directions) {
##	
##	my $location = $directions[$count];
##	
##	$count++;
##	
##	if ($location =~ /miles away/) {
##	    $mile = $location;
##	    
##	    $mile =~ /([0-9\.]*) miles away/g;
##	    $mile = $1; 
##	    
##	    $local_count = 7;
##	}
##	
##	if ($local_count < 9) {
##	    if (($location =~ /More.../) ||
##		($location =~ /nbsp/) ||
##		($location =~ /Maps/) ||
##		($location =~ /Show These/) ||
##		($location =~ /Web Site/) ||
##		($location =~ /Movie Showtimes/) ||
##		($location =~ /Driving Directions/)) {
##		$local_count = 7;
##	    }
##	    else {
##		if ($local_count == 8) {
##		    $local_count = 1;
##		}
##	    }
##	    
##	    if ($local_count < 7) { 
##		if ($local_count == 1) {
##		    $count_num++;
##		    if(length($business_name) == 0) {
##			print DATA "name           $location             \n";
##		    }
##		    else {
##			print DATA "name           $business_name     \n";
##		    }
##		    print DATA "specification  $business_spe      \n";
##		    print DATA "type           $business_type     \n";
##		}
##		
##		if ($local_count == 2) {
##		    my $local_address = $location;
##		    if ($local_address =~ /,/) {
##			
##			$local_address =~ /([^\,]*)/g;
##			$local_address = $1;
##			print DATA "address        $local_address     \n";
##		    }
##		    else {
##			$local_count = $local_count - 1;
##		    }
##		}
##		if ($local_count == 2) {
##		    my ($cityname) = $location;
##		    my ($statename) =$location;
##		    
##		    if ($cityname =~ /PO Box/) {
##			@fields = split(/, /, $cityname);
##			
##			$statename = $fields[3];
##			$cityname = $fields[2];
##		    }
##		    else {
##			@fields = split(/, /, $cityname);
##			$cityname = $fields[1];
##			$statename = $fields[2];
##		    }
##		    print DATA "city           $cityname     \n";
##		    print DATA "state          $statename     \n";
##		}
##		if ($local_count == 3) {
##		    if($location =~ /[0-9]*/) {
##			print DATA "telephone      $location     \n";
##			print DATA "miles          $mile     \n";
##			print DATA "refer_name     $ref_name   \n";
##			print DATA "refer_add      $ref_add    \n";
##		    }
##		}
##	    }
##	    
##	    $local_count = $local_count + 1;
##	}
##    }
##    
##    &status("received $count_num location(s) from mapquest");
##    
##    close DATA;
##    
##    if ($count == 0){
##	@busy = $file =~ /System Busy/g;
##	# $#array refers to the last populated element, or -1 if it is
##	# empty. So rather than have to loop, just do a simple math op
##	#
##	$count = $count - ($#busy + 1);
##	#foreach $c (@busy){
##	#    $count = $count - 1;
##	#}
##	
##	@busy = $file =~ /Mapquest.com Busy/g;
##	$count = $count - ($#busy + 1);
##	
##	@busy = $file =~ /Unexpected Response/g;
##	$count = $count - ($#busy + 1);
##	
##    }
##    return $count_num;
##}

############################
# web_location_1.perl
############################

sub goto_web_location_1 {
    my ($business_type, $start_add, $city, $state, $zip, $myval) = @_;
    
    # Check parameters!
    #
    defined $business_type ?
	print " business_type is [$business_type]\n"
	: die "business_typ is not defined!\n";
    
    defined $myval ?
	print " myval is [$myval]\n"
	: die "myval is not defined!\n";
    
    defined $city ?
	print " city is [$city]\n"
	: die "city is not defined!\n";
    
    defined $state ?
	print " state is [$state]\n"
	: die "state is not defined!\n";
    
    defined $zip ?
	print " zip is [$zip]\n"
	: die "zip is not defined!\n";
    
    defined $start_add ?
	print " start_add is [$start_add]\n" : "no-op";
    
    # backup all variables without the change of +'s
    #
    my $orig_business_type = $business_type;
    #these are only used in the query so I can safely destroy them
    #and not have to worry about having backups
    #my $orig_city = $city;
    #my $orig_state = $state;
    #my $orig_zip = $zip;
    #my $orig_start_add = $start_add;
    
    # Relace all the spaces with +'s so that in the http query I format it
    # correctly.
    #
    $business_type =~ tr/ /+/;
    $city =~ tr/ /+/;
    $state =~ tr/ /+/;
    $zip =~ tr/ /+/;
    $start_add =~ tr/ /+/;
    
    my $count = 0;
    my $RETRY_TIME_OUT = 5;
    my $cookie_jar = HTTP::Cookies::Netscape->new(
						  File => $COOKIE_FILENAME,
						  AutoSave => 1);
    
    # Create the user-agent
    #
    my $ua = new LWP::UserAgent;
    $ua->cookie_jar($cookie_jar);
    $ua->agent("Mozilla/4.08 [en] (X11; U; Linux 2.0.32 i686)");
    
    # Setup the request
    #
    my $req = new HTTP::Request POST => 'http://www.mapquest.com/maps/map.adp';
    $req->content_type('application/x-www-form-urlencoded');  
    
    #print STDERR "\n business type:  $business_type\n";
    #print STDERR " suffix value:  $myval\n\n";
    
    &status("Submiting location query");
    
    #no reference address
    #formtype=search&searchtype=search&country=US&addtohistory=&cat=walmart&city=Starkville&state=MS&zipcode=39759
    #"formtype=search&searchtype=search&country=US&addtohistory=&cat=".$business_type."&city=".$city."&state=".$state."&zipcode=".$zip
    
    #yes reference address
    #country=US&addtohistory=&formtype=searchadv&searchtype=search&cat=walmart&address=UniversityDr&city=Starkville&state=MS&zipcode=39759
    #"country=US&addtohistory=&formtype=searchadv&searchtype=search&cat=".$business_type."&address=".$start_add."&city=".$state."&state=".$zip."&zipcode=".$zip

    
    #create the query for the page
    if ($start_add ne "") { #if start_address is NOT empty
	$sendString = "country=US&addtohistory=&formtype=searchadv&searchtype=search&cat=".$business_type."&address=".$start_add."&city=".$state."&state=".$zip."&zipcode=".$zip
	}
    else { #if start_address is empty
	$sendString = "formtype=search&searchtype=search&country=US&addtohistory=&cat=".$business_type."&city=".$city."&state=".$state."&zipcode=".$zip;
    }
    
    print "debug: sendString is [$sendString]\n";
    #print STDERR "$sendString";
    
    # Make the request
    #
    $req->content($sendString);
    my $res = $ua->request($req);
    
    #print "debug: Result content is [$res->content]\n"; 
    
    if ($res->is_success) {
	print "debug: Result content is good.\n";
	#print "Value is [$res->content]\n";
    }
    else {
	print "debug: Result content is bad!\n";
	#print "Value is [$res->content]\n";
	return -1;
    }
    
    my $local_file = $res->content;
    
    # DEBUG
    $outfile = "$TEMP_PATH/isip_db_perl_actual_html.html";
    open(DEBUGDATA,">$outfile") || die "CANNOT OPEN $outfile";
    print DEBUGDATA $local_file;
    close DEBUGDATA;
    # END DEBUG
    
    # if the returned page include "Most Popular Matching Categories", go
    # to the first meaningful links in this page #
        
    #look for most popular
    $local_file = &find_popular_matching($local_file, $req, $ua);

    # the page downloaded right now is in $res->content as basically a string
    # this function returns the number of things found on the page as valid
    # addresses
    
    print "entering my web_location_1_page_new\n";
    $count = 0;
    # if the returned page include "Next Page", go to the first 
    # meaningful links in this page
    #
    while (1) {
	$count += &parse_web_location_1_page_new($local_file, $business_type,
						 $myval);
	
	$local_file = &find_next_page($local_file, $req, $ua);
	if ($local_file eq "") {
	    last;
	}
    }
    
    return $count;
}

# -----------------------------------------------------------------------------
#
#  Routine which parses the returned web page from mapquest.com for
#  address information The output is written to a file called
#  "flatmap".
#
#  ---------------------------------------------------------------------------

#2005-03-25
#this is now the layout for the result html file

#layout 1: If the file has the miles away
#<td class=boldText><b>Depot Citgo</b></td></tr><tr><td></td><td>&nbsp;</td><td class=size12>1410 Main St, Columbus, MS&nbsp;(23.33 miles away)<tr><td></td><td>&nbsp;</td><td class=size12>662-327-0926</td>

#layout 2: If the file does not have the miles away
#note: ignoring layout #2 in all of my parsing
#<td class=boldText>*&nbsp;Shoppette / Class Vi Store</td></tr><tr><td></td><td>&nbsp;</td><td class=size12>Bldg 161, Columbus Afb, MS<tr><td></td><td>&nbsp;</td><td class=size12>662-434-6026</td>

sub parse_web_location_1_page_new {
    
    my ($file, $mybusiness, $myval) = @_;
    
    # Declare some variables
    my $num = 0;
    my $mile;
    
    # save only the part of the html that has the table that
    # contains the information about the locations
    # it just happens to be that directly before there is a javascript func
    # named 'selectSortBy' and afterwards there is a 'openGoogle'. If these
    # change on the Mapquest page then they need to be changed here
    #
    my $html = $file;
    $html =~ /function selectSortBy(.*)function openGoogle()/gs;
    $html = $1;
    
    #print STDERR "html value is [$html]\n";
    
    #DEBUG--
    # This is sample code for matching in this html. 
    #my $direction = $html;
    #@miles = $direction =~ /([0-9\.]*) miles away/g;
    #print "miles is ".@miles."\n";
    #my $num = 0;
    #foreach my $unused_var (@miles) {
    #my $mile = $miles[$num];
    #$num++;
    #print STDERR "num : $num: [$unused_var] $mile\n";
    #}
    #print STDERR "total num : $num\n";
    #END DEBUG---------
    
    # if output file is open, fileopen is 1. it will be opened
    # the first time there is output to the file
    my $fileopen = 0;
    my $outfile1 = 0;
    
    my $found = 0;
    my $linenum = 0;
    @lines = split(/\n+/, $html);
    foreach my $line (@lines) {
	
	@entries = $line =~ /<td class=boldText><b>([^<]*)<\/b><\/td><\/tr><tr><td><\/td><td>&nbsp;<\/td><td class=size12>([^&]*)&nbsp;\(([0-9.]*) miles away\)<tr><td><\/td><td>\&nbsp;<\/td><td class=size12>([^<]*)<\/td>/g;
	
	#there are 4 entries per line.
	#example as such:
	#0: Highway 82 Texaco & Tobacco
	#1: 101 Martin Luther King Jr Dr W, Starkville, MS
	#2: 1.30           <<- hint: This is "miles away"
	#3: 662-323-1222
	
	if (@entries == 4) {
	    
	    # open the output file if it hasn't been opened yet
	    if ($fileopen == 0) {
		$outfile1="$OUT_PATH/flatmap_$myval";
		open(DATA, ">>$outfile1") || 
		    die "ERROR: can't open $outfile.\n";
		$fileopen = 1;
	    }
	    
	    #split out the address, city, state
	    my ($name, $address, $city, $state, $miles, $phone) =
		(@entries[0], "", "", "", @entries[2], @entries[3]);
	    
	    #populate the address, city, state from the parsing of the
	    #entries[1] splitting
	    #
	    #print "looking at entry 1 which is[".@entries[1]."]\n";
	    if (@entries[1] =~ /([^,]*),([^,]*),(.*)/g) {
		$address = trim($1);
		$city = trim($2);
		$state = trim($3);
	    }
	    
	    #print "on line $linenum there are some entries!\n";
	    print "Found as [$name] [$address] [$city] ".
		"[$state] [$miles] [$phone]\n";
	    
	    #DEBUG output everything in @entriesdebug output
	    #my $entnum = 0;
	    #foreach (@entries) {
		#print "$entnum: $_\n";
		#$entnum++;
	    #}
	    #END DEBUG -----
	    print DATA "name           $name      \n";
	    print DATA "address        $address   \n";
	    print DATA "address        $city      \n";
	    print DATA "state          $state     \n";
	    print DATA "telephone      $phone \n";
	    print DATA "miles          $miles     \n";
	    $found++;
	}
	$linenum++;
    }
    
    # if the output file was opened, close it
    if ($fileopen == 1) {
	close $outfile1;
    }

    return $found;
}

#sub parse_web_location_1_page {
#    
#    my ($file, $mybusiness, $myval) = @_;
##    
##    # Declare some variables
##    my $num = 0;
##    my $mile;
##    
##    # remove the html tags
##    #
##    foreach $rt ($file) {
##	$rt =~ s/<(?:[^>'"]*|(['"]).*?\1)*>/\n/gs; # '"])>
##	#so that emacs will color the rest of the file corrctly
##	#print STDERR $rt; 
##    }
##    
##    # DEBUG
##    print DEBUGDATA "\n<br>=========================AFTER removal HTML===\n";
##    print DEBUGDATA $file;
##    close DEBUGDATA;
##    # END DEBUG
##    
##    #my $temp_file = $file;
##    #my $distance = $temp_file;
##    my $html = $file;
##    
##    print "debug html is [html]\n"; 
##    
##    # pick up the listing stuff
##    #
##    # /gs options are g=global + s=treat as single line
##    $html =~ /Next Page(.*)Listings On a Map/gs;
##    
##    # The line below effectively does nothing because from ^ to $ is not null.
##    # and even if it is, it will be erased in the line below.
##    # mwe-this is the way I read it and experimentation shows.
##    # see "test.perl" in elliott directory.
##    #
##    #$distance =~ s/^(\s)*$//g;
##    
##    $html = $1;
##    
##    #print "*************************************\n";
##    #print "$html\n";
##    
##    my $direction = $html;
##    
##    # find the miles away from the start place
##    #
##    # This is not perfect because some might have an address and not
##    # a distance away.
##    #
##    @miles = $direction =~ /[0-9\.]* miles away\n/g;
##    
##    #DEBUG BEGIN
##    my $num = 0;
##    foreach my $unused_var (@miles) {
##	
##	my $mile = $miles[$num];
##	$num++;
##	print STDERR "num : $num: $mile\n";
##    }
##    print STDERR "total num : $num\n";
##    #END DEBUG
##    
##    @directions = $html =~ /\w.*/g;
##    
##    $outfile1="$OUT_PATH/flatmap_$myval";
##    open(DATA, ">>$outfile1") || die "ERROR: can't open $outfile.\n";
##    
##    # Pattern Found looks like this once the html has all been removed
##    # 
##    # TODO PUT HERE
##    
##    # This foreach loop assumes that the place begins with
##    # "#.## miles away"
##    # and that the data comes in the following order
##    # Name, Address, City, State, Telephone
##    # with the telephone number coming on a line by itself
##    #
##    # It is a simple FSM that would be very simple to break
##    # if mapquest changes up their page.
##    #
##    my $count = 0;
##    my $local_count = 9;
##    my $local_flag = 1;
##    foreach $r (@directions) {
##	
##	my $placename = $directions[$count];
##	print STDERR "Step $local_count: $placename\n";
##	$count++;
##	
##	# First look for "miles away" because that shows we are about
##	# to find a place
##	if ($placename =~ /miles away/) {
##	    $mile = $placename;
##	    print STDERR "entire mile phrase: *****$mile*****\n";      
##	    $mile =~ /([0-9\.]*) miles away/g;
##	    $mile = $1; 
##	    print STDERR "just miles: *****$mile*****\n";
##	    $local_count = 7;
##	}
##	
##	if ($local_count < 9) {
##	    if (($placename =~ /More.../) ||
##		($placename =~ /nbsp/) ||
##		($placename =~ /Maps/) ||
##		($placename =~ /Show These/) ||
##		($placename =~ /Web Site/) ||
##		($placename =~ /Driving Directions/)) {
##		$local_count = 7;
##	    }
##	    else {
##		if ($local_count == 8) {
##		    $local_count = 1;  # means print out now
##		}
##	    }
##	    
##	    print STDERR "$local_count===I saw $placename\n";
##	    
##	    if ($local_count < 7) { 
##		if ($local_count == 1) {
##		    print DATA "name           $placename     \n";
##		}
##		if ($local_count == 2) {
##		    my $local_address = $placename;
##		    if ($local_address =~ /,/) {
##			#              $local_address =~ /([0-9]+.*)/g;
##			$local_address =~ /([^\,]*)/g;
##			$local_address = $1;
##			print DATA "address        $local_address     \n";
##		    }
##		    else {
##			$local_count = $local_count - 1;
##		    }
##		}
##		if ($local_count == 2) {
##		    # This section is quite bad because it has a "worst-case"
##		    # of putting the same thing for both city and state
##		    my ($cityname) = $placename;
##		    my ($statename) = $placename;
##		    if ($cityname =~ /PO Box/) {
##			@fields = split(/, /, $cityname);
##			
##			$cityname = $fields[2];
##			$statename = $fields[3];
##		    }
##		    else {
##			@fields = split(/, /, $cityname);
##			$cityname = $fields[1];
##			$statename = $fields[2];
##		    }
##		    
##		    print DATA "city           $cityname     \n";
##		    print DATA "state          $statename     \n";
##		}
##		if ($local_count == 3) {
##		    # This gets here only after the above has been written
##		    # and it has at least one number in it.
##		    #
##		    if($placename =~ /[0-9]*/) {
##			print DATA "telephone      $placename     \n";
##			print DATA "miles          $mile     \n";
##		    }
##		}#end 3
##		}
##	    
##	    $local_count = $local_count + 1;
##	}
##    }
##    
##    #print STDERR "=======================================================\n";
##    #print STDERR "directions:@directions\n\n";
##    #print STDERR "*******************************************************\n";
##    
##    &status("received $count directions from mapquest");
##    
##    close DATA;
##    
##    if ($count == 0) {
##	@busy = $file =~ /System Busy/g;
##	# $#array refers to the last populated element, or -1 if it is
##	# empty. So rather than have to loop, just do a simple math op
##	#
##	$count = $count - ($#busy + 1);
##	#foreach $c (@busy){
##	#    $count = $count - 1;
##	#}
##	
##	@busy = $file =~ /Unexpected Response/g;
##	$count = $count - ($#busy + 1);
##    }
#    
#    return $count;
#}


############################
# web_location.perl - Not working, left in for historical reasons
############################
#
#sub goto_web_location {
#    #These are the arguments
#    #business_type   = 0
#    #street_name     = 1
#    #city            = 2
#    #state           = 3
#    #zip             = 4
#    #myval         = 5
#    my ($business_type, $street_name, $city, $state, $zip, $myval) = @_;
#    
#    print "entering goto_web_location() with:\n";
#    #print "business_type is [$business_type]\n";
#    #print "zip is [$zip]\n";
#    
#    # Parameter checking!
#    #
#    defined $business_type ?
#	print " business type is [$business_type]\n"
#	: die "business type is not defined!\n";
#    
#    defined $street_name ?
#	print " street name is [$street_name]\n" 
#	: die "street name is not defined!\n";
#    
#    defined $city ?
#	print " city is [$city]\n"
#	: die "city is not defined!\n";
#    
#    defined $state ?
#	print " state is [$state]\n"
#	: die "state is not defined!\n";
#    
#    defined $zip ?
#	print " zip is [$zip]\n"
#	: die "zip is not defined!\n";
#    
#    # Create everything that goes with the HTTP request
#    #
#    my @result=();
#    my $RETRY_TIME_OUT = 5;
#    my $cookie_jar = HTTP::Cookies::Netscape->new(File => "cookies.txt",
#						  AutoSave => 1);
#    my $ua = new LWP::UserAgent;
#    $ua->cookie_jar($cookie_jar);
#    $ua->agent("Mozilla/4.08 [en] (X11; U; Linux 2.0.32 i686)"); # . $ua->agent);
#    
#    my $req = new HTTP::Request POST => 'http://yp.mapquest.com/main.adp';
#    
##  my $req = new HTTP::Request POST => 'http://dps1.travelocity.com/lognmain.ctl';
#    $req->content_type('application/x-www-form-urlencoded');  
#    #mwe-removed
#    #print STDERR "\n hotel city code:  $city\n";
#    #print STDERR " user: $userName \n password: $password\n\n";
#    #mwe-end
#    
#    #$sendString = "es_alias=".$userName."&es_passwd=".$password.
#    "&tr_module=CARS";
#    
#    
#    # Set up some variables
#    # That don't seem to be used anymore
#    #
##  $outsuccessflag = 0;
##  $insuccessflag = 0;
##  $retryCount = 0;
##  $num_retry = 1;
#    
#    # Submit the request to the web
#    #
#    $req->content($sendString);                         
#    &status("Entering goto_web_location - accessing web");
#    my $res = $ua->request($req);  
#    &status("Submitting query");
#    
##  my $req = new HTTP::Request POST => 'http://www.mapquest.com/directions/main.adp';
#    
## http://www.mapquest.com/directions/main.adp?go=1&do=nw&ct=NA&1y=US&1a=&1p=&1c=&1s=&1z=39759&1ah=&2y=US&2a=&2p=&2c=Jackson&2s=MS&2z=&2ah=&lr=2&x=52&y=18
#    
#    # $req->content_type('application/x-www-form-urlencoded');
#    
## $sendString = "1z=".$start_zip."&2c=".$end_city."&2s=".$end_state;
#    
## http://www.mapquest.com/directions/main.adp?go=1&do=nw&2n=Hinds&ct=NA&1y=US&1a=110+LINCOLN+GREEN&1p=&1c=Starkville&1s=MS&1z=&1ah=&2y=US&2a=&2p=&2c=Jackson&2s=MS&2z=&2ah=&lr=2&x=59&y=6
#    
##  $sendString = "go="."1"."&do="."nw"."&ct="."NA"."&1y="."US"."&1a="."&1p"."="."&1c"."="."&1s"."="."&1z"."=".$start_zip."&1ah="."&2y"."="."US"."&2a="."&2p"."="."&2c"."=".$end_city."&2s=".$end_state."&2z="."&2ah"."="."&lr"."="."2"."&x="."52"."&y="."18";
##  $sendString = "go="."1"."&do="."nw"."&ct="."NA"."&1y="."US"."&1a="."110 Lincoln Green"."&1p"."="."&1c"."="."&1s"."="."&1z"."=".$start_zip."&1ah="."&2y"."="."US"."&2a="."&2p"."="."&2c"."=".$end_city."&2s=".$end_state."&2z="."&2ah"."="."&lr"."="."2"."&x="."52"."&y="."18"."&mo=";
#    
##  $sendString = "_dirnamesearch=".$business_type."&_diraction="."catsearch"."&_dircity=".$city."&_dirstate=".$state."&_dirlat=334459"."&_dirlong=-888177"."&_dirzip=".$zip;
#    
#    #mwe begin
##For this section, goto
##http://www.mapquest.com/maps/main.adp?formtype=searchadv&address=&city=&state=&zipcode=
##and it has "Find a Business Near an Address" box with
##1) Business Name or Catagory
##2) Address
##3) City
##4) State
##5) Zip
##
## Three examples: Should be pretty self-explanatory
##
##http://www.mapquest.com/maps/map.adp?formtype=searchadv&searchtype=search&country=US&addtohistory=&cat=AAABBBCCC&address=DDDEEEFFF&city=GGGHHHIII&state=ZZ&zipcode=55555
##http://www.mapquest.com/maps/map.adp?formtype=searchadv&searchtype=search&country=US&addtohistory=&cat=hotels&address=university+drive&city=Starkville&state=MS&zipcode=39759
##http://www.mapquest.com/maps/map.adp?formtype=searchadv&searchtype=search&country=US&addtohistory=&cat=aaa+bbb&address=ccc+ddd&city=Starkville&state=MS&zipcode=39759
#    #mwe end
#    
#    
##2005-02-04-CHANGED
##  $sendString = "_diraddressloc=".$street_name."&_dirlocation=39759&_dirchange=1&_dirretpage=modify&_diraction=location&_dircity=".$city."&_dirstate=".$state."&_dirlat=334623&_dirlong=-887990&_dirzip=".$zip."&_dirmyaddr=&_dircat=701101&_dirnamesearch=".$business_type;
#    $sendString = "_diraddressloc=".$street_name."&_dirlocation=39759&_dirchange=1&_dirretpage=modify&_diraction=location&_dircity=".$city."&_dirstate=".$state."&_dirlat=334623&_dirlong=-887990&_dirzip=".$zip."&_dirmyaddr=&_dirnamesearch=".$business_type;
#    
##  $sendString = "_dirnamesearch="."banks"."&_diraction="."search"."&_dirskip=10"."&_dircity="."Starkville"."&_dirstate="."MS"."&_dirlat=334459"."&_dirlong=-888177"."&_dirzip=39759";
#    
#    
#    print "send string is: [$sendString]\n";
#    #debug
#    #print STDERR "send string is: [$sendString]\n";
#    #end debug
#    
#    $req->content($sendString);
#    my $res = $ua->request($req);
#    
#    print "result content is [$res->content]\n"; 
#    
#    my ($local_file) = $res->content;
#    
#    # DEBUG
#    $outfile = "actual_html.html";
#    open(DEBUGDATA,">$outfile") || die "CANNOT OPEN $outfile";
#    print DEBUGDATA $local_file;
#    close DEBUGDATA;
#    # END DEBUG
#    
#    
#    my @second;
#    my $links;
#    
#    # if the returned page include "multiple addresses", go to the first 
#    # meaningfullinks in this page
#    #
#    if ($local_file =~ /multiple addresses/) {
#	print $local_file;		
#	print "=====================\n";		
#	print $street_name;		
#	print "=====================\n";		
#	
#	@second = $local_file =~ /href=\".*MS</g;
#	print "[$second[0]]*******\n";
#	$links = $second[0];
#	print "**[$links]*****\n";
#	$links =~ /href=\"main.adp\?([^\"]+)/g;
#	$links = $1;
#	print "*****[$links]*****\n";
#	
#	$req->content($links);
#	$res = $ua->request($req);
#	
#	print "$res->content"; 
#	
#    }
#    
#    $local_file = $res->content;
#    
#    # if the returned page include "Refine Your Search", go to the first 
#    # meaningfullinks in this page
#    #
#    if ($local_file =~ /Refine Your Search/) {
#	print $local_file;		
#	@second = $local_file =~ /href=\".*matches/g;
#	print "[$second[0]]*******\n";
#	$links = $second[0];
#	print "**[$links]*****\n";
#	$links =~ /href=\"main.adp\?([^\"]+)/g;
#	$links = $1;
#	print "*****[$links]*****\n";
#	
#	$req->content($links);
#	$res = $ua->request($req);
#	
#	print "$res->content"; 
#	
#    }
#    
#    
#    $count = &parse_web_location_page($res->content, $street_name, $myval);
#    return $count;
#}
#
#sub parse_web_location_page {
#    
#    my ($file, $street_name, $myval) = @_;
#    
#    print "$file"; 
#    
#    my @result; my @final;
#    
#    my $num = 0;
#    
#    foreach $rt ($file) {
#	$rt=~ s/<(?:[^>'"]*|(['"]).*?\1)*>/\n/gs; # '"])> #fix
#	#fix so that emacs will display this right		 
#	#print STDERR $rt; 
#    }
#    
#    
#    my $temp_file = $file;
#    
#    my $distance = $temp_file;
#    
#    $distance =~ /Listings By Distance(.*)Listings* [0-9]/gs;
#    $distance =~ s/^(\s)*$//g;
#    
#    $distance = $1;
#    
##   print STDERR "$distance\n";
#    
##   my $time = $distance;
##    my $from = $distance;
##   my $to = $distance;
#    my $direction = $distance;
#    
#    @miles = $direction =~ /[0-9\.]* miles\n/g;
#    my $num = 0;
#    foreach $r (@miles) {
#	
#	my $mile = $miles[$num];
#	$num++;
#	print STDERR "num : $num: $mile\n";
#    }
#    
#    print STDERR "num : $num\n";
#    
##   @directions = $direction =~ /Add.*[0-9\.]*miles/gs;
#    @directions = $distance =~ /\w.*/g;
#    
#    my $count = 0;
#    
#    $outfile1="$OUT_PATH/flatmap_$myval";
#    open(DATA, ">>$outfile1") || die "ERROR: can't open $outfile.\n";
#    my $local_count = 1;	
#    my $local_flag = 1;	
#    
#    foreach $r (@directions) {
#	
#	my $hotel = $directions[$count];
#	print STDERR "Step $local_count: $hotel\n";
#	$count++;
##      print STDERR "count: *****$count*****\n";
#	
#	
#	if (($hotel =~ /from you/) ||
#	    ($hotel =~ /Get Maps/) ||
#	    ($hotel =~ /Find Places Nearby/) ||
#	    ($hotel =~ /Read Reviews/) ||
#	    ($hotel =~ /Add Your Review/)) {
#	    $local_count = 6;
#	}
#	else {
#	    if ($local_count == 7) {
#		$local_count = 1;
#	    }
#	    print STDERR "I saw $hotel\n";
#	}
#	
#	
#	if ($local_count < 6) { 
#	    if ($local_count == 1) {
#		print DATA "name           $hotel     \n";
#	    }
#	    if ($local_count == 2) {
#		my $local_address = $hotel;
#		#  $local_address =~ /([0-9\]+.*)/g;
#		print DATA "address        $local_address     \n";
#	    }
#	    if ($local_count == 3) {
#		print DATA "city           $hotel     \n";
#	    }
#	    if ($local_count == 4) {
#		print DATA "telephone      $hotel     \n";
#	    }
#	    if ($local_count == 5) {
#		print DATA "miles          $hotel     \n";
#		print DATA "startaddress   $street_name     \n";
#	    }
#	}
#	
#	$local_count = $local_count + 1;
#	
#    }
#    
#    print STDERR "=======================================================\n";
#    print STDERR "directions:@directions\n\n";
#    print STDERR "*******************************************************\n";
#    
#    
#    $outfile="$OUT_PATH/flatmap_$myval";
#    open(DATA, ">>$outfile") || die "ERROR: can't open $outfile.\n";
#    
#    print DATA "from           $from\n";
#    print DATA "to             $to\n";
#    print DATA "total_distance $distance\n";
#    print DATA "total_time     $time\n";
#    print DATA "directions     $count     \n";
#    print DATA "directions     $street_name     \n";
#    
#    my $number =0;
#    foreach $r (@directions) {
#	my $direction = $directions[$number];
#	my $mile = $miles[$number];
#	print DATA "  $direction         $mile";
#	$number++;
#    }
#    print DATA "\n";
#    
#    &status("received $count directions from mapquest");
#    
#    close DATA;
#    
#    if ($count == 0){
#	@busy = $file =~ /System Busy/g;
#	foreach $c (@busy){
#	    $count = $count - 1;
#	}
#	@busy = $file =~ /Travelocity Busy/g;
#	foreach $c (@busy){
#	    $count = $count - 1;
#	}
#	@busy = $file =~ /Unexpected Response/g;
#	foreach $c (@busy){
#	    $count = $count - 1;
#	}
#	
#    }
#    
#    
#    return $count;
#}

############################
# web_map.perl
############################

sub goto_web_map {
    my ($start_name, $start_add, $start_interest,
	$start_city, $start_state, $start_zip,
	$end_name, $end_add, $end_interest,
	$end_city, $end_state, $end_zip, $myval) = @_;    
    
    defined $start_add ?
	print " start_add is [$start_add]\n"
	: die "start_add is not defined!\n";
    
    defined $start_city ?
	print " start_city is [$start_city]\n"
	: die "start_city is not defined!\n";
    
    defined $start_state ?
	print " start_state is [$start_state]\n"
	: die "start_state is not defined!\n";
    
    defined $end_add ?
	print " end_add is [$end_add]\n"
	: die "end_add is not defined!\n";
    
    defined $end_city ?
	print " end_city is [$end_city]\n"
	: die "end_city is not defined!\n";

    defined $end_state ?
	print " end_state is [$end_state]\n"
	: die "end_state is not defined!\n";
    
    defined $myval ?
	print " myval is [$myval]\n"
	: die "myval is not defined!\n";
    
    # save the originals to start with so that i can have a copy without +s
    # for the print out (eg, flatmap)
    #
    my $orig_start_add = $start_add;
    my $orig_start_name = $start_name;
    my $orig_end_add = $end_add;
    my $orig_end_name = $end_name;
    
    # Make all the names have '+'s instead of ' 's
    #
    $start_name =~ tr/ /+/;
    $start_add =~ tr/ /+/;
    $start_interest =~ tr/ /+/;
    $start_city =~ tr/ /+/;
    $start_state =~ tr/ /+/;
    $start_zip =~ tr/ /+/;
    $end_name =~ tr/ /+/;
    $end_add =~ tr/ /+/;
    $end_interest =~ tr/ /+/;
    $end_city =~ tr/ /+/;
    $end_state =~ tr/ /+/;
    $end_zip  =~ tr/ /+/;
    
    # Create the cookie file
    #
    my $cookie_jar = HTTP::Cookies::Netscape->new(
						  File => $COOKIE_FILENAME,
						  AutoSave => 1);
    
    # Make the UserAgent
    #
    my $ua = new LWP::UserAgent;
    $ua->cookie_jar($cookie_jar);
    $ua->agent("Mozilla/4.08 [en] (X11; U; Linux 2.0.32 i686)");
    
    # Make the request object
    #
    my $req = new HTTP::Request 
	POST => 'http://www.mapquest.com/directions/main.adp';
    
    $req->content_type('application/x-www-form-urlencoded');  

    # need to make sure the address has a number with it. The follow code will
    # make sure there is a starting digit on the address
    #
    # !~ is the same as !(=~)
    # if it begins with a [ that most likely means
    # it's a [100-200] address range
    #
    if ($start_add !~ /^\d.*/g && $start_add !~ /^\[.*/g && $start_add ne "") {
	$start_add = "100 ". $start_add;
    }

    if ($end_add !~ /^\d.*/g && $end_add !~ /^\[.*/g && $end_add ne "") {
	$end_add = "100 ". $end_add;
    }

    
    print STDERR "from: $start_add $start_city $start_zip\n";
    print STDERR "to: $end_add $end_city $end_zip\n\n";
    
    &status("Submiting directions query");
    
    $sendString = "go=1&do=nw&un=m&cl=EN&ct=NA&rsres=1&1y=US&1a=".$start_add."&1c=".$start_city."&1s=".$start_state."&1z=".$start_zip."&2y=US&2a=".$end_add."&2c=".$end_city."&2s=".$end_state."&2z=".$end_zip."&dir=Get+Directions";
    
    #Example from where i made the above sendString
    #
    #http://www.mapquest.com/directions/main.adp?go=1+&1a=1660+Hwy+19+South&1c=Meridian&1s=MS&1z=39301&1y=US&2a=522+University+Dr.&2c=Starkville&2s=MS&2z=39759&2y=US&dir=Get+Directions
    
    # Found this string at the following webpage:
    # > http://www.mapquest.com/features/main.adp?page=lf_dir
    # It has help for how to use mapquest. For some reason it doesn't seem to
    # like the "CID" part.
    #
    #$sendString = "go=1&do=nw&un=m&cl=EN&ct=NA&rsres=1&1y=US&1a=".$start_add."&1c=".$start_city."&1s=".$start_state."&1z=".$start_zip."&2y=US&2a=".$end_add."&2c=".$end_city."&2s=".$end_state."&2z=".$end_zip."&CID=lfddwid";
    
    print "String used for query:\n[$sendString]\n";
    
    # Make request and get back result
    #
    $req->content($sendString);
    my $res = $ua->request($req);
    my $local_file = $res->content;
    
    #DEBUGDATA
    open(DEBUGDATA,">$TEMP_PATH/isip_db_perl_actual_html.html") ||
	die "unable to open outfile!";
    print DEBUGDATA "$local_file"; 
    close DEBUGDATA;
    #END DEBUGDATA
    
    #print "This is where the local_file would be [local_file]\n"; 
    #print $local_file;
    
    # START and END locations are the same
    #
    if ($local_file =~ /START and END locations are the same/) {
	
	$outfile="$OUT_PATH/flatmap_$myval";
	open(DATA, ">$outfile") || die "ERROR: can't open $outfile.\n";
	
	my $distance;
	my $time;
	my $from;
	my $to;
	my $direction;
	my $count = 1;
	
	print DATA "start_name          $orig_start_name\n";
	print DATA "start_add           $orig_start_add\n";
	
	print DATA "start_interest      $start_interest\n";
	print DATA "start_city          $start_city\n";
	print DATA "start_state         $start_state\n";
	print DATA "start_zip           $start_zip\n";
	
	print DATA "end_name            $orig_end_name\n";
	print DATA "end_add             $orig_end_add\n";
	print DATA "end_interest        $end_interest\n";
	print DATA "end_city            $end_city\n";
	print DATA "end_state           $end_state\n";
	print DATA "end_zip             $end_zip\n";
	
	print DATA "total_distance      0 \n";
	print DATA "total_time          0.0 minutes\n";
	# On the below line, $count is always '0';
	print DATA "directions          $count     \n";
	
	print DATA "It appears that your START and END locations are the same or very close to each other\n";
	print DATA "0.0\n";
	print DATA "\n";
	return 1;
    }
    
    # TODO THIS SECTION NEEDS TO BE RE-WRITTEN. SEE "web_map_similar.html" IN
    # THIS SECTION
    # Done, This is now new'ly written!
    if ($local_file =~ /found.*similar/) {
	# this document now fills in the form boxes with the data we want
	# so pull it out of the form
	# 1a 1s 1c 1z =      source address, state, city, zip
        # 2a 2s 2c 2z = destination address, state, city, zip

	my ($new_start_add, $new_start_city, $new_start_zip,
	    $new_end_add, $new_end_city, $new_end_zip) = ("","","","","","");

	@lines = split(/\n+/, $local_file);
	print "lines array is ".($#lines+1)." large\n";
	foreach my $line (@lines) {
	    # find the source information
	    #
	    if ($line =~ /.*id=\"1a\"(.*)value=\"([^\"]*)\".*/g) {
		print "Found repl source address as [".$1."] [".$2."]\n";
		$new_start_add = $2;
	    }

	    if ($line =~ /.*id=\"1c\"(.*)value=\"([^\"]*)\".*/g) {
		print "Found repl source city as [".$1."] [".$2."]\n";
		$new_start_city = $2;
	    }

	    if ($line =~ /.*id=\"1s\"(.*)value=\"([^\"]*)\".*/g) {
		print "Found repl source state as [".$1."] [".$2."]\n";
		$new_start_state = $2;
	    }

	    if ($line =~ /.*id=\"1z\"(.*)value=\"([^\"]*)\".*/g) {
		print "Found repl source zip as [".$1."] [".$2."]\n";
		$new_start_zip = $2;
	    }

	    # find the destination information
	    #
	    if ($line =~ /.*id=\"2a\"(.*)value=\"([^\"]*)\".*/g) {
		print "Found repl destin address as [".$1."] [".$2."]\n";
		$new_end_add = $2;
	    }

	    if ($line =~ /.*id=\"2c\"(.*)value=\"([^\"]*)\".*/g) {
		print "Found repl destin city as [".$1."] [".$2."]\n";
		$new_end_city = $2;
	    }

	    if ($line =~ /.*id=\"2s\"(.*)value=\"([^\"]*)\".*/g) {
		print "Found repl destin state as [".$1."] [".$2."]\n";
		$new_end_state = $2;
	    }

	    if ($line =~ /.*id=\"2z\"(.*)value=\"([^\"]*)\".*/g) {
		print "Found repl destin zip as [".$1."] [".$2."]\n";
		$new_end_zip = $2;
	    }
	}

	# Now I should have information that can be parsed by this
	# function so make a recursive call
	#
	print "Calling goto_web_map() again with new data\n\n";
	return &goto_web_map($orig_start_name, $new_start_add, "",
			     $new_start_city, $new_start_state, $new_start_zip,
			     $orig_end_name, $new_end_add, "",
			     $new_end_city, $new_end_state, $new_end_zip,
			     $myval);
#	my @first;
#	my @second;
#	my $links;
#	
#	print $local_file;
#	print "=====================\n";
#	print $orig_start_add;
#	print "=====================\n";
#	
##     @second = $local_file =~ /\<option value=\".*>/g;
#	print @second;
#	
#	
#	my ($start) = $local_file =~ /Start(.*)/gs;
#	my ($end) = $local_file =~ /End(.*)/gs;
#	@first = $start =~ /\<option value=\".*>/g;
#	@second = $end =~ /\<option value=\".*>/g;
#	
#	my $num = 0;
#	foreach $r (@first) {
#	    
#	    my $mile = $first[$num];
#	    print STDERR $mile;
#	    print STDERR "\n\n";
#	    $num++;
#	}
#	
#	print STDERR "number = $num\n";
#	
#	$num = 0;
#	foreach $r (@second) {
#	    
#	    my $mile = $second[$num];
#	    print STDERR $mile;
#	    print STDERR "\n\n";
#	    $num++;
#	}
#	
#	print STDERR "number = $num\n";
#	
#	my ($first_item) = $first[0] =~ /<option value=\"(.*)\">/g;
#	my ($second_item) = $second[0] =~ /<option value=\"(.*)\">/g;
#	print STDERR "$first_item\n";
#	print STDERR "$second_item\n";
#	
#	$sendString = "go=1&do=nw&ct=NA&1ex=1&1y=US&1aa=".$first_item."&1c=&1s=&1z=39759&1ah=&2ex=1&2y=US&2aa=".$second_item."&2c=&2s=&2z=39759&2ah=&lr=2&x=64&y=12";
#	
#	print "Second time printing sendString: [$sendString]\n";
#	
#	print "[$second[0]]*******\n";
#	$links = $second[0];
#	print "**[$links]*****\n";
#	$links =~ /href=\"main.adp\?([^\"]+)/g;
#	$links = $1;
#	print "*****[$links]*****\n";
#	
#	$req->content($sendString);
#	$res = $ua->request($req);
#	
#	print "$res->content";
#	
#	print STDERR "Similar address found\n"; 
    }
    #END RE-WRITE
    
    #parse_page still works pretty well
    #
    $count = &parse_web_map_page($local_file, $orig_start_name,
				 $orig_start_add, $orig_end_name,
				 $orig_end_add, $myval);
    return $count;
}

#------------------------------------------------------------------------------
#
#  Routine which parses the returned web page from mapquest for directions
#  information
#  The output is written to a file called "flatmap_%d".
#
#------------------------------------------------------------------------------

sub parse_web_map_page {
    
    my ($file, $start_name, $start_add, $end_name, $end_add, $myval) = @_;
    
    # remove all the html tag
    #
    #   foreach $rt ($file) {
    #       $rt=~ s/<(?:[^>'"]*|(['"]).*?\1)*>//gs; # '"])>
    #   }
    
    #print $file;
    #print "This is where the file would print to screen\n";
    
    my $orig_file = $file;
    
    my $time = $orig_file;
    $time =~ /Total Est. Time:(.*)/g;
    $time = $1;
    #print STDERR "Total Estimated Line is: [$time]\n";
    
    #NEW CODE
    # if hours is present, save both hours and minutes
    #
    my $hours = 0;
    my $minutes = 0;
    
    if ($time =~ /([0-9\.]*) hours/g) {
	$hours = $1;
    }
    if ($time =~ /([0-9\.]*) minutes/g) {
	$minutes = $1;
    }
    $time = $hours * 60 + $minutes;
    print STDERR "Total Estimated Time is: $time minutes\n";
    #END NEW CODE
    
    #ORIGINAL CODE
    #$time =~ /([0-9\.]*) minutes/g;
    #$time = $1;
    #print STDERR "Total Estimated Time is: $time minutes\n";
    #END ORIGINAL CODE
    
    # Find the total distance of the trip
    #
    my $distance = $orig_file;
    $distance =~ /Total Est. Distance:(.*)/g;
    $distance = $1;
    $distance =~ /([0-9\.]*) miles/g;
    $distance = $1;
    print STDERR "Total Estimated Distance is: $distance miles\n";
    
    # Find the location of the start location
    # This should be the same as $start_add but just to make sure we save it
    #
    my $from = $orig_file;
    $from =~ /alt="Start" \/>[\s]*(.*)/gs;
    $from = $1;
    $from =~ /<p><span>[\s](.*)<\/span><br \/>[\s](.*)/g;
    $from = $1;
    #print STDERR "From : [$from] [$2]\n";
    
    # Find the location of the end location
    # This should be the same as $end_add but just to make sure we save it
    #
    my $to = $orig_file;
    $to =~ /alt="End" \/>[\s]*(.*)/gs;
    $to = $1;
    $to =~ /<p><span>[\s](.*)<\/span><br \/>[\s](.*)/g;
    $to = $1;
    #print STDERR "To : [$to] [$2]\n";
    
    my $direction = $orig_file;
    
    # get all direction between Maneuvers and Map
    #
    $direction =~ /<td class="num"(.*)function openGoogle/gs;
    $direction = $1;
    #print STDERR "direction=[$direction]\n";
    
    @miles = $direction =~ /([0-9\.]* )miles/g;
    
    #DEBUG BEGIN
    #my $num = 0;
    #foreach $r (@miles) {
    #my $mile = $miles[$num];
    #print STDERR "[$num]=MILE=[$mile]\n";
    #$num++;
    #}
    #DEBUG END
    
    $direction = $orig_file;
    
    # get all the directions for this query
    # This will parse all of 
    #
    #<td class="num">1:</td>
    #<td>Start out going NORTH on MS-19 N toward BOLEN RD.</td>
    #
    # Into an array that contains all the directions
    #
    @directions = $direction =~ 
	/<td class="num">[0-9]+:<\/td>[\s]*<td[^>]*>([^<]*)/gs;
    
    my $count = 0;
    foreach $r (@directions) {
	
	$directions[$count] = trim($directions[$count]);
	
	# Have a 'miles' value for every direction
	# If the array isn't that big, make it that big and put 0 in it
	#
	if (!$miles[$count]) {
	    $miles[$count] = 0;
	}
	print STDERR "Step ".($count+1).
	    ": $directions[$count]  -- Miles: $miles[$count]\n";
	
	$count++;
    }
    
    #print STDERR "=======================================================\n";
    #print STDERR "directions:@directions\n\n";
    #print STDERR "*******************************************************\n";
    
    $outfile="$OUT_PATH/flatmap_$myval";
    open(DATA, ">$outfile") || die "ERROR: can't open $outfile.\n";
    
    print DATA "start_name          $start_name\n";
    
    if (length(start_add) == 0 ) {
	print DATA "start_add           $from\n";
    }
    else {
	print DATA "start_add           $start_add\n";
    }
    print DATA "start_interest      $start_interest\n";
    print DATA "start_city          $start_city\n";
    print DATA "start_state         $start_state\n";
    print DATA "start_zip           $start_zip\n";
    
    print DATA "end_name            $end_name\n";
    
    if (length(end_add) ==0 ) {
	print DATA "end_add             $to\n";
    }
    else {
	print DATA "end_add             $end_add\n";
    }
    
    print DATA "end_interest        $end_interest\n";
    print DATA "end_city            $end_city\n";
    print DATA "end_state           $end_state\n";
    print DATA "end_zip             $end_zip\n";
    
    print DATA "total_distance      $distance\n";
    print DATA "total_time          $time\n";
    print DATA "directions          $count\n";
    
    # This prints out two lines at a time. First line has the direction
    # and the second line prints out the distance of before direction
    # The previous thing printed named "directions $count" is the number
    # of pairs of lines that exist.
    #
    $num = 0;
    foreach $r (@directions) {
	my $direction = &trim($directions[$num]);
	my $mile = $miles[$num];
	print DATA "$direction\n";
	print DATA "$mile\n";
	$num++;
    }
    print DATA "\n";
    
    &status("received $count directions from mapquest");
    
    close DATA;
    
    if ($count == 0){
	@busy = $file =~ /System Busy/g;
	
	# $#array refers to the last populated element, or -1 if it is
	# empty. So rather than have to loop, just do a simple math op
	#
	$count = $count - ($#busy + 1);
	#foreach $c (@busy){
	#    $count = $count - 1;
	#}
	@busy = $file =~ /Unexpected Response/g;
	$count = $count - ($#busy + 1);
    }
    
    return $count;
}

