Admin/website/build/cl-1.0.1.pl

removed distinct, renamed gen_distinct to distinct;

#!/usr/local/bin/perl
#
#   Checklinks 1.0.1
#
#     Starting at one or more seed HTML files, recursively check the
#     validity of all links on the site.  Major features:
#
#       * Local URLs are read from the filesystem when possible (much
#           faster than going through HTTP server).
#       * Basic server-side includes (aka SSI or SHTML) are checked.
#       * Latest standards are supported-- HTML 4.0, HTTP 1.1, URIs
#           according to RFC 2396.
#       * Links are traversed breadth-first.
#
#   To list command-line options, run "cl -?" or see &usage() below.
#
#   TO CONFIGURE: 
#
#   1) Set $LOCAL_HOST and $DOCUMENT_ROOT, just below.  If you don't, the
#      program will try to guess them in set_needed_globals(), but it's more
#      reliable if you enter them here.
#
#   2) If needed, set any further server configuration below-- things like
#      path aliases and so forth.  If you have the srm.conf file, you can 
#      feed it into this script with "-c srm.conf"; otherwise, the default 
#      settings will probably work OK.
#
#   You can set a few parameters with the undocumented "-D <name=value>"
#   command-line option, e.g. "-D LOCAL_HOST=www.myhost.com".
#
#   Further comments, including an overview of script internals, are at
#   the end of this file.
#
#   Copyright (C) 1998, 2000 by James Marshall, [e-mail]
#   see http://www.jmarshall.com/tools/cl/ for more info
#
#
#   CHANGES IN 1.0.1:
#
#     This is just a bug fix release.  Fixes include:
#       . Aliases are handled correctly now.  Sorry 'bout that.
#       . A redirect + relative URL no longer results in infinitely
#           recursing URLs.
#       . More HTML tags are searched for links.
#       . Non-HTML files are no longer searched for links.
#       . There were other minor bug fixes.
#
#----------------------------------------------------------------------

#use strict ;

my( $LOCAL_HOST, $DOCUMENT_ROOT, $USER_DIR, @DIRECTORY_INDEX,
    %ALIAS, %ALIAS_MATCH, %SCRIPT_ALIAS, %SCRIPT_ALIAS_MATCH, %UN_ALIAS,
    @SHTML_EXTENSIONS, @CGI_EXTENSIONS, @INCLUDE_PATTERNS, @EXCLUDE_PATTERNS,
          @INCLUDE_STATUS, @EXCLUDE_STATUS,
    $verbose_report, $max_depth, $file_check, $full_http_check,
    $MAX_REDIRECTS, $MAX_ATTEMPTS, $HTML_BY_NAME, $SUPPORT_NCSA_BUG,
    @NO_PROXY, $DOC_ROOT_DEV, $DOC_ROOT_INODE, $DOC_ROOT_EXISTS, $CWD,
    %html_urls, %non_html_urls, %e_to_ch,

    %home_dir, %dir_to_user, %inode_to_user,

    %url, @urlstoget, 

    $debug, $CL_VERSION,
  ) ;


#----- User Configuration ---------------------------------------------

# This should be 'localhost', or a hostname of the Web server.  URLs at
#   this host will be assumed to be local; URLs not at this host will not be
#   traversed into. If this names a remote host, the program will not work.
# Note that 'localhost' doesn't necessarily point to your local Web server.

# $LOCAL_HOST= 'localhost' ;
# $LOCAL_HOST= 'www.example.com' ;
$LOCAL_HOST='isabelle.in.tum.de';

# This is your root Web directory, i.e. the directory that the Web server
#   sends the user if the URL "http://$LOCAL_HOST" is requested.  It's in
#   the configuration file srm.conf (and is read by -c option).
# If you don't know the document root of your server, but you don't need
#   it because you're only checking URLs whose path starts with ~, put a
#   non-existent path here rather than leave it blank (a hack).

# $DOCUMENT_ROOT= '/home/www/htdocs' ;
$DOCUMENT_ROOT='/home/proj/isabelle';

#----- variables equivalent to srm.conf entries 

# These globals are from the equivalent entries in srm.conf, etc.
# See the command-line option -c <config-file>, to read values directly 
#   from srm.conf instead.

# $USER_DIR= 'public_html' ;
$USER_DIR='.html-data';
@DIRECTORY_INDEX= qw( index.html index.cgi index.shtml ) ;

# Used in &url_to_filename(), and possibly elsewhere
# Note that ALIAS_MATCH and SCRIPT_ALIAS_MATCH use Perl (not standard) regexps.
# If order of multiple e.g. "Alias" directives is important, this may not work.
%ALIAS= () ;
%ALIAS_MATCH= () ;
%SCRIPT_ALIAS= () ;
%SCRIPT_ALIAS_MATCH= () ;

# The list of file extensions to interpret as CGI scripts or
#   server-parsed HTML files.
# These are not specific settings in srm.conf, but are combinations of
#   AddHandler directives and possibly AddType directives.
@CGI_EXTENSIONS=   qw( .cgi ) ;
@SHTML_EXTENSIONS= qw( .shtml ) ;

#----- end of variables equivalent to srm.conf entries 

# Specify patterns here to only include URLs that match at least one
#   pattern.  As a special case, an empty list includes all URLs, i.e.
#   does not restrict URLs by name (except perhaps by @EXCLUDE_PATTERNS).
# This can be added to or cleared with the -I command-line option.
@INCLUDE_PATTERNS= () ;

# Specify patterns here to cause matching URLs to be excluded,
#   e.g. '\?' means ignore all URLs that query.
# This can be added to or cleared with the -X command-line option.
# @EXCLUDE_PATTERNS=  qw( \? ) ;

# Only report URLs whose status codes start with one of these patterns.
#   As a special case, an empty list reports all URLs, i.e. does not 
#   restrict URLs by status code (except perhaps by @EXCLUDE_STATUS).
# This can be added to or cleared with the -i command-line option.
@INCLUDE_STATUS= () ;

# Don't report URLs whose status codes start with these patterns.  Default
#   is qw( 200 ).
# This can be added to or cleared with the -x command-line option.
@EXCLUDE_STATUS= qw( 200 ) ;

# For 302 or 303 HTTP redirection, redirect no more than this many times.
$MAX_REDIRECTS= 5 ;

# If a connection times out, etc., attempt no more than this many times.
$MAX_ATTEMPTS= 5 ;

# The old version determined whether a file was HTML by the -T test (text
#   file), and so traversed all HTML-like links in any text file that wasn't
#   a CGI script.  It's probably more appropriate to check the file
#   extension, to exclude source code, .txt files, etc.  Leave $HTML_BY_NAME
#   set to use the filename, or unset it to traverse all HTML-like links in
#   any text files, as the old version did.
$HTML_BY_NAME= 1 ;

# Some old NCSA servers, including 1.5.2, don't report the HTTP version
#   correctly in the status line; they return e.g. "HTTP 200 OK".  To allow
#   this, leave the variable here set.
$SUPPORT_NCSA_BUG= 1 ;


#----- DO NOT CHANGE ANYTHING BELOW THIS LINE, unless you want to... ---

#----- Further Global Variable Initialization --------------------------

$CL_VERSION= '1.0.1' ;

$ENV{'http_proxy'}||= $ENV{'HTTP_PROXY'} ;
@NO_PROXY= split(/[\s,]+/, $ENV{'no_proxy'} || $ENV{'NO_PROXY'} ) ;


# If output's not going directly to terminal, this ensures autoflushing.
$|= 1 ;

#----- End of Configuration --------------------------------------------

use strict 'vars' ;
use IO::Socket ;

&usage unless @ARGV ;

# Process command-line options
&getopts ;

# Make any final needed adjustments to globals, after the hard-coded
#   values above and any options have been processed.
&adjust_all_globals ;

# Default to "." if no starting filenames given.
# 3-6-98: Anh, decided against it.
#@ARGV= ('.') unless @ARGV ;

# &add_url() sets $url{$_} and pushes to @urlstoget, only if not already
#   added, plus any other initialization.
# Only add a file if it can be accessed with a URL.
foreach my $arg (@ARGV) {
    if ($arg=~ m#^http://#i) {
        &add_url($arg, '-', 0) ;
    } else {
        my($URL)= &filename_to_url($arg, $CWD) ;
        if (defined($URL)) {
            &add_url($URL, '-', 0) ;
        } else {
	    die "ERROR: $arg is not accessible through the Web server.\n" ;
        }
    }
}


# Check the URLs, in order.  @urlstoget may grow and rearrange.
while (@urlstoget) {
    my($url)= shift(@urlstoget) ;
    if ( !$url->{'ishtml'} or !$url->{'islocal'} or $url->{'dontfollow'}
         or (length($max_depth) and $url->{'depth'} > $max_depth ) ) {
        &verify_url($url) ;    # may set ishtml=true
    }
    if ( $url->{'ishtml'} and $url->{'islocal'} and !$url->{'dontfollow'}
         and (!length($max_depth) or $url->{'depth'} <= $max_depth ) ) {
        my($HTML)= &load_url($url) ;  # may set ishtml=false
        # 11-30-99 JSM: fixed to handle rel URLs in redirected pages correctly
        my($base_url)= $url->{'location'} || $url->{'URL'} ;
        &extract_urls($HTML, $base_url, $url->{'URL'}, $url->{'depth'}+1) 
            if $url->{'ishtml'} ;      # big, calls &add_url()
    }

    # If we get an error response that may be corrected with another
    #   attempt, put it back in the queue.  Such errors include 408,
    #   503, 504, and the homegrown codes 600, 601, 602, and 603.
    if ($url->{'status'}=~ /^(408|503|504|600|601|602|603)\b/ ) {
        push(@urlstoget, $url) if ( $url->{'numtries'} < $MAX_ATTEMPTS ) ;
    }

}

&make_report() ;

exit ;



#----- Process command-line options -----------------------------------

# Process any command-line options.
sub getopts {
    my($opt, $param) ;
    while ($ARGV[0]=~ /^-/) {
        $opt= shift(@ARGV) ;
        ($opt, $param)= $opt=~ /^-(.)(.*)/ ;

        # Turn on verbose reporting
        if ($opt eq 'v') {
            $verbose_report= ($param ne '-') ;

        # User-specified patterns to exclude ('' to clear list)
        } elsif ($opt eq 'I') {
            $param= shift(@ARGV) unless length($param) ;
            if (length($param)) { push(@INCLUDE_PATTERNS, $param) }
            else { @INCLUDE_PATTERNS= () }

        # User-specified patterns to exclude ('' to clear list)
        } elsif ($opt eq 'X') {
            $param= shift(@ARGV) unless length($param) ;
            if (length($param)) { push(@EXCLUDE_PATTERNS, $param) }
            else { @EXCLUDE_PATTERNS= () }

        # User-specified response codes to ignore ('' to clear list)
        } elsif ($opt eq 'i') {
            $param= shift(@ARGV) unless length($param) ;
            if (length($param)) { push(@INCLUDE_STATUS, $param) }
            else { @INCLUDE_STATUS= () }

        # User-specified response codes to ignore ('' to clear list)
        } elsif ($opt eq 'x') {
            $param= shift(@ARGV) unless length($param) ;
            if (length($param)) { push(@EXCLUDE_STATUS, $param) }
            else { @EXCLUDE_STATUS= () }

        # Maximum traversal depth
        } elsif ($opt eq 'd') {
            $param= shift(@ARGV) unless length($param) ;
            $max_depth= $param ;

        # Make it a "file check"-- only read local files, do not use HTTP
        } elsif ($opt eq 'f') {
            $file_check= ($param ne '-') ;

        # Use HTTP for all URL's, even local files
        } elsif ($opt eq 'h') {
            $full_http_check= ($param ne '-') ;

        # Read configuration parameters from srm.conf-like file
        } elsif ($opt eq 'c') {
            $param= shift(@ARGV) unless length($param) ;
            &read_srm_conf($param) ;
            
        # Print current configuration parameters
        } elsif ($opt eq 'q') {
            &print_config ;
            exit ;   # jsm-- should we exit?

        # Allow certain parameters to be defined via the command line
        } elsif ($opt eq 'D') {
            $param= shift(@ARGV) unless length($param) ;
            $debug=1, unshift(@ARGV,$param), next if $param=~ /^-/ ;
            my($name,$value)= split(/=/, $param, 2) ;
            $value= 1 unless length($value) ;
            if ($name=~ /^(LOCAL_HOST|DOCUMENT_ROOT|USER_DIR|DEBUG|debug)$/) {
                eval "\$$name= \$value" ;
                #$$name= $value ;  # this doesn't work, because of initial my()
            }

        } elsif ($opt eq '?') {
            &usage ;

        # End command-line option processing on "--"
        } elsif ($opt eq '-') {
            return ;

        } else {
            print STDERR 
                "Illegal option-- '$opt'.  Enter \"$0 -?\" for help.\n" ;
            exit ;
        }

    }

    if ($file_check and $full_http_check) {
        print STDERR "You cannot use both the -f and the -h options.\n" ;
        exit ;
    }

}


# Read appropriate values from the given file, typically srm.conf.  If a
#   directory is named, default to filename "srm.conf".
# Note that opening "-" will open STDIN.
sub read_srm_conf {
    my($fname)= @_ ;
    local(*SRM) ;

    # default to srm.conf if only a directory is named
    if (-d $fname) {
        $fname=~ s#/$## ;
        $fname.= "/srm.conf" ;
    }

    # Clear old values
    $DOCUMENT_ROOT= $USER_DIR= '' ;
    @DIRECTORY_INDEX= @CGI_EXTENSIONS= @SHTML_EXTENSIONS= () ;
    %ALIAS= %ALIAS_MATCH= %SCRIPT_ALIAS= %SCRIPT_ALIAS_MATCH= () ;

    open(SRM, "<$fname") || die "Can't open $fname: $!" ;
    while (<SRM>) {
        s/#.*// ;
        next unless /\S/ ;
        my($name, @param)= /(\S+)/g ;

        if ($name eq 'DocumentRoot') {
            $DOCUMENT_ROOT= $param[0] ;

        } elsif ($name eq 'UserDir') {
            $USER_DIR= $param[0] ;

        } elsif ($name eq 'DirectoryIndex') {
            @DIRECTORY_INDEX= @param ;

        } elsif ($name eq 'Alias') {
            $ALIAS{$param[0]}= $param[1] ;

        } elsif ($name eq 'AliasMatch') {
            $ALIAS_MATCH{$param[0]}= $param[1] ;

        } elsif ($name eq 'ScriptAlias') {
            $SCRIPT_ALIAS{$param[0]}= $param[1] ;

        } elsif ($name eq 'ScriptAliasMatch') {
            $SCRIPT_ALIAS_MATCH{$param[0]}= $param[1] ;

        } elsif ($name eq 'AddHandler') {
            if ($param[0] eq 'cgi-script') {
                push(@CGI_EXTENSIONS, $param[1]) ;
            } elsif ($param[0] eq 'server-parsed') {
                push(@SHTML_EXTENSIONS, $param[1]) ;
            }
        }

    }
    close(SRM) ;
}


# Make any final settings to global variables, after the hard-coded values
#   and command-line options have been processed.
# Most non-user-configurable globals are also set here.
sub adjust_all_globals {

    # Standardize $USER_DIR to never have trailing slash
    $USER_DIR=~ s#/$## ;

    # If no $LOCAL_HOST set, try to read it from first URL in list, or
    #   use the string 'localhost' if that URL contains no hostname.
    unless (length($LOCAL_HOST)) {
        $LOCAL_HOST= (&parse_url($ARGV[0]))[1] || 'localhost' ;
        print STDERR "LOCAL_HOST set to \"\L$LOCAL_HOST\E\"\n" ;
    }
    $LOCAL_HOST= lc($LOCAL_HOST) ;

    # If no $DOCUMENT_ROOT, try to guess it from $HOME, username, $USER_DIR.
    unless (length($DOCUMENT_ROOT)) {
        my($home) ;
        unless ($home= $ENV{'HOME'}) {
            my($uname)= getpwuid($<) || $ENV{'USER'} || `whoami` || `id -un` ;
            chomp($uname) ;
            &read_home_dirs unless %home_dir ;   # only read when needed
            $home= $home_dir{$uname} ;
        }
        $DOCUMENT_ROOT= "$home/$USER_DIR" ;

        die "Could not determine DOCUMENT_ROOT; edit the $0 script to set it.\n"
            unless (-d $DOCUMENT_ROOT) ;

        print STDERR "DOCUMENT_ROOT set to \"$DOCUMENT_ROOT\"\n" ;
    }
    $DOCUMENT_ROOT=~ s#/$## ;

    # Allows &filename_to_url() to unalias as best as possible.  Note that 
    #   use of &filename_to_url() can be avoided by the user; see note in 
    #   that routine.
    %UN_ALIAS= (reverse (%ALIAS, %SCRIPT_ALIAS) ) ;

    # These are to compare equivalency to later, in &filename_to_url().
    ($DOC_ROOT_DEV, $DOC_ROOT_INODE)= stat("$DOCUMENT_ROOT/.") ;
    
    $DOC_ROOT_EXISTS= -e _ ;
    
    # Set CWD from shell variable, else from `pwd`.
    $CWD= $ENV{'PWD'} || `pwd` || die "couldn't run pwd: $!" ;
    chomp($CWD) ;


    # These are used by &extract_urls().
    # This is a complete list of URL-type attributes defined in HTML 4.0,
    #   plus any others I found, like nonstandard ones or from an earlier HTML.
    # Only a few of these are commonly used, as of early 1998.
    # The set in %html_urls could possibly link to HTML resources, while the
    #   set in %non_html_urls could not.  The %special(.*) sets, here for 
    #   reference only, include URL attributes that require special handling.

    %html_urls= ( 'a'          => [ 'href' ],
                  'area'       => [ 'href' ],
                  'frame'      => [ 'src', 'longdesc' ],
                  'link'       => [ 'href', 'urn' ],
                  'img'        => [ 'longdesc', 'usemap' ],
                  'q'          => [ 'cite' ],
                  'blockquote' => [ 'cite' ],
                  'ins'        => [ 'cite' ],
                  'del'        => [ 'cite' ],
                  'object'     => [ 'usemap' ],
                  'input'      => [ 'usemap' ],
                  'iframe'     => [ 'src', 'longdesc' ],
		  'ilayer'     => [ 'src' ],
		  'layer'      => [ 'src' ],
		  'fig'        => [ 'imagemap' ],
		  'overlay'    => [ 'imagemap' ],
		  'meta'       => [ 'url' ],
		  'note'       => [ 'src' ],
                ) ;

    %non_html_urls= ( 'body'    => [ 'background' ], 
                      'img'     => [ 'src', 'lowsrc', 'dynsrc' ],
                      'input'   => [ 'src' ],
                      'script'  => [ 'src', 'for' ],

		      'fig'     => [ 'src' ],
		      'overlay' => [ 'src' ],
		      'select'  => [ 'src' ],
		      'ul'      => [ 'src' ],
		      'h1'      => [ 'src' ],
		      'h2'      => [ 'src' ],
		      'h3'      => [ 'src' ],
		      'h4'      => [ 'src' ],
		      'h5'      => [ 'src' ],
		      'h6'      => [ 'src' ],
		      'hr'      => [ 'src' ],
		      'table'   => [ 'src' ],
		      'td'      => [ 'src' ],
		      'th'      => [ 'src' ],
		      'tr'      => [ 'src' ],

		      'bgsound' => [ 'src' ],
		      'embed'   => [ 'src' ],
                  ) ;

    # %special_urls= ( 'base' => [ 'href' ] ) ;
    #
    # %special_html_urls= ( 'object' => [ 'codebase', 'data' ] ) ;
    #
    # %special_non_html_urls=
    #      ( 'head'   => [ 'profile' ],
    #        'object' => [ 'codebase', 'archive', 'classid' ],
    #        'applet' => [ 'codebase', 'code', 'object', 'archive' ],
    #        'form'   => [ 'action', 'script' ]
    #      ) ;


    # This is a translation from entity character references to characters, 
    #   used in &HTMLunescape().
    # This simplified version only supports &quot; &lt; &gt; &amp;, but that
    #   should be enough for URL-type attributes.
    # See http://www.w3.org/TR/REC-html40/sgml/entities.html for full entity 
    #   list.

    %e_to_ch= (quot => '"', 
               'lt' => '<', 
               'gt' => '>', 
               amp  => '&') ;

}


#----------------------------------------------------------------------

# Add the URL to our data structures; specifically, to %url and @urlstoget.
# Returns a pointer to the structure in %url, or undef if already defined
#   or on error.
# Currently, this always receives the URL with the host name lowercase,
#   either from &absolute_url() or from using $LOCAL_HOST.
sub add_url {
    my($URL, $referer, $depth, $ishtml, $iscgi, $dontfollow)= @_ ;

    # Allow the user to restrict URL patterns:  URLs must be in 
    #   @INCLUDE_PATTERNS but not in @EXCLUDE_PATTERNS (but only restrict 
    #   by @INCLUDE_PATTERNS if it's not empty).
    return undef if @INCLUDE_PATTERNS &&
                    !grep( $URL=~ /$_/, @INCLUDE_PATTERNS ) ;
    return undef if grep( $URL=~ /$_/, @EXCLUDE_PATTERNS ) ;

    # Canonicalize URL, so we don't get a page multiple times
    $URL= &canonicalize($URL) ;
    
    # for obscure case involving a <form action=___.cgi>-extracted URL being
    #   overwritten by <a href=___.cgi> extraction (don't fret over this)
    $url{$URL}{'dontfollow'}&&= $dontfollow if $url{$URL} ;

    # Don't add the record a second time!  Or will infinitely traverse.
    return undef if $url{$URL} ;  # or add to @referers, for 301 correction...?

    # Only HTTP URLs are currently supported
    return undef unless $URL=~ /^http:/i ;

    # Any self-referral here indicates a bug in the program.  It's happened.
    die "PROGRAM ERROR: $URL shows its first referer as itself.\n"
        if $referer eq $URL ;

    my(%u) ;
    @u{qw(URL referer depth ishtml iscgi dontfollow)}= 
        ($URL, $referer, $depth, $ishtml, $iscgi, $dontfollow) ;
    $u{'islocal'}= ($URL=~ m#^http://\Q$LOCAL_HOST\E/#io) + 0 ; # make length>0
    if ($u{'islocal'}) {
#        $u{'filename'}= &url_to_filename($URL) ;
        @u{'filename', 'location'}= &url_to_filename($URL) ;
        $u{'iscgi'}= &is_cgi($u{'filename'}, $URL)  if $u{'iscgi'} eq '' ;

	# 2-27-00 JSM:  Detect ishtml by filename, not -T test.
	if ( $u{'ishtml'} eq '' ) {
	    $u{'ishtml'}=  $HTML_BY_NAME
		?  ( !$u{'iscgi'} && -e $u{'filename'} && 
		     $u{'filename'}=~ /\.html?$/i ) + 0
		:  (!$u{'iscgi'} && -e $u{'filename'} && -T _) + 0 ;
	}
#        $u{'ishtml'}= (!$u{'iscgi'} && -e $u{'filename'} && -T _) + 0
#            unless length($u{'ishtml'}) ;

    }

    #  If we're only doing a file check, don't add URLs that require HTTP
    return undef if ($file_check and (!$u{'islocal'} or $u{'iscgi'}) ) ;

    push(@urlstoget, \%u) ;
    $url{$URL}= \%u ;

    # return \%u ;   # unneeded because of previous statement
}


# Guess if a file is a CGI script or not.  Returns true if the (regular) file
#   is executable, has one of @CGI_EXTENSIONS, or if the URL is in a 
#   ScriptAlias'ed directory.
# $fname must be absolute path, but $URL is optional (saves time if available).
# Note that URLs like "/path/script.cgi?a=b" are handled correctly-- the
#   previously extracted filename is tested for CGI-ness, while the URL is
#   checked for ScriptAlias matching (which is unaffected by final query
#   strings or PATH_INFO).
sub is_cgi {
    my($fname, $URL)= @_ ;
    return 1 if (-x $fname && ! -d _ ) ;   # should we really do this?
    foreach (@CGI_EXTENSIONS) { return 1 if $fname=~ /\Q$_\E$/i }

    $URL= &filename_to_url($fname) unless length($URL) ;  # currently unused
    my($URLpath)= $URL=~ m#^http://[^/]*(.*)#i ;
    foreach (keys %SCRIPT_ALIAS)        { return 1 if $URLpath=~ /^\Q$_\E/ }
    foreach (keys %SCRIPT_ALIAS_MATCH)  { return 1 if $URLpath=~ /^$_/ }

    return 0 ;
}
   

# Put the URL in such a form that two URLs that point to the same resource
#   have the same URL, to avoid superfluous retrievals.
# Host name is lowercased elsewhere-- this routine is only called from
#   &add_url; see note there.  To lowercase the host name here would be
#   inefficient.
sub canonicalize {
    my($URL)= @_ ;

    $URL=~ s/#.*// ;    # remove any "#" fragment from end of URL

    return $URL ;
}


#----- File reading/downloading routines (includes networking) --------

# Verify that a URL exists, and set $url->{'status'} accordingly.  Do
#   this either by checking the local filesystem or by using the HTTP HEAD 
#   method for remote sites or CGI scripts.
# Set $url->{'ishtml'} accordingly if discovered from Content-Type:.
# This does not support various Redirect directives in srm.conf.
sub verify_url {
    my($url)= @_ ;

    print STDERR "verifying $url->{'URL'}\n" if $debug ;


    # Depending on the state of $url->{islocal, iscgi, dontfollow} and
    #   $full_http_check, take appropriate actions to check/set the
    #   status code for this URL.
    
    # NOTE: In some situations, specifically when checking a CGI script
    #   named in a <form action> (thus implying that dontfollow is set),
    #   and using HTTP to check the URL (because the script is remote or
    #   $full_http_check is set), the HTTP response code may not be
    #   accurate.  This is because there is no form data sent with the
    #   request, as there normally would be.  In these cases, a cautionary
    #   note is appended to $url->{'status'}.  Additionally, an empty 
    #   $url->{'status'} is changed to an explanatory note (maybe we should
    #   do that in load_url() too?).

    # Use HEAD if file is remote, or if $full_http_check is set.
    if (!$url->{'islocal'} or $full_http_check) {
        &load_url_using_HTTP($url, 'HEAD') ;
        $url->{'status'}= '[no status returned]'
            unless length($url->{'status'}) ;
        $url->{'status'}.= ' (NOTE: Form was not submitted normally)'
            if $url->{'dontfollow'} ;

    # URL is local:  If it's not CGI, do a normal local file check
    } elsif (!$url->{'iscgi'}) {
        $url->{'status'}= (-e $url->{'filename'})  
            ? "200 Local File Exists"  : "404 File Not Found" ;

    # URL is local CGI:  Use HEAD unless dontfollow is set
    } elsif (!$url->{'dontfollow'}) {
        &load_url_using_HTTP($url, 'HEAD') ;

    # Else it's a local CGI with dontfollow set:  Check for executable file
    } else {
        $url->{'status'}= 
             (! -e $url->{'filename'})  ? "404 File Not Found"
           : (! -x $url->{'filename'})  ? "403 Local File Is Not Executable"
           :                              "200 Local Executable File Exists"

    }
        

# Old verify routine below:
#
#    # If is a local non-CGI file, check it directly from the filesystem
#    if ($url->{'islocal'} and !$url->{'iscgi'} and !$full_http_check) {
#        $url->{'status'}= (-e $url->{'filename'})  
#            ? "200 Local File Exists"  : "404 File Not Found" ;
#
#    # Otherwise, download its HEAD from its HTTP server
#    } else {
#        &load_url_using_HTTP($url, 'HEAD') ;
#    }


}



# Load entire file/resource and return its contents, setting $url->{'status'}
#    accordingly.  Do this either by checking the local filesystem or by 
#    using the HTTP GET method for remote sites or CGI scripts.
# Set $url->{'ishtml'} accordingly if discovered from Content-Type:.
# This does not support various Redirect directives in srm.conf.
sub load_url {
    my($url)= @_ ;
    my($HTML) ;

    print STDERR "loading $url->{'URL'}\n" if $debug ;

    # If is a local non-CGI file, read it directly from the filesystem
    if ($url->{'islocal'} and !$url->{'iscgi'} and !$full_http_check) {
        my($iscgi) ;
        ($HTML, $url->{'ssierrs'}, $iscgi)= 
            &read_expanded_file($url->{'filename'}, $url->{'URL'}) ;
        $url->{'status'}= 
            !defined($HTML)
                  ? sprintf("450 Can't read file: %s (%s)", $!, $!+0)
            : @{$url->{'ssierrs'}}
                  ? sprintf("451 SSI Error(s) (%s total)",
                            scalar @{$url->{'ssierrs'}})

            :       "200 Local File Read OK" ;

        # $url->{'iscgi'} may be set if an SHTML file included CGI calls.
        # Don't set it if we're doing a file check, in which case we'll
        #   keep whatever $HTML we could get.
        $url->{'iscgi'}= $iscgi unless $file_check ;
    }

    # Otherwise (or if rereckoned), download the resource from its HTTP server
    if (!$url->{'islocal'} or $url->{'iscgi'} or $full_http_check) {
        (undef, undef, $HTML)= &load_url_using_HTTP($url, 'GET') ;
    }

    # Note that this will be set even when URL is to be reloaded, like
    #   for a 601 (timeout) response.
    $url->{'hasbeenloaded'}= 1 ;
    return $HTML ;
}


# Read a local file and return its contents.  If a file is SSI (aka SHTML),
#   expand any SSI <!--#include--> directives as needed, recursively 
#   including nested files.
# This is used for all local reads, SHTML or not, but the vast bulk of this
#   routine is for SHTML files.
#
# If file is SHTML, this routine also returns a structure of error data,
#   and a boolean saying if this file needs to be downloaded via HTTP
#   for a complete check (e.g. includes CGI calls).
#
# $fname must be canonicalized absolute path, but $URL parameter is optional.
# %$parents contains all "include"-ancestors of the file, to prevent loops.
#   If omitted, assumes no ancestors (and a fresh hash is started).
#
# This routine seems much bigger and more complex than it needs to be.
#   It could be one third the size and much simpler if we didn't have to
#   worry about full error reporting on nested includes.
#
# Note: This routine was made to mimic what Apache would return to a client.
#   However, the result differs from Apache's in two slight ways, both
#   involving nested SSI within <!--#include file="..." -->, and both
#   apparent bugs in Apache 1.1 (may be fixed in later versions):
#
#   1) If a <file="..."> value contains no "/" (i.e. in current directory),
#      then Apache always parses the included file as SHTML, regardless of
#      extension.  This routine checks @SHTML_EXTENSIONS for all included
#      files.
#   2) If a <file="..."> value containing a "/" loads an SHTML file  
#      containing a <virtual="..."> tag with a relative path, the directive
#      fails in Apache.  This routine tries to guess the correct path/URL.
#
#
# Notes on this routine, and SHTML files in general:
#
# At first thought, it seems like we could load each included file
# only once, instead of once for every file that includes it.
# However, because of the fact that relative URLs are resolved
# relative to the top-level including file, the top-level file will
# need to be expanded every time.  (It's legal (though of questionable
# wisdom) to include a file from e.g. both /a/index.shtml and
# /b/index.shtml, so links from the included file point to different
# URLs.)
#
# Note that while URLs in included files (e.g. <a href="...">) are
# resolved relative to the top-level including file, nested include tags
# are resolved relative to the direct includer.
#
# We could possibly be more efficient in time (but costly in memory)
# by storing the expanded contents and $errlist of each included file,
# since those will be constant (except $errlist's include-loop
# reporting might vary somewhat).  There are probably other ways to
# eek out savings of time and memory, at the cost of complexity.
#
# The main loop here is inside of an s/// statement.  Unusual, but it's an
# appropriate way to handle the recursion. Recursion is needed, since each
# included file may or may not be SHTML.
#
# $iscgi is set if a file includes "<!--#exec", or if it contains an
#   <!--#include virtual="..." --> tag that points to a CGI file, or if 
#   any of its include-children sets $iscgi.
#
#
# Notes to help clarify data structures, if (God forbid) you have to modify 
# this routine:
#
# Each error is a list of files in an "include chain", and $errlist is a
# list of errors.  $errlist is associated with the current $HTML.  Each
# error in $errlist is associated with some tag in $HTML, as iterated in
# the s/// loop.  When this routine returns ($HTML, $errlist), the
# errors in $errlist should all have as their first element tags that
# were found in $HTML.
#
# Each error's final element (the "leaf") is associated with a tag that
# tried to load an invalid file.  Each leaf will always be the complete 
# set of errors for that tag (i.e. it has no children, since it couldn't
# load the file).
#
# If the file can be validly loaded, then we may have 0 or multiple errors
# associated with this file/tag (and returned from this routine).  Each
# file's $errlist is an accumulation of all of its children's $errlist's.
#
# Errors that come from loading a child are associated with the child's
# $HTML and tags.  Before adding them to the parent's (current) $errlist,
# they must have the CHILD's path/tag unshifted onto the front of the 
# include chain; all errors are then added to the current $errlist.
# This ensures that:
#   a) The errors are now all associated with the tag that loaded the child.
#   b) The errors in the current $errlist are always associated with tags
#      from the current $HTML.
#

sub read_expanded_file {
    my($fname, $URL, $parents)= @_ ;
    my($HTML, $errlist, $iscgi) ;
    my($isshtml) ;

    $parents->{$fname}= 1 ;

    $HTML= &read_file($fname) ;
    return(undef) unless defined($HTML) ;

    foreach (@SHTML_EXTENSIONS) {
        $isshtml= 1, last if ($fname=~ /\Q$_\E$/i) ;
    }

    if ($isshtml) {
        $errlist= [] ;
        $iscgi= ($HTML=~ /<!--#exec\b/i) ;

        $HTML=~ s{(<!--#include\b.*?>)} {
            do {
                my($childfname, $childURL) ;
                my($childHTML, $childerrlist, $childiscgi) ;
                my($tagst) = $1 ;
                my($tag)= &parse_tag($tagst) ;
              GET_CHILD: {
                  if (length($tag->{'attrib'}{'virtual'})) {
                      $URL= &filename_to_url($fname) unless length($URL) ;
                      $childURL= 
                          &absolute_url($tag->{'attrib'}{'virtual'}, $URL) ;
                      ($childfname)= &url_to_filename($childURL) ;

                      # If it's a CGI, don't follow it, but no error either
                      if (&is_cgi($childfname, $childURL)) {
                          $iscgi= 1 ;
                          last GET_CHILD ;
                      }

                  } elsif (length($tag->{'attrib'}{'file'})) {
                      $childfname= $tag->{'attrib'}{'file'} ;
                      if ($childfname=~ m#^(/|~)#) {
                          push(@$errlist, [ {'path' => $childfname, 
                                             'tag' => $tagst, 'errmsg' =>
                                  'Absolute paths are not allowed in '
                                . '<!--#include file="..." -->.'}]);
                          last GET_CHILD ;
                      }
                      if ($childfname=~ m#\.\.(/|$)#) {
                          push(@$errlist, [ {'path' => $childfname, 
                                             'tag' => $tagst, 'errmsg' =>
                                  'Paths can not contain "../" in '
                                . '<!--#include file="..." -->.'}]);
                          last GET_CHILD ;
                      }
                      $childfname= ($fname=~ m#(.*/)#)[0] . $childfname ;

                  } else {
                      push(@$errlist, [ {'path' => '',
                                         'tag' => $tagst, 'errmsg' =>
                              'Tag must contain either the "file" or '
                            . '"virtual" attribute.'}]);
                      last GET_CHILD ;

                  }

                  # canonicalize filename for %$parents
                  1 while $childfname=~ s#/\.(/|$)#/# ;
                  1 while $childfname=~ s#/(?!\.\./)[^/]+/\.\.(/|$)#/# ;

                  # Guarantee that file exists, is regular, and is readable
                  unless (-e $childfname) {
                      push(@$errlist, [{'path' => $childfname, 'tag' => $tagst,
                                        'errmsg' => 'File not found'} ] ) ;
                      last GET_CHILD ;
                  }
                  unless (-f $childfname) {
                      push(@$errlist, [{'path' => $childfname, 'tag' => $tagst,
                                        'errmsg' => 'File is not a regular'
                                            . ' file.' } ] ) ;
                      last GET_CHILD ;
                  }
                  unless (-r $childfname) {
                      push(@$errlist, [{'path' => $childfname, 'tag' => $tagst,
                                        'errmsg' => 'File is not readable by'
                                            . ' current user.' } ] ) ;
                      last GET_CHILD ;
                  }

                  # Guard against include loops
                  if ($parents->{$childfname}) {
                      push(@$errlist, [{'path' => $childfname, 'tag' => $tagst,
                                        'errmsg' => 'An "include" loop exists'
                                            . ' involving this file.' } ] ) ;
                      last GET_CHILD ;
                  }


                  # Get the included file, with any error data
                  ($childHTML, $childerrlist, $childiscgi)= 
                      &read_expanded_file($childfname, $childURL, $parents) ;

                  # Log if there was any error reading the file
                  push(@$errlist, [{'path' => $childfname, 'tag' => $tagst,
                                    'errmsg' => "Can't read file: $!." } ] )
                      unless defined($childHTML) ;

                  # Add any errors to the current (parent) error list
                  foreach my $error (@$childerrlist) {
                      unshift(@$error, 
                              { 'path' => $childfname, 'tag' => $tagst } ) ;
                  }
                  push(@$errlist, @$childerrlist) ;

                  # Parent is a CGI if any of its children is a CGI
                  $iscgi||= $childiscgi ;
            
              }   # GET_CHILD


              $childHTML ;   # final value to replace in main s/// construct

          }   # do {}

        }gie ;   # $HTML=~ s{} {}

    } # if ($isshtml)

    delete $parents->{$fname} ;

    return($HTML, $errlist, $iscgi) ;
}



# Returns the contents of the named file, or undef on error.
sub read_file {
    my($fname)= @_ ;
    local(*F, $/) ;
 
    undef $/ ;
    open(F, "<$fname") || return undef ;
    my($ret)= <F> ;
    close(F) ;
    
    return $ret ;
}


# Try to get the given URL with the given HTTP method, and return the
#   status line, headers, and body.
# Set $url->{'status'} accordingly, and set $url->{'ishtml'} accordingly
#   if Content-Type: header is returned.
# This is specific to this program, and calls the more general &get_url().
# This could be slightly more efficient if 302 or 303 was handled in the
#   calling routine, where it could take advantage of a new URL being local.
sub load_url_using_HTTP {
    my($url, $method)= @_ ;
    my($status_line, $headers, $body) ;

    # We should not get here if $file_check is set
    die "mistakenly called load_url_using_HTTP($url->{'URL'})" if $file_check ;

    GETFILE: {
        ($status_line, $headers, $body)=
            &get_url( ($url->{'location'} || $url->{'URL'}), $method) ;

        # If HEAD failed (as on some servers), sigh and use GET
        ($status_line, $headers, $body)=
            &get_url( ($url->{'location'} || $url->{'URL'}), 'GET')
                unless length($status_line) ;

        ($url->{'status'})=  $status_line=~ m#^HTTP/[\d.]+\s+(.*)# ;

	# 2-27-00 JSM:  Allow old NCSA servers to not include the HTTP version.
	if ($SUPPORT_NCSA_BUG and $url->{'status'} eq '') {
	    ($url->{'status'})=  $status_line=~ m#^HTTP(?:/[\d.]+)?\s+(.*)# ;
	}

        # Redirect to new location if status is 302 or 303
        if ($url->{'status'}=~ /^(301|302|303)\b/) {
            ($url->{'location'})= $headers=~ m#^Location:[ \t]+(\S+)#im ;
            last GETFILE unless length($url->{'location'}) ;
            $url->{'location'}= 
                &absolute_url($url->{'location'}, $url->{'URL'}) ;
            redo GETFILE
                if    ($url->{'status'}=~ /^(302|303)\b/)
                   && (++$url->{'numredirects'} <= $MAX_REDIRECTS) ;
        }
    }

    $url->{'numtries'}++ ;
    $url->{'lasttried'}= time ;

    # If successful response included Content-Type:, set ishtml accordingly
    $url->{'ishtml'}= (lc($1) eq 'text/html') + 0
        if $url->{'status'}=~ /^2/
           and $headers=~ m#^content-type:[ \t]*(\S+)#im ;

    print STDERR "status: $status_line\n" if $debug ;

    return($status_line, $headers, $body) ;
}


# Request the HTTP resource at the given absolute URL using the given method,
#   and return the response status line, headers, and body.
# jsm-- in the future, this should support downloading to a file, in case
#   the download is too large to fit in memory.
sub get_url {
    my($URL, $method)= @_ ;
    my($host, $uri, $endhost) ;
    my($S, $rin) ;
    my($response, $status_line, $headers, $body, $status_code) ;
    my($content_length) ;
    $method= uc($method) ;
    $method= 'GET' unless length($method) ;

    ($host, $uri)= $URL=~ m#^http://([^/]*)(.*)$#i ;
    $uri= '/' unless length($uri) ;
    $endhost= $host ;

    # use an HTTP proxy if $ENV{'http_proxy'} is set
    USEPROXY: {
        last USEPROXY unless $host=~ /\./ ;
        if (length($ENV{'http_proxy'})) {
            foreach (@NO_PROXY) {
                last USEPROXY if $host=~ /$_$/i ;
            }
            ($host)= $ENV{'http_proxy'}=~ m#^(?:http://)?([^/]*)#i ;
            $uri= $URL ;
        }
    }

    # Open socket
    $S= IO::Socket::INET->new(PeerAddr => $host,  # may contain :port
                              PeerPort => 80,     # default if none in PeerAddr
                              Proto => 'tcp') ;
    return("HTTP/1.1 600 Can't create socket: $@") unless defined($S) ;
    $S->autoflush() ;   # very important!!

    # Send HTTP 1.1 request
    print $S "$method $uri HTTP/1.1\015\012",
             "Host: $endhost\015\012",
             "Connection: close\015\012",
             "User-agent: CheckLinks/$CL_VERSION\015\012",
             "\015\012" ;

    # Wait for socket response with select()
    vec($rin= '', fileno($S), 1)= 1 ;
    select($rin, undef, undef, 60) 
        || return("HTTP/1.1 601 Connection timed out") ;

    local($/)= "\012" ;

    # Handle "100 Continue" responses for HTTP 1.1: loop until non-1xx.
    do {
        $status_line= <$S> ;
        $status_line=~ s/\015?\012$// ;
        ($status_code)= $status_line=~ m#^HTTP/\d+\.\d+\s+(\d+)# ;

        $headers= '' ;
        while (<$S>) {
            last if /^\015?\012/ ;
            $headers.= $_ ;
        }
        $headers=~ s/\015?\012[ \t]+/ /g ;
    } until $status_code!~ /^1/ ;

    # Body length is determined by HTTP 1.1 spec, section 4.4:  these
    #   certain conditions implying no body, then chunked encoding,
    #   then Content-length: header, then server closing connection.
    if ($method eq 'HEAD' or $status_code=~ /^(1|204\b|304\b)/) {
        $body= undef ;

    # else chunked encoding
    } elsif ($headers=~ /^transfer-encoding:[ \t]*chunked\b/im) {
        # 7-16-99:  Old code was only saving last chunk.  Fix using
        #   $this_chunk contributed by Mark Trotter.
        my($this_chunk, $chunk_size, $readsofar, $thisread) ;
        while ($chunk_size= hex(<$S>)) {
            $readsofar= 0 ;
            while ($readsofar!=$chunk_size) {
                last unless $thisread=
                    read($S, $this_chunk, $chunk_size-$readsofar, $readsofar) ;
                $readsofar+= $thisread ;
            }
            return("HTTP/1.1 603 Incomplete chunked response", $headers, $body)
                if $readsofar!=$chunk_size ;
            $_= <$S> ;    # clear CRLF after chunk
            $body.= $this_chunk ;
        }

        # Read footers if they exist
        while (<$S>) {
            last if /^\015?\012/ ;
            $headers.= $_ ;
        }
        $headers=~ s/\015?\012[ \t]+/ /g ;


    # else body length given in Content-length:
    } elsif (($content_length)= $headers=~ /^content-length:[ \t]*(\d+)/im) {
        my($readsofar, $thisread) ;
        while ($readsofar!=$content_length) {
            last unless $thisread=
                read($S, $body, $content_length-$readsofar, $readsofar) ;
            $readsofar+= $thisread ;
        }
        return(sprintf("HTTP/1.1 602 Incomplete response (%s of %s bytes)",
                       $readsofar+0, $content_length),
               $headers, $body)
            if $readsofar!=$content_length ;


    # else body is entire socket output
    } else {
        local($/)= undef ;
        $body= <$S> ;
    }

    close($S) ;

    return($status_line, $headers, $body) ;
}


#----- URL-parsing routines -------------------------------------------

# The routines parse_url(), unparse_url(), and absolute_url() are based on
#   different sections in the Internet Draft "Uniform Resource Identifiers
#   (URI): Generic Syntax and Semantics", 11-18-97, by Berners-Lee,
#   Fielding, and Masinter, filename draft-fielding-uri-syntax-01.

# Parse a URL into its components, according to URI draft, sections 4.3, 4.4.
# This regular expression is straight from Appendix B, modified to use Perl 5.
# Returns scheme, site, path, query, and fragment.  All but path may have
#   the undefined value.
sub parse_url {
    my($URL)= @_ ;
    my($scheme, $site, $path, $query, $fragment)=
        ($URL=~ m{^(?:    ([^:/?\#]+):)?
                   (?: // ([^/?\#]*))?
                          ([^?\#]*)
                   (?: \? ([^\#]*))?
                   (?: \# (.*))?
                 }x
        ) ;
        
        
    # Un-URL-encode the path, to equivalate things like %7E --> ~
    # Note that in some situations, this may cause problems with URLs that
    #   contain the % character: if the unescaped URL is then used in 
    #   relative URL calculation, it may be unescaped again (rare).
    $path=~ s/\+/ /g ;
    $path=~ s/%([0-9A-Fa-f]{2})/chr(hex($1))/ge ;

    # Note that in HTTP, the presence of a host implies a path beginning with
    #   '/', so $path should be '/' for URLs like "http://www.somehost.com"
    $path= '/' if !length($path) && length($site) && lc($scheme) eq 'http' ;

    return($scheme, $site, $path, $query, $fragment) ;

}


# Returns a full URL string, given its components
# The full procedure is described in the URI draft, section 5.2, step 7.
sub unparse_url {
    my($scheme, $site, $path, $query, $fragment)= @_ ;
    my($URL) ;

    $URL= "$scheme:"    if defined($scheme) ;
    $URL.= "//$site"    if defined($site) ;
    $URL.= $path ;
    $URL.= "?$query"    if defined($query) ;
    $URL.= "#$fragment" if defined($fragment) ;

    return $URL ;
}


# Returns a canonicalized absolute URL, given a relative URL and a base URL.
# The full procedure is described in the URI draft, section 5.2.
# Note that a relative URI of "#fragment" should be resolved to "the current
#   document", not to an absolute URL.  This presents a quandary for this
#   routine:  should it always return an absolute URL, thus violating the
#   spec, or should it not always return an absolute URL, thus requiring any
#   caller to check for this special case?  This routine leaves that up to
#   the caller, with $return_rel_fragment-- if set, stick to the spec;
#   otherwise, always return an absolute URL.  See section G.4 of the draft.
# Note that the pathname reduction in steps 6.c-f messes up any PATH_INFO
#   that has ./ or ../ in it, which may be a bug in the spec.
sub absolute_url {
    my($relurl, $baseurl, $return_rel_fragment)= @_ ;
    my(@relurl, @baseurl) ;

    # parse_url() returns scheme, site, path, query, fragment
    @relurl= &parse_url($relurl) ;      # Step 1
    @baseurl= &parse_url($baseurl) ;

    COMBINE: {

        # Step 2
        # See note above about $return_rel_fragment
        if (  $relurl[2] eq '' && 
              !defined($relurl[0]) &&
              !defined($relurl[1]) &&
              !defined($relurl[3]) ) {
            @relurl[0..3]= @baseurl[0..3] ;
            return $relurl if $return_rel_fragment ;   # see note above
            last COMBINE ;
        }

        last COMBINE if defined($relurl[0]) ;    # Step 3
        $relurl[0]= $baseurl[0] ;

        last COMBINE if defined($relurl[1]) ;    # Step 4
        $relurl[1]= $baseurl[1] ;

        last COMBINE if $relurl[2]=~ m#^/# ;     # Step 5

        # Step 6-- resolve relative path
        my($path)= $baseurl[2]=~ m#^(.*/)# ;     # Step 6.a
        $relurl[2]= $path . $relurl[2] ;         # Step 6.b
        
    } # COMBINE

    # Put the remaining steps outside of the block to canonicalize the path.
    # Arguably, this is not allowed.  To avoid such arguments at the expense of
    #   path canonicalization, put steps 6.c-f back in the COMBINE block.

    1 while $relurl[2]=~ s#(^|/)\./#$1# ;    # Step 6.c
    $relurl[2]=~ s#(^|/)\.$#$1# ;            # Step 6.d

    # Step 6.e
    my($oldpath) ;
    while ($relurl[2]=~ s#(([^/]+)/\.\./)# ($2 eq '..')  ? $1  : '' #ge) {
        last if ($relurl[2] eq $oldpath) ;
        $oldpath= $relurl[2] ;
    }

    # Step 6.f
    $relurl[2]=~ s#(([^/]+)/\.\.$)# ($2 eq '..')  ? $1  : '' #ge ;

    # Step 6.g: allow leading ".." segments to remain in path
    # Step 6.h: relurl[2] is already the buffer string

    # To canonicalize further, lowercase the hostname (is this valid for all
    #   schemes?)
    $relurl[1]= lc($relurl[1]) if defined($relurl[1]) ;

    return &unparse_url(@relurl) ;                  # Step 7
}



# Convert a local URL into a canonicalized absolute path, or undef if
#   not on this host or other error.
# Result should only be used as filename.
# Supports UserDir (e.g. public_html) for "/~username/path/file" URLs.
# Supports Alias, AliasMatch, ScriptAlias, and ScriptAliasMatch from srm.conf
#   (but note use of Perl regex's instead of standard regex's).
# Inserts index.html, etc. (from @DIRECTORY_INDEX) if result is a directory,
#   but just return directory name (ending in '/') if none of those exists.
# Removes PATH_INFO, if any, from filename.
# Directory names are always returned with trailing slash (which would not
#   be appropriate if PATH_INFO was to be retained).
# While this routines makes some tests (e.g. if the file is a directory),
#   it does not verify that file at the resulting $filename exists.
# Note that not all URLs point to files, so this routine is not always
#   appropriate.  In this program, the result from this routine is only
#   used when we know the URL is not a CGI script (and is therefore a file),
#   except in &is_cgi() itself, which tests if a file is a CGI script.
#   If it weren't for &is_cgi(), we could ignore cases when the URL isn't
#   a file.
# 12-1-99 JSM:  Changed to also return "redirected" location, in case URL
#   is a directory but not ending in a slash, so relative URLs will resolve
#   correctly against the redirected URL.
sub url_to_filename {
    my($URL)= @_ ;
    my($URLpath, $path, $location, $docroot, $user) ;
    return undef unless $URL=~ m#^http://\Q$LOCAL_HOST\E/#io ;
    $URLpath= (&parse_url($URL))[2] ;
    die "couldn't get path from [$URL]" unless length($URLpath) ;

    # Note minor security hole:  if this script is run setuid, then any 
    #   file on the system could be read by using an ALIAS to point to the 
    #   file.  Note also that if a $URLpath such as "/alias/dir/../.." is
    #   passed to this routine, the alias will be substituted BEFORE the
    #   ".." path segments are traversed.  A case like this probably a
    #   mistake in the URL anyway.

    # Make no more than one alias substitution-- is there a precedence order?
    # Note that %(.*)_MATCH use Perl regex's, not standard regex's.
    # 3-29-99 JSM:  These all alias to actual directory, not to a resulting
    #   URL, so no further conversion should be done if one of these matches.
    # 3-29-99 JSM:  Changed ALIAS_MATCH and SCRIPT_ALIAS_MATCH blocks to
    #   allow $1-type substitution in targets; MUST TEST!
    ALIAS: {
        foreach (keys %ALIAS) 
            { $path= $URLpath, last ALIAS 
                  if $URLpath=~ s/^\Q$_\E/$ALIAS{$_}/ }
        foreach (keys %ALIAS_MATCH) 
            { $path= $URLpath, last ALIAS 
                  if eval "\$URLpath=~ s/^\$_/$ALIAS_MATCH{$_}/" }
        foreach (keys %SCRIPT_ALIAS) 
            { $path= $URLpath, last ALIAS 
                  if $URLpath=~ s/^\Q$_\E/$SCRIPT_ALIAS{$_}/ }
        foreach (keys %SCRIPT_ALIAS_MATCH) 
            { $path= $URLpath, last ALIAS 
                  if eval "\$URLpath=~ s/^\$_/$SCRIPT_ALIAS_MATCH{$_}/" }
    }


    # If $path has been set in above ALIAS block, no further conversion is
    #   needed.  
    if ($path eq '') {

        # Must check for ^/.. before PATH_INFO check, in case $URL's path
        #   is e.g. '/../conf/access.conf'
        return undef if $URLpath=~ m#^/\.\.(/|$)# ;   # ^/.. is not allowed

        # Set $docroot and $path for this file, based on the URL (contains '~' ?)
        if (length($USER_DIR) and ($user,$path)= $URLpath=~ m#^/~([^/]+)(.*)# ) {
            &read_home_dirs unless %home_dir ;   # only read when needed
            return undef unless length($home_dir{$user}) ;
            $docroot= "$home_dir{$user}/$USER_DIR" ;
            $path= '/' unless length($path) ;
        } else {
            # If we have no $DOCUMENT_ROOT, we can't handle URLs without ~.
            return undef unless $DOC_ROOT_EXISTS ;
            $docroot= $DOCUMENT_ROOT ;
            $path= $URLpath ;
        }

        # Handle PATH_INFO: remove path segments until an existing file is named.
        # Note that directories cannot have PATH_INFO after them.
        unless (-e "$docroot$path") {
            for (my($path2)= $path ; $path2=~ m#/# ; $path2=~ s#/[^/]*$##) {
                 last if -d "$docroot$path2" ;
                 $path= $path2, last if -e "$docroot$path2" ;
             }
        }

        # canonicalize path, and recheck for ^/.. (handles an obscure error,
        #   when $URL's path is e.g. '/a/b/../../..'; but must canonicalize
        #   after PATH_INFO check, in case path is e.g. '/a/b.cgi/../../..').
        1 while $path=~ s#/\.(/|$)#/# ;
        1 while $path=~ s#/(?!\.\./)[^/]+/\.\.(/|$)#/# ;
        return undef if $path=~ m#^/\.\.(/|$)# ;   # ^/.. is not allowed

        $path= "$docroot$path" ;

    }


    # Add "index.html", etc. if appropriate
    if (-d $path) {
        $path.= '/' unless $path=~ m#/$# ;
        # 12-1-99 JSM: set "redirected" location also
        $location= "$URL/" unless $URL=~ m#/$# ;
        foreach (@DIRECTORY_INDEX) {
            $path.= $_, last if -f "$path$_" ;
        }
    }
    return ($path, $location) ;
}


# Convert a local (possibly relative) pathname into a canonicalized URL.
# If filename is relative and no $basepath is given, assume it's in the 
#   current directory.
# Supports UserDir (e.g. public_html) for "/~username/path/file" URLs.
# Each path segment is checked to see if it's the same as $DOCUMENT_ROOT,
#   by comparing inodes.  When a match is found, it's cut off the front,
#   and an absolute URL is constructed.  If $DOCUMENT_ROOT is never matched,
#   then $USER_DIR is scanned for.  If that doesn't match (i.e. the file 
#   is not served to the Web), undef is returned.
# Note that $DOC_ROOT_DEV and $DOC_ROOT_INODE are set at the start of the
#   program for efficiency, but are an integral part of this routine.
# %ALIAS is supported using %UN_ALIAS, as best as possible.  See next note
#   on avoiding use of this routine.
# This is currently only used when parsing command-line filenames, and when
#   an <!--#include file="..." --> includes an <!--#include virtual="..." -->
#   (which may be an error anyway).  Thus, it can be avoided if needed, such
#   as when complex aliasing makes results ambiguous.
# jsm-- should this add/remove @DIRECTORY_INDEX, to avoid some duplication?
# 3-29-99 JSM:  Changed UNALIAS handling-- if it's unaliased, then no other
#   conversion is necessary.
sub filename_to_url {
    my($path, $basepath)= @_ ;
    my($URLpath) ;
    unless ($path=~ m#^/#) {
        $basepath= $CWD unless length($basepath) ;
        $basepath.= '/' if -d $basepath && $basepath!~ m#/$# ;
        $basepath=~ s#/[^/]*$#/# ;
        $path= "$basepath$path"  ;
    }

    # canonicalize filename by removing ./ and ../ where appropriate
    1 while $path=~ s#/\.(/|$)#/# ;
    1 while $path=~ s#/(?!\.\./)[^/]+/\.\.(/|$)#/# ;

    # canonicalize directory to include final /
    $path.= '/' if -d $path && $path!~ m#/$# ;

    # First, if path can be unaliased, return that.
    # (Added 3-29-99 by JSM.)
    foreach (keys %UN_ALIAS) 
        { $URLpath= $path, last if $path=~ s/^\Q$_\E/$UN_ALIAS{$_}/ }
    return "http://\L$LOCAL_HOST\E$URLpath" if $URLpath ne '' ;

    # Then, check if file is under $DOCUMENT_ROOT tree, and convert if so.
    if ($DOC_ROOT_EXISTS) {
        my($doc_root)= $path ;
        while ($doc_root=~ s#/[^/]*$##) {
            my($dev,$inode)= stat("$doc_root/.") ;
            if ( ($dev==$DOC_ROOT_DEV) && ($inode==$DOC_ROOT_INODE) ) {
                $path=~ s/^$doc_root// ;
#                foreach (keys %UN_ALIAS) 
#                    { last if $path=~ s/^\Q$_\E/$UN_ALIAS{$_}/ }
                return "http://\L$LOCAL_HOST\E$path" ;
            }
        }
    }

    # Handle possible case of "~username/$USER_DIR/$path"
    # I don't think %ALIAS applies here, does it?
    # This misses some when $HOME/$USER_DIR points through a symbolic link,
    #   and $CWD isn't set to match %dir_to_user.  Work around by avoiding
    #   this routine, e.g. using only URLs on command line.
    if (length($USER_DIR)) {
        if ($path=~ m#^(.*?)/$USER_DIR(/.*)# ) {
            # First, see if path is in %dir_to_user
            &read_home_dirs unless %dir_to_user ;   # only read when needed
            return "http://\L$LOCAL_HOST\E/~$dir_to_user{$1}$2"
                if length($dir_to_user{$1}) ;

            # If not, then we must check inodes to equivalate directories
            &read_inode_to_user unless %inode_to_user ; # only read when needed
            my($dev,$inode)= stat("$1/.") ;
            return "http://\L$LOCAL_HOST\E/~$inode_to_user{$dev}{$inode}$2"
                if length($inode_to_user{$dev}{$inode}) ;
        }
    }

    return undef ;
}



# Reads all users' home directory into %home_dir, from /etc/passwd.
# Also creates %dir_to_user, which is faster than %inode_to_user (below).
# Only used when $USER_DIR is used, for "/~username/path/file" URLs.
# 2-27-00 JSM:  Changed to use getpwent, instead of reading /etc/passwd.
sub read_home_dirs {
    my($user, $homedir) ;

    setpwent ;   # to rewind, in case getpwent has already been used
    while ( ($user, $homedir)= (getpwent)[0,7] ) {
        $home_dir{$user}= $homedir ;
        $dir_to_user{$homedir}= $user
            unless $dir_to_user{$homedir} ne '' ;
    }
    endpwent ;   # clean way to end getpwent processing
}


# Reads home directory inode information into %inode_to_user, from /etc/passwd.
# Because this is time-consuming, it is only called if needed, and only once.
# Only used when $USER_DIR is used, for "/~username/path/file" URLs.
# On SPARCstation-10 with 3000 /etc/passwd records, this takes ~2 seconds.
# 2-27-00 JSM:  Changed to use already-existing %home_dir, instead of reading
#   /etc/passwd again.
sub read_inode_to_user {
    my($user, $homedir) ;
    my($dev, $inode) ;
    &read_home_dirs unless %home_dir ;   # only read when needed

    while ( ($user, $homedir)= each %home_dir ) {
        ($dev,$inode)= stat("$homedir/.") ;
        $inode_to_user{$dev}{$inode}= $user 
            unless $inode_to_user{$dev}{$inode} ne '' ;
    }
}



#----- Extracting URLs from HTML ------------------------------------


# Parse an SGML tag, and return a hash structure with a "name" scalar and
#   an "attrib" hash.
# Parses first tag in string, ignoring all surrounding text.
# Results are canonicalized to lower case wherever case-insensitive.
sub parse_tag {
    my($tag)= @_ ;       # will be mangled
    my($tagname,%attrib) ;

    # only parse first tag in string
    ($tag)= split(/>/, $tag) ;  # remove all after >
    $tag=~ s/^([^<]*<)?\s*// ;  # remove pre-<, <, and leading blanks
    ($tagname,$tag)= split(/\s+/, $tag, 2) ;        # split out tag name

    # Extract name/value (possibly quoted), lowercase name, set $attrib{}.
    # If quoted, is delimited by quotes; if not, delimited by whitespace.
    $attrib{lc($1)}= &HTMLunescape($+)
        while ($tag=~ s/\s*(\w+)\s*=\s*(([^"']\S*)|"([^"]*)"?|'([^']*)'?)//) ;

    # now, get remaining non-valued (boolean) attributes
    $tag=~ s/^\s*|\s*$//g ;             # skip leading/trailing blanks
    foreach (split(/\s+/, $tag)) {
        $_= lc($_) ;
        $attrib{$_}= $_ ;   # booleans have values equal to their name
    }

    return { 'name'   => lc($tagname), 
             'attrib' => \%attrib       } ;
}


# Unescape any HTML character references and return resulting string.
# Support entity character references in %e_to_ch (which is incomplete),
#   plus "$#ddd;" and "&#xdd;" forms for values<256.
# Note that not decoding a valid character is erroneous, in that a
#   subsequent re-escaping will not return the original string, because
#   of the ampersand.  Nonetheless, that's preferable to losing the data.
#   Q:  Is there an appropriate general way to represent an unescaped string?
sub HTMLunescape {
    my($s)= @_ ;

    # Try alpha, decimal, and hex representations, only substituting if valid
    $s=~ s/&(([a-zA-Z][a-zA-Z0-9.-]*);?|#([0-9]+);?|#[Xx]([0-9a-fA-F]+);?)/
              length($2)   ? ( defined($e_to_ch{$2}) ? $e_to_ch{$2}  : "&$1" )
            : length($3)   ? ( $3 < 256              ? chr($3)       : "&$1" )
            : length($4)   ? ( hex($4) < 256         ? chr(hex($4))  : "&$1" )
                           :   "&$1"
          /ge ;

    return $s ;
}


# Given a block of HTML, extracts all URLs referenced in it, and adds them
#   to our data structures to be downloaded or checked (i.e. calls 
#   &add_url()).
# Note that %html_urls and %non_html_urls are set at the start of the
#   program for efficiency, but are an integral part of this routine.
# Currently, this extracts all <.*?> patterns, which may not be valid if
#   "<" or ">" characters are e.g. inside a <script> element.
sub extract_urls {
    my($HTML, $baseurl, $referer, $depth)= @_ ;
    my(@tags) ;

    # Remove comments before extracting links, as pointed out by Tim Hunter.
    $HTML=~ s/<!--.*?--.*?>//gs ;

    # We must look for <base> tag before all the work, so we must parse
    #   all tags first.  :(  Therefore, we save this large array of
    #   structures for efficiency, hoping we don't run out of memory.
    my($i)= -1 ;  # to start at array element 0

    foreach ($HTML=~ /(<.*?>)/gs) {
        $tags[++$i]= &parse_tag($_) ;
        $baseurl= $tags[$i]{'attrib'}{'href'}
            if ($tags[$i]{'name'} eq 'base') 
                and (length($tags[$i]{'attrib'}{'href'})) ;
    }

    # For each tag, call &add_url() for each URL in the tag
    foreach my $tag (@tags) {
        next if $tag->{'name'}=~ m#^/# ;

        # Handle the "regular" tag-attributes, in %html_urls and %non_html_urls

        foreach (@{$html_urls{$tag->{'name'}}}) {
            &add_url(&absolute_url($tag->{'attrib'}{$_}, $baseurl), 
                     $referer, $depth) 
                if length($tag->{'attrib'}{$_}) ;
        }

        foreach (@{$non_html_urls{$tag->{'name'}}}) {
            &add_url(&absolute_url($tag->{'attrib'}{$_}, $baseurl), 
                     $referer, $depth, 0) 
                if length($tag->{'attrib'}{$_}) ;
        }

        # Now handle each tag-attribute that needs special attention

        if ($tag->{'name'} eq 'object') {
            my($codebase)=
                &absolute_url($tag->{'attrib'}{'codebase'}, $baseurl) ;

            &add_url(&absolute_url($tag->{'attrib'}{'data'}, $codebase),
                     $referer, $depth)
                if length($tag->{'attrib'}{'data'}) ;

            # There seems to be a contradiction between the HTML 4.0 spec
            #   section 13.3.3 and RFC 1808 section 4 step 2b regarding
            #   the URL resolution of e.g. classid="java:program.start".
            #   For now, we'll stick with the RFC 1808 method.
            &add_url(&absolute_url($tag->{'attrib'}{'classid'}, $codebase),
                     $referer, $depth, 0)
                if length($tag->{'attrib'}{'classid'}) ;

            # <object> tag's "archive" attribute is a space-separated list
            foreach (split(/\s+/, $tag->{'attrib'}{'archive'})) {
                &add_url(&absolute_url($_, $codebase), $referer, $depth, 0) ;
            }

        } elsif ($tag->{'name'} eq 'head') {
            # "profile" attribute is a space-separated list
            foreach (split(/\s+/, $tag->{'attrib'}{'profile'})) {
                &add_url(&absolute_url($_, $baseurl), $referer, $depth, 0) ;
            }

        } elsif ($tag->{'name'} eq 'applet') {
            my($codebase)=
                &absolute_url($tag->{'attrib'}{'codebase'}, $baseurl) ;

            &add_url(&absolute_url($tag->{'attrib'}{'code'}, $codebase),
                     $referer, $depth, 0)
                if length($tag->{'attrib'}{'code'}) ;

            &add_url(&absolute_url($tag->{'attrib'}{'object'}, $codebase),
                     $referer, $depth, 0)
                if length($tag->{'attrib'}{'object'}) ;

            # <applet> tag's "archive" attribute is a comma-separated list
            foreach (split(/\s*,\s*/, $tag->{'attrib'}{'archive'})) {
                &add_url(&absolute_url($_, $codebase), $referer, $depth, 0) ;
            }

        # Check form script for existence only, but don't follow hyperlinks.
        # Handles the unlikely case when a CGI is referred to by both a
        #   <form action=___.cgi> tag and an <a href=___.cgi> tag: If a CGI
        #   is only pointed to by <form action>, then don't follow the link
        #   (i.e. set $url->{'dontfollow'} to verify, not load).  If a CGI
        #   is called by at least one <a href> tag, then do follow the link.
        } elsif ($tag->{'name'} eq 'form') {
            &add_url(&absolute_url($tag->{'attrib'}{'action'}, $baseurl),
                     $referer, $depth, undef, 1, 1)
                if length($tag->{'attrib'}{'action'}) ;

        }   # if ($tag->{'name'} eq '...')


    }   # foreach $tag (@tags)

}   # &extract_urls()



#----- output routines ------------------------------------------------

# Generate final report
sub make_report {
    if ($verbose_report) { &make_verbose_report }
    else                 { &make_short_report }
}



# Generate a verbose report of all URLs that have been checked
sub make_verbose_report {
    my($rootlist)= join("\n    ", @ARGV) ;
    my($numurls)= scalar keys %url ;

    print <<EOH ;
======================================================================
Report of invalid links on $LOCAL_HOST, 
recursively followed starting with the files/URLs:

    $rootlist

EOH

    if (@INCLUDE_PATTERNS) {
        my($includelist)= join("\n    ", @INCLUDE_PATTERNS) ;
        print <<EOH ;
Only including URLs matching at least one of the following patterns:
    $includelist

EOH
    }

    if (@EXCLUDE_PATTERNS) {
        my($excludelist)= join("\n    ", @EXCLUDE_PATTERNS) ;
        print <<EOH ;
Excluding URLs matching any of the following patterns:
    $excludelist

EOH
    }

    print "Total URLs checked: $numurls\n\n" ;

    print( "Only reporting response codes beginning with: ", 
           join(", ", @INCLUDE_STATUS), "\n" )
        if @INCLUDE_STATUS ;

    print( "Excluding response codes beginning with: ",
           join(", ", @EXCLUDE_STATUS), "\n" )
        if @EXCLUDE_STATUS ;

    print "Maximum traversal depth: $max_depth\n" if length($max_depth) ;
    print "Only local files were checked; no CGI scripts were invoked.\n"
        if $file_check ;
    print "Local URLs were read from the filesystem where possible.\n"
        unless $full_http_check ;

    print "\n" ;

    # Only report status errors if there are any.
    # Using grep()s is slightly inefficient, but a *lot* cleaner.
    my($has_statuserrs) ;
    foreach my $URL (sort keys %url) {
        next if @INCLUDE_STATUS &&
                !grep($url{$URL}{'status'}=~ /^$_/, @INCLUDE_STATUS) ;
        next if grep($url{$URL}{'status'}=~ /^$_/, @EXCLUDE_STATUS) ;

        $has_statuserrs=1, last ;
    }


    if ($has_statuserrs) {
        print <<EOH ;
======================================================================

RESULTS FROM ALL URLS WITH SELECTED RESPONSE STATUS CODES
------------------------------------------------------------

EOH

        foreach (sort keys %url) {
            my($u)= $url{$_} ;

            next if @INCLUDE_STATUS &&
                    !grep($u->{'status'}=~ /^$_/, @INCLUDE_STATUS) ;
            next if grep($u->{'status'}=~ /^$_/, @EXCLUDE_STATUS) ;

            print "$u->{'URL'}\n", '-' x 50, "\n" ;
            print "Status:   $u->{'status'}\n" ;
            print "Moved to: $u->{'location'}\n" if $u->{'location'} ;
            print "Depth:    $u->{'depth'}\n" ;

            for (my($URL)= $u->{'referer'}, my($tab)= '  ' ;
                 length($url{$URL}) ;
                 $URL= $url{$URL}{'referer'}, $tab.= '  ' )
            {
                print "${tab}referred by $url{$URL}{'URL'}\n" ;
                die "PROGRAM ERROR: apparent infinite referer loop.\n"
                    if length($tab)>200 ;      # simple-minded sanity check
            }

            print "\n\n" ;
        }  # URL

    }



    # Only report SSI errors if there are any
    my($has_ssierrs) ;
    foreach (sort keys %url) { 
        $has_ssierrs=1, last if @{$url{$_}{'ssierrs'}} ;
    }


    if ($has_ssierrs) {
        print <<EOH ;
======================================================================

PROBLEMS WITH SERVER-SIDE INCLUDE (AKA SHTML) DIRECTIVES
------------------------------------------------------------

EOH
        foreach (sort keys %url) {
            my($u)= $url{$_} ;
            if (@{$u->{'ssierrs'}}) {
                print "$u->{'URL'}\n", '-' x 50, "\n" ;
                printf "Total %s SSI Errors:\n",  $#{$u->{'ssierrs'}}+1 ;
                foreach my $i (0..$#{$u->{'ssierrs'}}) {
                    printf "  %s) ", $i+1 ;
                    my($tab)= ' ' x (4+length($i+1)) ;
                    my($tab2) ;
                    foreach my $level (@{$u->{'ssierrs'}[$i]}) {
                        print "${tab2}file:   $level->{'path'}\n" ;
                        print "${tab}in tag: $level->{'tag'}\n" ;
                        print "${tab}error:  $level->{'errmsg'}\n" 
                            if $level->{'errmsg'} ;
                        $tab.= '  ' ;
                        $tab2= $tab ;
                    }
                }
                print "\n\n" ;
            }
        }
    }


    unless ($has_statuserrs or $has_ssierrs) {
        print <<EOH ;
======================================================================

                    <<     NO ERRORS FOUND     >>

EOH
    }

    print '=' x 70, "\n" ;

} # &make_verbose_report()


# Generate a one-line-per-URL report
sub make_short_report {
    my($numurls)= scalar keys %url ;

    print "Total $numurls URLs checked\n" ;
    URL: foreach my $URL (sort keys %url) {
        next if @INCLUDE_STATUS &&
                !grep($url{$URL}{'status'}=~ /^$_/, @INCLUDE_STATUS) ;
        next if grep($url{$URL}{'status'}=~ /^$_/, @EXCLUDE_STATUS) ;

        print "$url{$URL}{'URL'}\t$url{$URL}{'status'}\n" ;
    }  # URL
}



# Print a summary of usage and exit
sub usage {
    print <<EOF ;

Checklinks $CL_VERSION

To recursively check all HTML links on the local site, enter:
    $0  <options>  [ start-file | start-URL ] ...

Options include:
  -v                    Generate full (verbose) report, including full
                          referral information and detailed SSI error
                          reporting.
  -I <include-pattern>  Only check URLs matching <include-pattern>.
  -X <exclude-pattern>  Don't check URLs matching <exclude-pattern>.
  -i <include-status>   Only report URLs whose response code starts with
                          the pattern <include-status>.
  -x <exclude-status>   Don't report URLs whose response code starts with
                          the pattern <exclude-status>.  Default is to
                          exclude "200" responses only (i.e. "-x 200").
  -d <max-depth>        Traverse links no deeper than <max-depth>.
  -f                    "File mode"-- only read files from the filesystem;
                          do not go through the HTTP server at all.  This
                          will skip all URLs that point to CGI scripts.
  -h                    "HTTP mode"-- use HTTP to check ALL URLs, even
                          if they could be read from the filesystem.
                          Incompatible with "-f" option.
  -c <config-file>      Read appropriate configuration parameters from
                          <config-file>, typically srm.conf.  Use '-' to
                          read from STDIN.  If a directory is named, use
                          "srm.conf" in that directory.
  -q                    Print current configuration parameters.
  -?                    Print this help message.
  --                    End command-line option processing.

Don't stack options like "-vf"; use "-v -f" instead.

For -I, -X, -i, and -x:
  Values are interpreted as Perl 5 regular expressions.
  Use multiple options to build a list (e.g. "-I include1 -I include2").
  Use a value of '' to clear a list (e.g. -x '' means "report all responses",
    "-x '' -x 401" means "report all but 401 responses").
  As a special case, an empty -I or -i list implies no include-restrictions.
  If an item is in both the include and exclude list, it is excluded.
  Note that -I and -X restrict which URLs are traversed into, so they may
    prune large areas of your Web structure.

EOF
    exit ;
}


#----- debugging routines below ---------------------------------------

# Print current configuration settings
sub print_config {
    print "\n----- OPTIONS SPECIFIC TO THIS EXECUTION -----------------------------\n\n" ;
    
    print "Include only URLs containing one of the following patterns:\n",
          ( map { "    $_\n" } @INCLUDE_PATTERNS ),
          "\n"
        if @INCLUDE_PATTERNS ;

    print "Exclude URLs containing one of the following patterns:\n",
          (map { "    $_\n" } @EXCLUDE_PATTERNS ),
          "\n"
        if @EXCLUDE_PATTERNS ;

    print "Only report response codes beginning with:  ",
          join(", ", @INCLUDE_STATUS), "\n"
        if @INCLUDE_STATUS ;

    print "Don't report response codes beginning with: ",
          join(", ", @EXCLUDE_STATUS), "\n"
        if @EXCLUDE_STATUS ;

    print "\nMaximum search depth:  $max_depth\n" if length($max_depth) ;

    print <<EOS ;

----- INSTALLATION PARAMETERS ----------------------------------------
    
Local Host:           $LOCAL_HOST
Document Root:        $DOCUMENT_ROOT
User Web Directory:   $USER_DIR
Default Filename(s):  @DIRECTORY_INDEX

File extensions indicating a CGI program:       @CGI_EXTENSIONS
File extensions indicating server-parsed HTML:  @SHTML_EXTENSIONS

EOS

    print "Directory Aliases:\n",
          map { sprintf("    %15s ==> %s\n", $_, $ALIAS{$_}) }
              sort keys %ALIAS, 
          "\n"
        if keys %ALIAS ;

    print "Directory Regular Expression Aliases:\n", 
          map { sprintf("    %15s ==> %s\n", $_, $ALIAS_MATCH{$_}) }
              sort keys %ALIAS_MATCH,
          "\n"
        if keys %ALIAS_MATCH ;

    print "Script Directory Aliases:\n", 
          map { sprintf("    %15s ==> %s\n", $_, $SCRIPT_ALIAS{$_}) }
              sort keys %SCRIPT_ALIAS,
          "\n"
        if keys %SCRIPT_ALIAS ;

    print "Script Directory Regular Expression Aliases:\n", 
          map { sprintf("    %15s ==> %s\n", $_, $SCRIPT_ALIAS_MATCH{$_}) }
              sort keys %SCRIPT_ALIAS_MATCH,
          "\n"
        if keys %SCRIPT_ALIAS_MATCH ;

    if ($ENV{'http_proxy'}) {
        print "HTTP Proxy:  $ENV{'http_proxy'}\n" ;
        print "Except to hosts ending in:  $ENV{'no_proxy'}\n" 
            if $ENV{'no_proxy'} ;
        print "\n" ;
    } else {
        print "Not using any HTTP Proxy.\n" ;
    }

    print "Maximum HTTP redirections allowed for a URL:  $MAX_REDIRECTS\n",
          "Maximum number of tries to get a single URL:  $MAX_ATTEMPTS\n\n";
          
    print '-' x 70, "\n\n" ;

}



# print configuration, in a style more for debugging (mostly from srm.conf)
sub print_debug_config {
    print <<EOF ;
DOCUMENT_ROOT=    [$DOCUMENT_ROOT]
USER_DIR=         [$USER_DIR]
DIRECTORY_INDEX=  [@DIRECTORY_INDEX]

CGI_EXTENSIONS=   [@CGI_EXTENSIONS]
SHTML_EXTENSIONS= [@SHTML_EXTENSIONS]

EOF

    foreach (sort keys %ALIAS) 
        { print "ALIAS{$_}= [$ALIAS{$_}]\n" }
    foreach (sort keys %ALIAS_MATCH) 
        { print "ALIAS_MATCH{$_}= [$ALIAS_MATCH{$_}]\n" }
    foreach (sort keys %SCRIPT_ALIAS) 
        { print "SCRIPT_ALIAS{$_}= [$SCRIPT_ALIAS{$_}]\n" }
    foreach (sort keys %SCRIPT_ALIAS_MATCH) 
        { print "SCRIPT_ALIAS_MATCH{$_}= [$SCRIPT_ALIAS_MATCH{$_}]\n" }

}


#----------------------------------------------------------------------
#
#   SPARSE DOCUMENTATION ON PROGRAM INTERNALS:
#
#----------------------------------------------------------------------
#
#   URLs are read directly from the local filesystem if possible, or
#   downloaded or checked with HTTP GET and HEAD methods if not.  CGI
#   scripts are called through the HTTP server, even if local (since the
#   resource their URLs refer to are not files).
#
#
#   Global variables:
#
#     %url holds all data regarding all URLs, using the text URL as the key:
#
#     %url{$URL}{qw( URL filename referer depth
#                    ishtml iscgi islocal dontfollow
#                    status location numredirects numtries lasttried
#                    ssierrs
#                    hasbeenloaded)}  <== only for testing
#         URL is same as hash key
#         ishtml means is type text/html; links MAY be extracted.  (Used in
#           main loop.)
#         iscgi means is a CGI script; MUST NOT be read directly from 
#           filesystem. (Used in verify_url() and load_url().)
#         islocal means served by local server:
#             a) MAY be read directly from filesystem
#             b) links MAY be extracted
#         dontfollow means links MUST NOT be extracted.
#         status begins with a number, but may have text afterward.
#
#         islocal= (URL=~ m#^http://$LOCAL_HOST/#i) , i.e. redundant.
#         ishtml and iscgi are guesses, may be 0, 1, or '' for unset.
#         iscgi only matters for local URLs, but may safely be set for 
#           remote URLs.
#         dontfollow is a hack to handle an obscure CGI-related situation.
#
#     In addition to the standard HTTP status codes, 'status' may take one
#     of the following values:
#         450 Can't read file: $!
#         451 SSI Error(s) (__ total)
#         600 Can't create socket: $@
#         601 Connection timed out
#         602 Incomplete response (%s of %s bytes)
#         603 Incomplete chunked response
#
#     Handling of 602 and 603 is somewhat hackish-- overwrites real status
#       line with artificial one.
#
#     $url->{'ssierrs'} is a list of errors relating to SSI includes.
#       Each error has a list of filenames representing the include
#       chain.  Each element in the chain stores the absolute
#       pathname, the <!--#include--> tag which called it, and
#       possibly an error message (usually only on the last file in
#       the include-chain).  Thus, the entire structure is
#
#         $url->{'ssierrs'}[$errid][$levelsdeep]{qw( path tag errmsg )}
#
#       This is only a memory hog with many errors of deeply nested files.
#
#
#     @urlstoget: canonicalized list of URLs left to get, in order
#         of appearance.  List elements are pointers to %url elements.
#         This array may grow from &extract_urls(), or be rearranged when
#         a URL needs to be retried.
#
#   Note that $url is normally a pointer, and $URL is normally a string.
#
#   Files/paths are canonicalized by removing unneeded "./" and "../"
#     segments.
#   Directories are canonicalized to always end in "/", including in URLs;
#     note that some system variables and local directory variables aren't so.
#   URLs that point to "/" end in "/", e.g. "http://www.foo.com/".
#
#
#   This program has a lot of unused functionality for possible extension,
#   so don't get confused by routines that do more than they are used for.
#
#----------------------------------------------------------------------
#
# To do:
#
#   * If you want a different output format (like in HTML), it should 
#       be easy enough to add.
#   * This could be extended to check remote sites without much trouble.
#   * Should probably support directories that don't have index.html.
#   * Check that pointed-to fragments exist in target files
#   * keep track of servers and don't hit them too often
#   * support robots.txt
#   * add option to read list of URLs from file
#   * support HTTP Basic Authentication somehow.  Note that currently, this
#       reads local files even when they should be protected by .htaccess
#       (unless using -h option).
#   * Read list of local host names from gethostbyname() ?
#   * check read permissions on files?
#   * In the event multiple URLs redirect to the same URL, this could be a
#       little more efficient if it saved the redirected loads/downloads.
#
#----------------------------------------------------------------------