unset meta attributes (so they are reset for next meta)

[tclrobot.git] / robot.tcl

diff --git a/robot.tcl b/robot.tcl
index ffbfce4..e67a9b8 100755 (executable)
--- a/robot.tcl
+++ b/robot.tcl
@@ -1,5 +1,5 @@
 #!/usr/bin/tclsh 
-# $Id: robot.tcl,v 1.30 2002/02/17 09:29:18 adam Exp $
+# $Id: robot.tcl,v 1.34 2002/06/18 19:57:53 adam Exp $
 #
 proc RobotFileNext1 {area lead} {
     # puts "RobotFileNext1 area=$area lead=$lead"
@@ -171,23 +171,18 @@ proc RobotFileOpen {area host path {mode w}} {
             exec mkdir $d
             cd ./$d
            if {![string compare $area unvisited] && $i == 1 && $mode == "w"} {
-               set out [open frobots.txt w]
-               puts "creating robots.txt in $d"
-               close $out
-                incr status(unvisited)
+               if {[string compare $path /robots.txt]} {
+                   set out [open frobots.txt w]
+                   puts "creating robots.txt in $d"
+                   close $out
+                    incr status(unvisited)
+               }
            }
         }
     }
     set d [lindex $comp $len]
     if {[string length $d]} {
         set out [open f$d $mode]
-        if {0} {
-            if {[file isfile $d/f]} {
-                set out [open $d/f $mode]
-            } else {
-                set out [open f$d $mode]
-            }
-        }
     } else {
         set out [open f $mode]
     }
@@ -293,12 +288,11 @@ proc RobotHref {url hrefx hostx pathx} {
     if {[string length $href] > 256} {
        return 0
     }
-    if {[string first {?} $href] >= 0} {
-       return 0
-    }
-    if {[string first {?} $url] >= 0 && [string first {?} $href] >= 0} {
-       return 0
-    }
+
+#   Skip pages that have ? in them
+#    if {[string first {?} $url] >= 0 && [string first {?} $href] >= 0} {
+#      return 0
+#    }
     # get method (if any)
     if {![regexp {^([^/:]+):(.*)} $href x method hpath]} {
        set hpath $href
@@ -523,7 +517,8 @@ proc RobotTextHtml {url out} {
             set metaname {}
             set metacontent {}
             puts -nonewline $out "<meta"
-            foreach a [array names parm] {
+           set al [array names parm]
+            foreach a $al {
                 set al [string tolower $a]
                puts -nonewline $out " $al"
                 puts -nonewline $out {="}
@@ -537,6 +532,7 @@ proc RobotTextHtml {url out} {
                         set metacontent $parm($a)
                     }
                 }
+               unset parm($al)
             }
            puts $out "></meta>"
             # go through robots directives (af any)
@@ -553,7 +549,7 @@ proc RobotTextHtml {url out} {
             # don't print title of document content if noindex is used
             if {!$noindex} {
                 puts $out "<title>$title</title>"
-                regsub -all {<!--[^-]*->} $body { } abody
+                regsub -all {<!--[^-]*-->} $body { } abody
                 regsub -all -nocase {<script[^<]*</script>} $abody {} bbody
                 regsub -all {<[^\>]+>} $bbody {} nbody
                 puts $out "<documentcontent>"