Ticket #18259: quickindex-array.diff

src/port/port.tcl

@@ -690 +690 @@
             set installed_epoch     [lindex $i 5]
 
             # Get info about the port from the index
-            if {[catch {set res [mportsearch $portname no exact]} result]} {
+            if {[catch {set res [mportlookup $portname]} result]} {
                 global errorInfo
                 ui_debug "$errorInfo"
-                fatal "search for portname $portname failed: $result"
+                fatal "lookup of portname $portname failed: $result"
             }
             if {[llength $res] < 2} {
                 if {[macports::ui_isset ports_debug]} {
@@ -1361 +1361 @@
         # otherwise try to map the portname to a url
         if {$porturl eq ""} {
         # Verify the portname, getting portinfo to map to a porturl
-            if {[catch {mportsearch $portname no exact} result]} {
+            if {[catch {mportlookup $portname} result]} {
                 ui_debug "$::errorInfo"
-                break_softcontinue "search for portname $portname failed: $result" 1 status
+                break_softcontinue "lookup of portname $portname failed: $result" 1 status
             }
             if {[llength $result] < 2} {
                 break_softcontinue "Port $portname not found" 1 status
             }
-            set found [expr [llength $result] / 2]
-            if {$found > 1} {
-                ui_warn "Found $found port $portname definitions, displaying first one."
-            }
             array unset portinfo
             array set portinfo [lindex $result 1]
             set porturl $portinfo(porturl)
@@ -1707 +1703 @@
 
     foreachport $portlist {
         if {$porturl eq ""} {
-            # Search for the port.
-            if {[catch {mportsearch $portname no exact} result]} {
+            # Look up the port.
+            if {[catch {mportlookup $portname} result]} {
                 ui_debug $::errorInfo
-                break_softcontinue "The search for '$portname' failed: $result" \
+                break_softcontinue "The lookup of '$portname' failed: $result" \
                                 1 status
             }
             if {[llength $result] < 2} {
@@ -2177 +2173 @@
             set installed_epoch [lindex $i 5]
 
             # Get info about the port from the index
-            if {[catch {set res [mportsearch $portname no exact]} result]} {
+            if {[catch {set res [mportlookup $portname]} result]} {
                 global errorInfo
                 ui_debug "$errorInfo"
                 break_softcontinue "search for portname $portname failed: $result" 1 status
@@ -2286 +2282 @@
     }
     foreachport $portlist {
         if {$porturl eq ""} {
-            # search for port
-            if {[catch {mportsearch $portname no exact} result]} {
+            # look up port
+            if {[catch {mportlookup $portname} result]} {
                 global errorInfo
                 ui_debug "$errorInfo"
-                break_softcontinue "search for portname $portname failed: $result" 1 status
+                break_softcontinue "lookup of portname $portname failed: $result" 1 status
             }
             if {[llength $result] < 2} {
                 break_softcontinue "Port $portname not found" 1 status
@@ -2603 +2599 @@
         if {$porturl == ""} {
         
             # Verify the portname, getting portinfo to map to a porturl
-            if {[catch {set res [mportsearch $portname no exact]} result]} {
+            if {[catch {set res [mportlookup $portname]} result]} {
                 global errorInfo
                 ui_debug "$errorInfo"
-                break_softcontinue "search for portname $portname failed: $result" 1 status
+                break_softcontinue "lookup of portname $portname failed: $result" 1 status
             }
             if {[llength $res] < 2} {
                 break_softcontinue "Port $portname not found" 1 status
@@ -2760 +2756 @@
         # otherwise try to map the portname to a url
         if {$porturl == ""} {
             # Verify the portname, getting portinfo to map to a porturl
-            if {[catch {set res [mportsearch $portname no exact]} result]} {
+            if {[catch {set res [mportlookup $portname]} result]} {
                 global errorInfo
                 ui_debug "$errorInfo"
-                break_softcontinue "search for portname $portname failed: $result" 1 status
+                break_softcontinue "lookup of portname $portname failed: $result" 1 status
             }
             if {[llength $res] < 2} {
                 break_softcontinue "Port $portname not found" 1 status

src/port/portindex.tcl

@@ -141 +141 @@
 mporttraverse pindex $directory
 close $fd
 file rename -force $tempportindex [file join $outdir PortIndex]
+mports_generate_quickindex [file join $outdir PortIndex]
 puts "\nTotal number of ports parsed:\t$stats(total)\
       \nPorts successfully parsed:\t[expr $stats(total) - $stats(failed)]\t\
       \nPorts failed:\t\t\t$stats(failed)\n"

src/macports1.0/macports.tcl

@@ -710 +710 @@
             unsetenv $envkey
         }
     }
+    
+    # load the quick index
+    _mports_load_quickindex
 }
 
 proc macports::worker_init {workername portpath porturl portbuildpath options variations} {
@@ -1691 +1694 @@
     return $matches
 }
 
+# Returns the PortInfo for a single named port. The info comes from the
+# PortIndex, and name matching is case-insensitive. Unlike mportsearch, only
+# the first match is returned, but the return format is otherwise identical.
+# The advantage is that mportlookup is much faster than mportsearch, due to
+# the use of the quick index.
+proc mportlookup {name} {
+    global macports::portdbpath macports::sources
+    
+    set sourceno 0
+    set matches [list]
+    foreach source $sources {
+        set source [lindex $source 0]
+        if {[macports::getprotocol $source] != "mports"} {
+            global macports::quick_index
+            if {![info exists quick_index($sourceno,[string tolower $name])]} {
+                incr sourceno 1
+                continue
+            }
+            # The quick index is keyed on the port name, and provides the
+            # offset in the main PortIndex where the given port's PortInfo
+            # line can be found.
+            set offset $quick_index($sourceno,[string tolower $name])
+            incr sourceno 1
+            if {[catch {set fd [open [macports::getindex $source] r]} result]} {
+                ui_warn "Can't open index file for source: $source"
+            } else {
+                try {
+                    seek $fd $offset
+                    gets $fd line
+                    set name [lindex $line 0]
+                    set len [lindex $line 1]
+                    set line [read $fd $len]
+                        
+                    array set portinfo $line
+
+                    switch -regexp -- [macports::getprotocol ${source}] {
+                        {^rsync$} {
+                            # Rsync files are local
+                            set source_url "file://[macports::getsourcepath $source]"
+                        }
+                        {^https?$|^ftp$} {
+                            if {[_source_is_snapshot $source filename extension]} {
+                                # daily snapshot tarball
+                                set source_url "file://[macports::getsourcepath $source]"
+                             } else {
+                                # default action
+                                set source_url $source
+                             }
+                        }
+                        default {
+                            set source_url $source
+                        }
+                    }
+                    if {[info exists portinfo(portarchive)]} {
+                        set porturl ${source_url}/$portinfo(portarchive)
+                    } elseif {[info exists portinfo(portdir)]} {
+                        set porturl ${source_url}/$portinfo(portdir)
+                    }
+                    if {[info exists porturl]} {
+                        lappend line porturl $porturl
+                        ui_debug "Found port in $porturl"
+                    } else {
+                        ui_debug "Found port info: $line"
+                    }
+                    lappend matches $name
+                    lappend matches $line
+                    close $fd
+                    break
+                } catch {*} {
+                    ui_warn "It looks like your PortIndex file may be corrupt."
+                    throw
+                } finally {
+                    catch {close $fd}
+                }
+            }
+        } else {
+            array set attrs [list name $name]
+            set res [macports::index::search $macports::portdbpath $source [array get attrs]]
+            if {[llength $res] > 0} {
+                eval lappend matches $res
+                break
+            }
+        }
+    }
+
+    return $matches
+}
+
+# Loads PortIndex.quick from each source into the quick_index, generating
+# it first if necessary.
+proc _mports_load_quickindex {args} {
+    global macports::sources macports::quick_index
+    
+    set sourceno 0
+    foreach source $sources {
+        unset -nocomplain quicklist
+        # chop off any tags
+        set source [lindex $source 0]
+        set index [macports::getindex $source]
+        if {![file exists ${index}.quick] || [file mtime ${index}] > [file mtime ${index}.quick]} {
+            # stale or nonexistent quick index file, so generate a new one
+            if {[catch {set quicklist [mports_generate_quickindex ${index}]}]} {
+                continue
+            }
+        }
+        # only need to read the quick index file if we didn't just update it
+        if {![info exists quicklist]} {
+            if {[catch {set fd [open ${index}.quick r]} result]} {
+                ui_warn "Can't open quick index file for source: $source"
+                continue
+            } else {
+                set quicklist [read $fd]
+                close $fd
+            }
+        }
+        foreach entry [split $quicklist "\n"] {
+            set quick_index($sourceno,[lindex $entry 0]) [lindex $entry 1]
+        }
+        incr sourceno 1
+    }
+    if {!$sourceno} {
+        return -code error "No index(es) found! Have you synced your source indexes?"
+    }
+}
+
+proc mports_generate_quickindex {index} {
+    if {[catch {set indexfd [open ${index} r]} result] || [catch {set quickfd [open ${index}.quick w]} result]} {
+        ui_warn "Can't open index file: $index"
+        return -code error
+    } else {
+        try {
+            set offset [tell $indexfd]
+            set quicklist ""
+            while {[gets $indexfd line] >= 0} {
+                if {[llength $line] != 2} {
+                    continue
+                }
+                set name [lindex $line 0]
+                append quicklist "[string tolower $name] ${offset}\n"
+
+                set len [lindex $line 1]
+                seek $indexfd $len current
+                set offset [tell $indexfd]
+            }
+            puts -nonewline $quickfd $quicklist
+        } catch {*} {
+            ui_warn "It looks like your PortIndex file may be corrupt."
+            throw
+        } finally {
+            close $indexfd
+            close $quickfd
+        }
+    }
+    if {[info exists quicklist]} {
+        return $quicklist
+    } else {
+        ui_warn "Failed to generate quick index for: $index"
+        return -code error
+    }
+}
+
 proc mportinfo {mport} {
     set workername [ditem_key $mport workername]
     return [$workername eval array get PortInfo]
@@ -1789 +1953 @@
         set dep_portname [lindex [split $depspec :] end]
         
         # Find the porturl
-        if {[catch {set res [mportsearch $dep_portname false exact]} error]} {
+        if {[catch {set res [mportlookup $dep_portname]} error]} {
             global errorInfo
             ui_debug "$errorInfo"
-            ui_error "Internal error: port search failed: $error"
+            ui_error "Internal error: port lookup failed: $error"
             return 1
         }
         
-        unset -nocomplain porturl
-        foreach {name array} $res {
-            array set portinfo $array
-            if {[info exists portinfo(porturl)]} {
-                set porturl $portinfo(porturl)
-                break
-            }
-        }
-
-        if {![info exists porturl]} {
+        array unset portinfo
+        array set portinfo [lindex $res 1]
+        if {[info exists portinfo(porturl)]} {
+            set porturl $portinfo(porturl)
+        } else {
             ui_error "Dependency '$dep_portname' not found."
             return 1
         }
@@ -1974 +2133 @@
     }
 
     # check if the port is in tree
-    if {[catch {mportsearch $portname false exact} result]} {
+    if {[catch {mportlookup $portname} result]} {
         global errorInfo
         ui_debug "$errorInfo"
-        ui_error "port search failed: $result"
+        ui_error "port lookup failed: $result"
         return 1
     }
     # argh! port doesnt exist!

portmgr/jobs/PortIndexRegen.sh

@@ -127 +127 @@
 echo "" >> $COMMIT_MSG
 grep Failed $FAILURE_LOG >> $COMMIT_MSG
 { cd ${SRCTREE}/dports/ && \
-    $SVN --config-dir $SVN_CONFIG_DIR commit -F $COMMIT_MSG PortIndex > $FAILURE_LOG 2>&1 ; } \
+    $SVN --config-dir $SVN_CONFIG_DIR commit -F $COMMIT_MSG PortIndex PortIndex.quick > $FAILURE_LOG 2>&1 ; } \
     || { echo "SVN commit failed." >> $FAILURE_LOG ; bail ; }
 
 # At this point the index was committed successfuly, so we cleanup before we exit.