Changeset 79093

Timestamp:

06/01/11 21:32:01 (4 years ago)

Author:

jmr@…

Message:

add support for sync and selfupdate via rsync of tarballs with detached signatures (work in progress)

Location:

trunk/base

Files:

• portmgr/jobs/mprsyncup (modified) (3 diffs)
• src/macports1.0/macports.tcl (modified) (8 diffs)
• src/macports1.0/macports_autoconf.tcl.in (modified) (1 diff)

trunk/base/portmgr/jobs/mprsyncup

@@ -39 +39 @@
 MKDIR="/bin/mkdir"
 LN="/bin/ln"
+TAR="/usr/bin/tar"
+OPENSSL="/usr/bin/openssl"
 
 # Paths we'll work on:
@@ -65 +67 @@
 SVNURL=https://svn.macports.org/repository/macports
 RELEASE_URL_FILE=config/RELEASE_URL
+
+# private key to use for signing
+# XXX set real path
+PRIVKEY=""
 
 # cleanup up the working copy if it is locked
@@ -146 +152 @@
     ${RM} -rf dports && ${LN} -s ../release/ports dports
 fi
+
+# generate and sign tarballs of base and dports
+# the signature always needs to match, so we try to make this look atomic to
+# clients by switching a symlink target
+
+TAR_CURDIR=${RSYNCROOT}/release/tarballs_current
+${MKDIR} -p ${TAR_CURDIR}
+cp -pR ${TAR_CURDIR} ${RSYNCROOT}/release/tarballs_old
+${LN} -sf tarballs_old ${RSYNCROOT}/release/tarballs
+${TAR} -cf ${TAR_CURDIR}/base.tar ${RSYNCROOT}/release/base
+${TAR} -cf ${TAR_CURDIR}/ports.tar --exclude 'PortIndex*' ${RSYNCROOT}/release/ports
+cp -pR ${RSYNCROOT}/release/ports/PortIndex_* ${TAR_CURDIR}
+# XXX needs PRIVKEY to be set above
+#${OPENSSL} dgst -ripemd160 -sign ${PRIVKEY} -out ${TAR_CURDIR}/base.tar.rmd160 ${TAR_CURDIR}/base.tar
+#${OPENSSL} dgst -ripemd160 -sign ${PRIVKEY} -out ${TAR_CURDIR}/ports.tar.rmd160 ${TAR_CURDIR}/ports.tar
+for index in ${TAR_CURDIR}/PortIndex_*/PortIndex; do
+    #${OPENSSL} dgst -ripemd160 -sign ${PRIVKEY} -out ${index}.rmd160 ${index}
+done
+${LN} -sf tarballs_current ${RSYNCROOT}/release/tarballs
+${RM} -rf ${RSYNCROOT}/release/tarballs_old

trunk/base/src/macports1.0/macports.tcl

@@ -1861 +1861 @@
     upvar $extension myextension
 
-    if {[regexp {^(?:https?|ftp)://.+/(.+\.(tar\.gz|tar\.bz2))$} $url -> f e]} {
+    if {[regexp {^(?:https?|ftp|rsync)://.+/(.+\.(tar\.gz|tar\.bz2|tar))$} $url -> f e]} {
         set myfilename $f
         set myextension $e
@@ -1905 +1905 @@
     global macports::sources macports::portdbpath macports::rsync_options tcl_platform
     global macports::portverbose
-    global macports::autoconf::rsync_path
+    global macports::autoconf::rsync_path macports::autoconf::tar_path macports::autoconf::openssl_path
     array set options $optionslist
 
@@ -1957 +1957 @@
                 set indexfile [macports::getindex $source]
                 set destdir [file dirname $indexfile]
+                set is_tarball [_source_is_snapshot $source]
                 file mkdir $destdir
-                # Keep rsync happy with a trailing slash
-                if {[string index $source end] != "/"} {
-                    append source "/"
-                }
-                # don't sync PortIndex yet; we grab the platform specific one afterwards
-                set exclude_option "'--exclude=/PortIndex*'"
+
+                if {$is_tarball} {
+                    set exclude_option ""
+                    # need to do a few things before replacing the ports tree in this case
+                    set destdir [file dirname $destdir]
+                } else {
+                    # Keep rsync happy with a trailing slash
+                    if {[string index $source end] != "/"} {
+                        append source "/"
+                    }
+                    # don't sync PortIndex yet; we grab the platform specific one afterwards
+                    set exclude_option "'--exclude=/PortIndex*'"
+                }
                 # Do rsync fetch
                 set rsync_commandline "${macports::autoconf::rsync_path} ${rsync_options} ${exclude_option} ${source} ${destdir}"
@@ -1972 +1980 @@
                     continue
                 }
+
+                if {$is_tarball} {
+                    # verify signature for tarball
+                    global macports::archivefetch_pubkeys
+                    set rsync_commandline "${macports::autoconf::rsync_path} ${rsync_options} ${exclude_option} ${source}.rmd160 ${destdir}"
+                    ui_debug $rsync_commandline
+                    if {[catch {system $rsync_commandline}]} {
+                        ui_error "Synchronization of the ports tree signature failed doing rsync"
+                        incr numfailed
+                        continue
+                    }
+                    set tarball "${destdir}/[file tail $source]"
+                    set signature "${tarball}.rmd160"
+                    set openssl [findBinary openssl $macports::autoconf::openssl_path]
+                    set verified 0
+                    foreach pubkey ${macports::archivefetch_pubkeys} {
+                        if {![catch {exec $openssl dgst -ripemd160 -verify $pubkey -signature $signature $tarball} result]} {
+                            set verified 1
+                            ui_debug "successful verification with key $pubkey"
+                            break
+                        } else {
+                            ui_debug "failed verification with key $pubkey"
+                            ui_debug "openssl output: $result"
+                        }
+                    }
+                    if {!$verified} {
+                        ui_error "Failed to verify signature for ports tree!"
+                        incr numfailed
+                        continue
+                    }
+
+                    # extract tarball and move into place
+                    set tar [macports::findBinary tar $macports::autoconf::tar_path]
+                    file mkdir ${destdir}/tmp
+                    set tar_cmd "$tar -C ${destdir}/tmp -xf ${tarball}"
+                    ui_debug $tar_cmd
+                    if {[catch {system $tar_cmd}]} {
+                        ui_error "Failed to extract ports tree from tarball!"
+                        incr numfailed
+                        continue
+                    }
+                    # save the local PortIndex data
+                    if {[file isfile $indexfile]} {
+                        file rename -force $indexfile ${destdir}/tmp/ports/
+                        if {[file isfile ${indexfile}.quick]} {
+                            file rename -force ${indexfile}.quick ${destdir}/tmp/ports/
+                        }
+                    }
+                    file delete -force ${destdir}/ports
+                    file rename ${destdir}/tmp/ports ${destdir}/ports
+                    file delete -force ${destdir}/tmp
+                }
+
                 # now sync the index if the local file is missing or older than a day
                 if {![file isfile $indexfile] || [expr [clock seconds] - [file mtime $indexfile]] > 86400} {
+                    if {$is_tarball} {
+                        # chop ports.tar off the end
+                        set source [string range $source 0 end-[string length [file tail $source]]]
+                    }
                     set remote_indexfile "${source}PortIndex_${macports::os_platform}_${macports::os_major}_${macports::os_arch}/PortIndex"
                     set rsync_commandline "${macports::autoconf::rsync_path} ${rsync_options} $remote_indexfile ${destdir}"
@@ -1980 +2045 @@
                         ui_debug "Synchronization of the PortIndex failed doing rsync"
                     } else {
-                        mports_generate_quickindex $indexfile
+                        set ok 1
+                        if {$is_tarball} {
+                            set ok 0
+                            # verify signature for PortIndex
+                            set rsync_commandline "${macports::autoconf::rsync_path} ${rsync_options} ${remote_indexfile}.rmd160 ${destdir}"
+                            ui_debug $rsync_commandline
+                            if {![catch {system $rsync_commandline}]} {
+                                foreach pubkey ${macports::archivefetch_pubkeys} {
+                                    if {![catch {exec $openssl dgst -ripemd160 -verify $pubkey -signature ${destdir}/PortIndex.rmd160 ${destdir}/PortIndex} result]} {
+                                        set ok 1
+                                        ui_debug "successful verification with key $pubkey"
+                                        break
+                                    } else {
+                                        ui_debug "failed verification with key $pubkey"
+                                        ui_debug "openssl output: $result"
+                                    }
+                                }
+                                if {$ok} {
+                                    # move PortIndex into place
+                                    file rename -force ${destdir}/PortIndex ${destdir}/ports/
+                                }
+                            }
+                        }
+                        if {$ok} {
+                            mports_generate_quickindex $indexfile
+                        }
                     }
                 }
@@ -2787 +2877 @@
     global macports::prefix macports::portdbpath macports::libpath macports::rsync_server macports::rsync_dir macports::rsync_options
     global macports::autoconf::macports_version macports::autoconf::rsync_path tcl_platform
+    global macports::autoconf::openssl_path macports::autoconf::tar_path
     array set options $optionslist
     
@@ -2803 +2894 @@
     }
 
+    # are we syncing a tarball? (implies detached signature)
+    set is_tarball 0
+    if {[string range ${rsync_dir} end-3 end] == ".tar"} {
+        set is_tarball 1
+        set mp_source_path [file join $portdbpath sources ${rsync_server} [file dirname ${rsync_dir}]]
+    } else {
+        if {[string index $rsync_dir end] != "/"} {
+            append rsync_dir "/"
+        }
+        set mp_source_path [file join $portdbpath sources ${rsync_server} ${rsync_dir}]
+    }
     # create the path to the to be downloaded sources if it doesn't exist
-    set mp_source_path [file join $portdbpath sources ${rsync_server} ${rsync_dir}/]
     if {![file exists $mp_source_path]} {
         file mkdir $mp_source_path
@@ -2814 +2915 @@
     if { [catch { system "$rsync_path $rsync_options rsync://${rsync_server}/${rsync_dir} $mp_source_path" } result ] } {
        return -code error "Error synchronizing MacPorts sources: $result"
+    }
+
+    if {$is_tarball} {
+        # verify signature for tarball
+        global macports::archivefetch_pubkeys
+        if { [catch { system "$rsync_path $rsync_options rsync://${rsync_server}/${rsync_dir}.rmd160 $mp_source_path" } result ] } {
+            return -code error "Error synchronizing MacPorts source signature: $result"
+        }
+        set openssl [findBinary openssl $macports::autoconf::openssl_path]
+        set tarball "${mp_source_path}/[file tail $rsync_dir]"
+        set signature "${tarball}.rmd160"
+        set verified 0
+        foreach pubkey ${macports::archivefetch_pubkeys} {
+            if {![catch {exec $openssl dgst -ripemd160 -verify $pubkey -signature $signature $tarball} result]} {
+                set verified 1
+                ui_debug "successful verification with key $pubkey"
+                break
+            } else {
+                ui_debug "failed verification with key $pubkey"
+                ui_debug "openssl output: $result"
+            }
+        }
+        if {!$verified} {
+            return -code error "Failed to verify signature for MacPorts source!"
+        }
+        
+        # extract tarball and move into place
+        set tar [macports::findBinary tar $macports::autoconf::tar_path]
+        file mkdir ${mp_source_path}/tmp
+        set tar_cmd "$tar -C ${mp_source_path}/tmp -xf ${tarball}"
+        ui_debug $tar_cmd
+        if {[catch {system $tar_cmd}]} {
+            return -code error "Failed to extract MacPorts sources from tarball!"
+        }
+        file delete -force ${mp_source_path}/base
+        file rename ${mp_source_path}/tmp/base ${mp_source_path}/base
+        file delete -force ${mp_source_path}/tmp
+        # set the final extracted source path
+        set mp_source_path ${mp_source_path}/base
     }
 

trunk/base/src/macports1.0/macports_autoconf.tcl.in

@@ -41 +41 @@
     variable macportsuser "@RUNUSR@"
     variable open_path "@OPEN@"
+    variable openssl_path "@OPENSSL@"
     variable pax_path "@PAX@"
     variable rsync_path "@RSYNC@"