Ticket #645: dp-mirrorlists.2.diff

base/src/port1.0

@@ -39 +39 @@
 target_prerun ${com.apple.fetch} fetch_start
 
 # define options: distname master_sites
-options master_sites patch_sites extract.sufx distfiles patchfiles use_zip use_bzip2 dist_subdir fetch.type cvs.module cvs.root cvs.password cvs.tag master_sites.subdir master_sites.listfile master_sites.listpath
+options master_sites patch_sites extract.sufx distfiles patchfiles use_zip use_bzip2 dist_subdir fetch.type cvs.module cvs.root cvs.password cvs.tag mirror_sites.listfile mirror_sites.listpath master_sites.mirror_subdir patch_sites.mirror_subdir
 # XXX we use the command framework to buy us some useful features,
 # but this is not a user-modifiable command
 commands cvs
@@ -62 +62 @@
 default fetch.pre_args ""
 default fetch.post_args {"${site}${distfile}"}
 
-default master_sites.listfile {"master_sites.tcl"}
-default master_sites.listpath {"${portresourcepath}/sitelists/"}
+default mirror_sites.listfile {"mirror_sites.tcl"}
+default mirror_sites.listpath {"${portresourcepath}/fetch/"}
 
 # Set distfiles
 default distfiles {[suffix $distname]}
@@ -137 +137 @@
 
 # Expand all variable references in each site variable, passing back a new 
 # expanded list
-proc expand-site-vars {sites} {
+proc expand-site-vars {mirrors subdir tag} {
     set x [list]
-    foreach element $sites {
+    foreach element $mirrors {
+        eval append element "${subdir}/${tag}"
         eval lappend x $element
     }
     return $x
 }
 
-# For a given master site type, e.g. "gnu" or "x11", check to see if there's a
+# For a given mirror site type, e.g. "gnu" or "x11", check to see if there's a
 # pre-registered set of sites, and if so, return them.
-proc master-sites-for {arg} {
-    global UI_PREFIX portresourcepath master_sites.listfile master_sites.listpath
-    include ${master_sites.listpath}${master_sites.listfile}
-    if ![info exists _master_sites($arg)] {
-        ui_msg "$UI_PREFIX [format [msgcat::mc "No master sites on file for class %s"] $arg]"
+proc mirror-sites-for {mirrors subdir tag} {
+    global UI_PREFIX portresourcepath mirror_sites.listfile mirror_sites.listpath
+    include ${mirror_sites.listpath}${mirror_sites.listfile}
+    if ![info exists _mirror_sites($mirrors)] {
+        ui_msg "$UI_PREFIX [format [msgcat::mc "No mirror sites on file for class %s"] $mirrors]"
         return {}
     }
-    return [expand-site-vars $_master_sites($arg)]
+    return [expand-site-vars $_mirror_sites($mirrors) $subdir $tag]
 }
 
 # Checks all files and their tags to assemble url lists for later fetching
@@ -164 +165 @@
 # sites
 proc checkfiles {args} {
     global distdir distfiles patchfiles all_dist_files patch_sites fetch_urls \
-            master_sites master_sites.subdir filespath
+            master_sites filespath master_sites.mirror_subdir \
+        patch_sites.mirror_subdir
 
     foreach list {master_sites patch_sites} {
         upvar #0 $list uplist
@@ -172 +174 @@
             continue
         }
         
-        # There should be a better way to get rid of the extra list created by 
-        # master-sites-for (the extra {}'s, one element in $uplist)
         set site_list [list]
         foreach site $uplist {
+            if {[regexp {([a-zA-Z]+://.+)} $site match site] == 1} {
             set site_list [concat $site_list $site]
+            } else {
+                if {[regexp {([a-zA-Z]+)[:]?([a-zA-Z/]+)?(:([a-zA-Z]+))?} $site match mirrors subdir tag] == 1} {
+                    if [info exists $list.mirror_subdir] {
+                        append subdir "/[set ${list}.mirror_subdir]"
+                    }
+                    set site_list [concat $site_list [mirror-sites-for $mirrors $subdir $tag]]
+                } else {
+                    ui_error [format [msgcat::mc "Unable to process mirror sites for: %s, ignoring."] $site]
+                }
+            }
         }
         
         foreach site $site_list {
-            if [info exists master_sites.subdir] {
-                eval append site ${master_sites.subdir}
-            }
-            if {[regexp {([a-zA-Z]+://.+/):([a-zA-z]+)} $site match site tag] == 1} {
+            if {[regexp {([a-zA-Z]+://.+/):([a-zA-Z]+)} $site match site tag] == 1} {
                 lappend portfetch::$tag $site
             } else {
                 lappend portfetch::$list $site

resources/Makefile

@@ -1 @@
-SUBDIR=         package sitelists
+SUBDIR=         package fetch
 INSTALLDIR=     ${DESTDIR}${datadir}/darwinports/resources/port1.0/
 RSRCS=
 

resources/fetch/Makefile

@@ +1 @@
+INSTALLDIR=     ${DESTDIR}${prefix}/share/darwinports/resources/port1.0/fetch
+
+RSRCS=          mirror_sites.tcl
+
+include ../../../../Mk/dports.autoconf.mk
+
+all:
+
+clean:
+
+distclean: clean
+
+install:
+        $(INSTALL) -d -o ${DSTUSR} -g ${DSTGRP} ${INSTALLDIR}
+        $(SILENT)set -x; for file in ${RSRCS}; do \
+                $(INSTALL) -o ${DSTUSR} -g ${DSTGRP} -m 444 $$file ${INSTALLDIR}; \
+        done

resources/fetch/mirror_sites.tcl

@@ +1 @@
+# $Id: $
+# mirror_sites.tcl
+#
+# List of master site classes for use in Portfiles
+# Most of these are taken shamelessly from FreeBSD.
+#
+# Please keep this list sorted.
+
+set _mirror_sites(apache) {
+    http://www.apache.org/dist/
+    ftp://ftp.planetmirror.com/pub/apache/dist/
+    ftp://apache.mirrored.ca/
+    ftp://ftp.is.co.za/Apache/dist/
+    http://apache.planetmirror.com.au/dist/
+    ftp://ftp.leo.org/pub/comp/general/infosys/www/daemons/apache/dist/
+    ftp://ftp.infoscience.co.jp/pub/net/apache/dist/
+}
+
+set _mirror_sites(gnome) {
+    http://archive.progeny.com/GNOME/
+    http://ftp.rpmfind.net/linux/gnome.org/
+    ftp://ftp.mirror.ac.uk/sites/ftp.gnome.org/pub/GNOME/
+    http://fr.rpmfind.net/linux/gnome.org/
+    http://ftp.linux.org.uk/mirrors/ftp.gnome.org/
+    http://planetmirror.com/pub/GNOME/
+    ftp://ftp.dit.upm.es/linux/gnome/
+    ftp://ftp.dataplus.se/pub/linux/gnome/
+    ftp://ftp.chg.ru/pub/X11/gnome/
+    ftp://ftp.kddlabs.co.jp/pub/GNOME/
+    ftp://ftp.dti.ad.jp/pub/X/gnome/
+    http://mirror.aarnet.edu.au/pub/GNOME/
+    ftp://ftp.cse.buffalo.edu/pub/Gnome/
+}
+
+set _mirror_sites(gnu) {
+    ftp://ftp.gnu.org/gnu/
+    ftp://gatekeeper.dec.com/pub/GNU/
+    ftp://ftp.uu.net/archive/systems/gnu/
+    ftp://ftp.de.uu.net/pub/gnu/
+    ftp://ftp.funet.fi/pub/gnu/prep/
+    ftp://ftp.leo.org/pub/comp/os/unix/gnu/
+    ftp://ftp.digex.net/pub/gnu/
+    ftp://ftp.wustl.edu/mirrors/gnu/
+    ftp://ftp.kddlabs.co.jp/pub/gnu/
+    ftp://ftp.dti.ad.jp/pub/GNU/
+    ftp://ftp.mirror.ac.uk/sites/ftp.gnu.org/gnu/
+    ftp://sunsite.org.uk/Mirrors/ftp.gnu.org/pub/gnu/
+    ftp://ftp.informatik.hu-berlin.de/pub/gnu/
+    ftp://ftp.rediris.es/mirror/gnu/
+    ftp://ftp.lip6.fr/pub/gnu/
+    ftp://ftp.chg.ru/pub/gnu/
+}
+
+set _mirror_sites(isc) {
+    ftp://ftp.isc.org/isc/
+    ftp://gd.tuwien.ac.at/infosys/servers/isc/
+    ftp://ftp.ciril.fr/pub/isc/
+    ftp://ftp.grolier.fr/pub/isc/
+    ftp://ftp.funet.fi/pub/mirrors/ftp.isc.org/isc/
+    ftp://ftp.freenet.de/pub/ftp.isc.org/isc/
+    ftp://ftp.fsn.hu/pub/isc/
+    ftp://ftp.kyushu-u.ac.jp/pub/Net/isc/
+    ftp://ftp.iij.ad.jp/pub/network/isc/
+    ftp://ftp.dti.ad.jp/pub/net/isc/
+    ftp://ftp.u-aizu.ac.jp/pub/net/isc/
+    ftp://ftp.linux.lv/pub/software/isc/
+    ftp://ftp.task.gda.pl/mirror/ftp.isc.org/isc/
+    ftp://ftp.cdu.elektra.ru/pub/unix/isc/
+    ftp://ftp.si.uniovi.es/mirror/isc/
+    ftp://ftp.sunet.se/pub/network/isc/
+    ftp://ftp.chl.chalmers.se/pub/unix/network/isc/
+    ftp://unix.hensa.ac.uk/mirrors/ftp.isc.org/isc/
+    ftp://ftp.epix.net/pub/isc/
+    ftp://ftp.nominum.com/pub/isc/
+    ftp://ftp.nerdc.ufl.edu/pub/mirrors/ftp.isc.org/isc/
+    ftp://ftp.ripe.net/mirrors/sites/ftp.isc.org/isc/
+    ftp://ftp.nl.uu.net/pub/mirrors/ftp.isc.org/
+    ftp://ftp.pop-mg.com.br/pub/isc/
+    ftp://ftp.ntua.gr/pub/net/isc/isc/
+    ftp://ftp.ulak.net.tr/pub/networking/isc/
+    ftp://ftp.metu.edu.tr/pub/mirrors/ftp.isc.org/
+}
+
+set _mirror_sites(kde) {
+    ftp://ftp.us.kde.org/pub/kde/
+    ftp://ftp.kde.org/pub/kde/
+    http://linux.dsi.internet2.edu/X11/kde/
+    ftp://ftp.tuniv.szczecin.pl/pub/kde/
+    ftp://ftp.kddlabs.co.jp/X11/kde/
+    ftp://ftp.mirror.ac.uk/sites/ftp.kde.org/pub/kde/
+    ftp://ftp.dataplus.se/pub/linux/kde/
+    ftp://ftp.informatik.hu-berlin.de/pub/Mirrors/ftp.kde.org/
+    ftp://ftp.chg.ru/pub/X11/kde/
+}
+
+set _mirror_sites(perl_cpan) {
+    http://www.cpan.dk/CPAN/modules/by-module/
+    ftp://ftp.funet.fi/pub/languages/perl/CPAN/modules/by-module/
+    ftp://ftp.cpan.org/pub/CPAN/modules/by-module/
+    ftp://uiarchive.uiuc.edu/pub/ftp/cpan.cse.msu.edu/modules/by-module/
+    ftp://ftp.kddlabs.co.jp/lang/perl/CPAN/modules/by-module/
+    ftp://ftp.dti.ad.jp/pub/lang/CPAN/modules/by-module/
+    ftp://ftp.sunet.se/pub/lang/perl/CPAN/modules/by-module/
+    ftp://mirror.hiwaay.net/CPAN/modules/by-module/
+    ftp://ftp.bora.net/pub/CPAN/modules/by-module/
+    ftp://ftp.mirror.ac.uk/sites/ftp.funet.fi/pub/languages/perl/CPAN/modules/by-module/
+    ftp://bioinfo.weizmann.ac.il/pub/software/perl/CPAN/modules/by-module/
+    ftp://csociety-ftp.ecn.purdue.edu/pub/CPAN/modules/by-module/
+    ftp://ftp.auckland.ac.nz/pub/perl/CPAN/modules/by-module/
+    ftp://ftp.isu.net.sa/pub/CPAN/modules/by-module/
+    ftp://ftp.ucr.ac.cr/pub/Unix/CPAN/modules/by-module/
+    ftp://ftp.cs.colorado.edu/pub/perl/CPAN/modules/by-module/
+    ftp://cpan.pop-mg.com.br/pub/CPAN/modules/by-module/
+    ftp://ftp.is.co.za/programming/perl/CPAN/modules/by-module/
+    http://cpan.shellhung.org/modules/by-module/
+    ftp://sunsite.org.uk/packages/perl/CPAN/modules/by-module/
+    ftp://ftp.chg.ru/pub/lang/perl/CPAN/modules/by-module/
+}
+
+set _mirror_sites(ruby) {
+    ftp://xyz.lcs.mit.edu/pub/ruby/
+    http://www.ibiblio.org/pub/languages/ruby/
+    ftp://ftp.iij.ad.jp/pub/lang/ruby/
+    ftp://ftp.ruby-lang.org/pub/ruby/
+    ftp://ftp.fu-berlin.de/unix/languages/ruby/
+    ftp://ftp.easynet.be/ruby/ruby/
+    ftp://ftp.ntua.gr/pub/lang/ruby/
+    ftp://ftp.chg.ru/pub/lang/ruby/
+    ftp://ftp.kr.FreeBSD.org/pub/ruby/
+    http://mirrors.sunsite.dk/ruby/
+    ftp://ftp.iDaemons.org/pub/mirror/ftp.ruby-lang.org/ruby/
+}
+
+set _mirror_sites(sourceforge) {
+    http://us.dl.sourceforge.net/
+    http://eu.dl.sourceforge.net/
+    ftp://us.dl.sourceforge.net/pub/sourceforge/
+    ftp://ftp.kddlabs.co.jp/sourceforge/
+    ftp://ftp.chg.ru/pub/sourceforge/
+}
+
+set _mirror_sites(sunsite) {
+    http://www.ibiblio.org/pub/Linux/
+    http://linux.dsi.internet2.edu/
+    ftp://ftp.unicamp.br/pub/systems/Linux/
+    ftp://ftp.tuwien.ac.at/pub/linux/ibiblio/
+    ftp://sunsite.cnlab-switch.ch/mirror/linux/sunsite/
+    ftp://ftp.cs.tu-berlin.de/pub/linux/Mirrors/sunsite.unc.edu/
+    ftp://ftp.fu-berlin.de/unix/linux/mirrors/sunsite.unc.edu/
+    ftp://ftp.uni-jena.de/pub/linux/MIRROR.sunsite/
+    ftp://ftp.rz.uni-karlsruhe.de/pub/mirror/ftp.uni-tuebingen.de/pub/linux/mirrors/ftp.metalab.unc.edu/pub/Linux/
+    ftp://ftp.uni-magdeburg.de/pub/mirror/linux/ftp.metalab.unc.edu/
+    ftp://ftp.uni-stuttgart.de/pub/mirror/sunsite.unc.edu/pub/Linux/
+    ftp://ftp.informatik.rwth-aachen.de/pub/comp/Linux/sunsite.unc.edu/
+    ftp://ftp.lip6.fr/pub/linux/sunsite/
+    ftp://ftp.uvsq.fr/pub5/linux/sunsite/
+    ftp://ftp.leidenuniv.nl/pub/linux/sunsite/
+    ftp://ftp.nvg.ntnu.no/pub/mirrors/metalab.unc.edu/
+    ftp://ftp.icm.edu.pl/vol/rzm1/linux-sunsite/
+    ftp://ftp.pku.edu.cn/pub/linux/
+    ftp://ftp.cse.cuhk.edu.hk/pub4/Linux/
+    ftp://ftp.kobe-u.ac.jp/pub/Linux/metalab.unc.edu/
+    ftp://ftp.kddlabs.co.jp/Linux/metalab.unc.edu/
+    ftp://ftp.dacom.co.kr/pub/mirrors/metalab.unc.edu/Linux/
+    ftp://ftp.funet.fi/pub/Linux/mirrors/metalab/
+    ftp://sunsite.doc.ic.ac.uk/packages/linux/sunsite.unc-mirror/
+    ftp://ftp.is.co.za/linux/sunsite/
+    ftp://ftp.chg.ru/pub/Linux/sunsite/
+}
+
+set _mirror_sites(tcltk) {
+    ftp://ftp.scriptics.com/pub/tcl/
+    ftp://sunsite.utk.edu/pub/tcl/
+    ftp://ftp.funet.fi/pub/languages/tcl/tcl/
+    ftp://ftp.kddlabs.co.jp/lang/tcl/ftp.scriptics.com/
+    ftp://ftp.srcc.msu.su/mirror/ftp.scriptics.com/pub/tcl/
+    ftp://ftp.mirror.ac.uk/sites/ftp.scriptics.com/pub/tcl/
+    ftp://sunsite.org.uk/Mirrors/ftp.scriptics.com/pub/tcl/
+}
+
+set _mirror_sites(xcontrib) {
+    ftp://crl.dec.com/pub/X11/contrib/
+    ftp://ftp.net.ohio-state.edu/disk/d/X11/contrib/
+    ftp://uiarchive.uiuc.edu/pub/ftp/ftp.x.org/contrib/
+    ftp://ftp.gwdg.de/pub/x11/x.org/contrib/
+    ftp://ftp.duke.edu/pub/X11/contrib/
+    ftp://ftp.x.org/contrib/
+    ftp://ftp.sunet.se/pub/X11/contrib/
+    ftp://ftp.dti.ad.jp/pub/X/XFree86/X.Org/contrib/
+    ftp://ftp.kddlabs.co.jp/X11/contrib/
+    ftp://mirror.xmission.com/X/contrib/
+    ftp://ftp2.x.org/contrib/
+    ftp://sunsite.tus.ac.jp/pub/archives/X11/contrib/
+    ftp://ftp.is.co.za/x/contrib/
+    ftp://ftp.sunet.se/pub/X11/contrib/
+    ftp://ftp.mirror.ac.uk/sites/ftp.x.org/contrib/
+    ftp://ftp.dl.ac.uk/src/X/contrib/
+    ftp://sunsite.org.uk/Mirrors/ftp.x.org/contrib/
+    ftp://ftp.chg.ru/pub/X11/contrib/
+}
+
+set _mirror_sites(xfree) {
+    ftp://ftp.xfree86.org/pub/XFree86/source/
+    ftp://ftp.rge.com/pub/X/XFree86/source/
+    ftp://archive.progeny.com/XFree86/source/
+    ftp://ftp.mirrorcentral.com/pub/XFree86/source/
+    ftp://ftp.dti.ad.jp/pub/X/XFree86/XFree86/source/
+    ftp://gd.tuwien.Ac.at/hci/X11/XFree86/source/
+    ftp://ftp.fit.vutbr.cz/pub/XFree86/source/
+    ftp://ftp.free.fr/pub/XFree86/source/
+    ftp://mir1.ovh.net/ftp.xfree86.org/source/
+    ftp://ftp.lami.univ-evry.fr/XFree86/source/
+    ftp://ftp.cs.tu-berlin.de/pub/X/XFree86/source/
+    ftp://ftp.gwdg.de/pub/xfree86/XFree86/source/
+    http://ftp-stud.fht-esslingen.de/pub/Mirrors/ftp.xfree86.org/XFree86/source/
+    ftp://ftp.rediris.es/mirror/XFree86/source/
+    ftp://ftp.esat.net/pub/X11/XFree86/source/
+    ftp://ftp.nl.uu.net/pub/XFree86/source/
+    ftp://sunsite.uio.no/pub/XFree86/source/
+    ftp://ftp.task.gda.pl/pub/XFree86/source/
+    ftp://ftp.physics.uvt.ro/pub/XFree86/source/
+    ftp://ftp.chg.ru/pub/XFree86/source/
+}