source: trunk/base/portmgr/PortIndex2MySQL.tcl @ 29342

Last change on this file since 29342 was 29342, checked in by jmpp@…, 12 years ago

Proper error detection and reporting comes to the PortIndex2MySQL script, with macports1.0
ui initialization to catch the messages and direct them to wherever we want, finally!
Currently messages are logged to a runlog file and then piped to mail(1) on exit if any
failure occurs, so that the recipient is hinted at what went wrong in the db update job.

On a related note, the script was somewhat reorganized to make room for the error detection
& reporting to work throughout and with a new cleanup proc that handles housekeeping on exit.

This commit will finally let us run this script unattended off cron/launchd so that the ports.php
page is updated on an automated basis (and reliably ;-), woot!

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 11.7 KB
Line 
1#!/usr/bin/env tclsh
2# -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:et:sw=4:ts=4:sts=4
3#
4# PortIndex2MySQL.tcl
5# Kevin Van Vechten | kevin@opendarwin.org
6# 3-Oct-2002
7# Juan Manuel Palacios | jmpp@macports.org
8# 30-Jul-2007
9# $Id: PortIndex2MySQL.tcl 29342 2007-09-21 18:05:40Z jmpp@macports.org $
10#
11# Copyright (c) 2007 Juan Manuel Palacios, MacPorts Team.
12# Copyright (c) 2003 Apple Computer, Inc.
13# Copyright (c) 2002 Kevin Van Vechten.
14# All rights reserved.
15#
16# Redistribution and use in source and binary forms, with or without
17# modification, are permitted provided that the following conditions
18# are met:
19# 1. Redistributions of source code must retain the above copyright
20#    notice, this list of conditions and the following disclaimer.
21# 2. Redistributions in binary form must reproduce the above copyright
22#    notice, this list of conditions and the following disclaimer in the
23#    documentation and/or other materials provided with the distribution.
24# 3. Neither the name of Apple Computer, Inc. nor the names of its contributors
25#    may be used to endorse or promote products derived from this software
26#    without specific prior written permission.
27#
28# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
32# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38# POSSIBILITY OF SUCH DAMAGE.
39
40#####
41# The PortIndex2MySQL script populates a database with key information extracted
42# from the Portfiles in the ports tree pointed to by the sources.conf file in a
43# MacPorts installation, found by loading its macports1.0 tcl package and initializing
44# it with 'mportinit' below. Main use of the resulting database is providing live
45# information to the ports.php page, a client tailored to poll it. For this very reason,
46# information fed to the database always has to be kept up to date in order to remain
47# meaningful, which is accomplished simply by calling 'macports::selfupdate' (which
48# updates the ports tree in use) and by installing the script on cron/launchd to be run
49# on a timely schedule (not any more frequent than the run of the mprsyncup script on
50# the MacPorts server, which is every half hour).
51#
52# Remaining requirement to successfully run this script is performing the necessary
53# MySQL admin tasks on the host box to create the database in the first place and the
54# MySQL user that will be given enough privileges to alter it. Values in the database
55# related variables provided below have to be adapted accordingly to match the chosen
56# setup.
57#####
58
59
60# Load macports1.0 so that we can use some of its procs and the portinfo array.
61catch {source \
62    [file join "@TCL_PACKAGE_DIR@" macports1.0 macports_fastload.tcl]}
63package require macports
64
65
66# Runtime information log file and reciepient.
67set runlog "/tmp/portsdb.log"
68set runlog_fd [open $runlog w+]
69set lockfile "/tmp/portsdb.lock"
70set DATE [clock format [clock seconds] -format "%A %Y-%m-%d at %T"]
71set subject "PortIndex2MySQL run failure on $DATE"
72set SPAM_LOVERS macports-dev@lists.macosforge.org
73
74# House keeping on exit.
75proc cleanup {} {
76    global sqlfile sqlfile_fd
77    global lockfile lockfile_fd
78    close $sqlfile_fd
79    close $lockfile_fd
80    file delete -force $sqlfile $lockfile
81}
82
83# What to do when terminating execution, depending on the $exit_status condition.
84proc terminate {exit_status} {
85    global runlog runlog_fd
86    global subject SPAM_LOVERS
87    if {$exit_status} {
88        seek $runlog_fd 0 start
89        exec -- mail -s $subject $SPAM_LOVERS <@ $runlog_fd
90    }
91    close $runlog_fd
92    file delete -force $runlog
93    exit $exit_status
94}
95
96
97# UI instantiation to route information/error messages wherever we want.
98proc ui_channels {priority} {
99    global ui_options runlog_fd
100    switch $priority {
101        debug {
102            if {[macports::ui_isset ui_options ports_debug]} {
103                return $runlog_fd
104            } else {
105                return {}
106            }
107        }
108        info {
109            if {[macports::ui_isset ui_options ports_verbose]} {
110                return $runlog_fd
111            } else {
112                return {}
113            }
114        }
115        msg {
116            if {[macports::ui_isset ui_options ports_quiet]} {
117                return $runlog_fd
118            } else {
119                return {}
120            }
121        }
122        error {
123            return $runlog_fd
124        }
125        default {
126            return {}
127        }
128    }
129}
130
131
132# Check if there are any stray sibling jobs before moving on, bail in such case.
133if {[file exists $lockfile]} {
134    ui_error "PortIndex2MySQL lock file found, is another job running?"
135    terminate 1
136} else {
137    set lockfile_fd [open $lockfile a]
138}
139
140
141# Initialize macports1.0 and its UI, in order to find the sources.conf file
142# (which is what will point us to the PortIndex we're gonna use) and use
143# the runtime information.
144array set ui_options {ports_verbose yes}
145if {[catch {mportinit ui_options} errstr]} {
146    ui_error "${::errorInfo}"
147    ui_error "Failed to initialize MacPorts, $errstr"
148    terminate 1
149}
150
151# Call the selfupdate procedure to make sure the MacPorts installation
152# is up-to-date and with a fresh ports tree.
153if {[catch {macports::selfupdate} errstr]} {
154    ui_error "${::errorInfo}"
155    ui_error "Failed to update the ports tree, $errstr"
156    terminate 1
157}
158
159
160# Procedure to catch the database password from a protected file.
161proc getpasswd {passwdfile} {
162    if {[catch {open $passwdfile r} passwdfile_fd]} {
163        ui_error "${::errorCode}: $passwdfile_fd"
164        terminate 1
165    }
166    if {[gets $passwdfile_fd passwd] <= 0} {
167        close $passwdfile_fd
168        ui_error "No password found in $passwdfile!"
169        terminate 1
170    }
171    close $passwdfile_fd
172    return $passwd
173}
174
175# Database abstraction variables:
176set sqlfile "/tmp/portsdb.sql"
177set dbcmd [macports::findBinary mysql5]
178set dbhost 127.0.0.1
179set dbuser macports
180set passwdfile "./password_file"
181set dbpasswd [getpasswd $passwdfile]
182set dbname macports_ports
183
184
185# Flat text file to which sql statements are written.
186if {[catch {open $sqlfile w+} sqlfile_fd]} {
187    ui_error "${::errorCode}: $sqlfile_fd"
188    terminate 1
189}
190
191# SQL string escaping.
192proc sql_escape {str} {
193    regsub -all -- {'} $str {\\'} str
194    regsub -all -- {"} $str {\\"} str
195    regsub -all -- {\n} $str {\\n} str
196    return $str
197}
198
199# Initial creation of database tables: log, portfiles, categories, maintainers, dependencies, variants and platforms.
200# Do we need any other?
201puts $sqlfile_fd "DROP TABLE IF EXISTS log;"
202puts $sqlfile_fd "CREATE TABLE IF NOT EXISTS log (activity VARCHAR(255), activity_time TIMESTAMP(14));"
203puts $sqlfile_fd "INSERT INTO log VALUES ('update', NOW());"
204
205puts $sqlfile_fd "DROP TABLE IF EXISTS portfiles;"
206puts $sqlfile_fd "CREATE TABLE portfiles (name VARCHAR(255) PRIMARY KEY NOT NULL, path VARCHAR(255), version VARCHAR(255),  description TEXT);"
207
208puts $sqlfile_fd "DROP TABLE IF EXISTS categories;"
209puts $sqlfile_fd "CREATE TABLE categories (portfile VARCHAR(255), category VARCHAR(255), is_primary INTEGER);"
210
211puts $sqlfile_fd "DROP TABLE IF EXISTS maintainers;"
212puts $sqlfile_fd "CREATE TABLE maintainers (portfile VARCHAR(255), maintainer VARCHAR(255), is_primary INTEGER);"
213
214puts $sqlfile_fd "DROP TABLE IF EXISTS dependencies;"
215puts $sqlfile_fd "CREATE TABLE dependencies (portfile VARCHAR(255), library VARCHAR(255));"
216
217puts $sqlfile_fd "DROP TABLE IF EXISTS variants;"
218puts $sqlfile_fd "CREATE TABLE variants (portfile VARCHAR(255), variant VARCHAR(255));"
219
220puts $sqlfile_fd "DROP TABLE IF EXISTS platforms;"
221puts $sqlfile_fd "CREATE TABLE platforms (portfile VARCHAR(255), platform VARCHAR(255));"
222
223
224# Load every port in the index through a search that matches everything.
225if {[catch {set ports [mportsearch ".+"]} errstr]} {
226    ui_error "${::errorInfo}"
227    ui_error "port search failed: $errstr"
228    cleanup
229    terminate 1
230}
231
232# Iterate over each matching port, extracting its information from the
233# portinfo array.
234foreach {name array} $ports {
235
236    array unset portinfo
237    array set portinfo $array
238
239    set portname [sql_escape $portinfo(name)]
240    if {[info exists portinfo(version)]} {
241        set portversion [sql_escape $portinfo(version)]
242    } else {
243        set portversion ""
244    }
245    set portdir [sql_escape $portinfo(portdir)]
246    if {[info exists portinfo(description)]} {
247        set description [sql_escape $portinfo(description)]
248    } else {
249        set description ""
250    }
251    if {[info exists portinfo(categories)]} {
252        set categories $portinfo(categories)
253    } else {
254        set categories ""
255    }
256    if {[info exists portinfo(maintainers)]} {
257        set maintainers $portinfo(maintainers)
258    } else {
259        set maintainers ""
260    }
261    if {[info exists portinfo(variants)]} {
262        set variants $portinfo(variants)
263    } else {
264        set variants ""
265    }
266    if {[info exists portinfo(depends_build)]} {
267        set depends_build $portinfo(depends_build)
268    } else {
269        set depends_build ""
270    }
271    if {[info exists portinfo(depends_lib)]} {
272        set depends_lib $portinfo(depends_lib)
273    } else {
274        set depends_lib ""
275    }
276    if {[info exists portinfo(depends_run)]} {
277        set depends_run $portinfo(depends_run)
278    } else {
279        set depends_run ""
280    }
281    if {[info exists portinfo(platforms)]} {
282        set platforms $portinfo(platforms)
283    } else {
284        set platforms ""
285    }
286
287    puts $sqlfile_fd "INSERT INTO portfiles VALUES ('$portname', '$portdir', '$portversion', '$description');"
288
289    set primary 1
290    foreach category $categories {
291        set category [sql_escape $category]
292        puts $sqlfile_fd "INSERT INTO categories VALUES ('$portname', '$category', $primary);"
293        incr primary
294    }
295   
296    set primary 1
297    foreach maintainer $maintainers {
298        set maintainer [sql_escape $maintainer]
299        puts $sqlfile_fd "INSERT INTO maintainers VALUES ('$portname', '$maintainer', $primary);"
300        incr primary
301    }
302
303    foreach build_dep $depends_build {
304        set build_dep [sql_escape $build_dep]
305        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$build_dep');"
306    }
307
308    foreach lib $depends_lib {
309        set lib [sql_escape $lib]
310        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$lib');"
311    }
312
313    foreach run_dep $depends_run {
314        set run_dep [sql_escape $run_dep]
315        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$run_dep');"
316    }
317
318    foreach variant $variants {
319        set variant [sql_escape $variant]
320        puts $sqlfile_fd "INSERT INTO variants VALUES ('$portname', '$variant');"
321    }
322
323    foreach platform $platforms {
324        set platform [sql_escape $platform]
325        puts $sqlfile_fd "INSERT INTO platforms VALUES ('$portname', '$platform');"
326    }
327
328}
329
330
331# Pipe the contents of the generated sql file to the database command,
332# reading from the file descriptor for the raw sql file to assure completeness.
333if {[catch {seek $sqlfile_fd 0 start} errstr]} {
334    ui_error "${::errorCode}: $errstr"
335    cleanup
336    terminate 1
337}
338if {[catch {exec -- $dbcmd --host=$dbhost --user=$dbuser --password=$dbpasswd --database=$dbname <@ $sqlfile_fd} errstr]} {
339    ui_error "${::errorCode}: $errstr"
340    cleanup
341    terminate 1
342}
343
344
345# And we're done regen'ing the MacPorts dabase! Cleanup and exit successfully.
346cleanup
347terminate 0
Note: See TracBrowser for help on using the repository browser.