source: branches/images-and-archives/base/portmgr/jobs/PortIndex2MySQL.tcl @ 51815

Last change on this file since 51815 was 51815, checked in by blb@…, 8 years ago

Merge from trunk

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 13.7 KB
Line 
1#!/opt/local/bin/tclsh
2# -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:filetype=tcl:et:sw=4:ts=4:sts=4
3#
4# PortIndex2MySQL.tcl
5# Kevin Van Vechten | kevin@opendarwin.org
6# 3-Oct-2002
7# Juan Manuel Palacios | jmpp@macports.org
8# 22-Nov-2007
9# $Id: PortIndex2MySQL.tcl 51815 2009-06-04 05:16:54Z blb@macports.org $
10#
11# Copyright (c) 2007 Juan Manuel Palacios, The MacPorts Project.
12# Copyright (c) 2003 Apple Computer, Inc.
13# Copyright (c) 2002 Kevin Van Vechten.
14# All rights reserved.
15#
16# Redistribution and use in source and binary forms, with or without
17# modification, are permitted provided that the following conditions
18# are met:
19# 1. Redistributions of source code must retain the above copyright
20#    notice, this list of conditions and the following disclaimer.
21# 2. Redistributions in binary form must reproduce the above copyright
22#    notice, this list of conditions and the following disclaimer in the
23#    documentation and/or other materials provided with the distribution.
24# 3. Neither the name of Apple Computer, Inc. nor the names of its contributors
25#    may be used to endorse or promote products derived from this software
26#    without specific prior written permission.
27#
28# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
32# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38# POSSIBILITY OF SUCH DAMAGE.
39
40
41#####
42# The PortIndex2MySQL script populates a database with key information extracted
43# from the Portfiles in the ports tree pointed to by the sources.conf file in a
44# MacPorts installation, found by loading its macports1.0 tcl package and initializing
45# it with 'mportinit' below. Main use of the resulting database is providing live
46# information to the ports.php page, a client tailored to poll it. For this very reason,
47# information fed to the database always has to be kept up to date in order to remain
48# meaningful, which is accomplished simply by calling the 'mportsync' proc in macports1.0
49# (which updates the ports tree in use) and by installing the script on cron/launchd to be
50# run on a timely schedule (not any more frequent than the run of the PortIndexRegen.sh
51# script on that creates a new PortIndex file, which is every twelve hours).
52#
53# Remaining requirement to successfully run this script is performing the necessary
54# MySQL admin tasks on the host box to create the database in the first place and the
55# MySQL user that will be given enough privileges to alter it. Values in the database
56# related variables provided below have to be adapted accordingly to match the chosen
57# setup.
58#####
59
60
61
62# Runtime information log file and reciepient.
63set runlog "/tmp/portsdb.log"
64set runlog_fd [open $runlog w+]
65set lockfile "/tmp/portsdb.lock"
66set mailprog "/usr/sbin/sendmail"
67set DATE [clock format [clock seconds] -format "%A %Y-%m-%d at %T"]
68
69#set SPAM_LOVERS example@hostname.com
70
71set SUBJECT "PortIndex2MySQL run failure on $DATE"
72set FROM macports-mgr@lists.macosforge.org
73set HEADERS "To: $SPAM_LOVERS\r\nFrom: $FROM\r\nSubject: $SUBJECT\r\n\r\n"
74
75# We first initialize the runlog with proper mail headers
76puts $runlog_fd $HEADERS
77
78# House keeping on exit.
79proc cleanup {args} {
80    foreach file_to_clean $args {
81        upvar $file_to_clean up_file_to_clean
82        upvar ${file_to_clean}_fd up_file_to_clean_fd
83        close $up_file_to_clean_fd
84        file delete -force $up_file_to_clean
85    }
86}
87
88# What to do when terminating execution, depending on the $exit_status condition.
89proc terminate {exit_status} {
90    global runlog runlog_fd
91    if {$exit_status} {
92        global subject SPAM_LOVERS mailprog
93        seek $runlog_fd 0 start
94        exec -- $mailprog $SPAM_LOVERS <@ $runlog_fd
95    }
96    cleanup runlog
97    exit $exit_status
98}
99
100# Check if there are any stray sibling jobs before moving on, bail in such case.
101if {[file exists $lockfile]} {
102    puts $runlog_fd "PortIndex2MySQL lock file found, is another job running?" 
103    terminate 1
104} else {
105    set lockfile_fd [open $lockfile a]
106}
107
108
109# Load macports1.0 so that we can use some of its procs and the portinfo array.
110if {[catch { source [file join "@TCL_PACKAGE_DIR@" macports1.0 macports_fastload.tcl] } errstr]} {
111    puts $runlog_fd "${::errorInfo}"
112    puts $runlog_fd "Failed to locate the macports1.0 Tcl package file: $errstr"
113    cleanup lockfile
114    terminate 1
115}
116if {[catch { package require macports } errstr]} {
117    puts $runlog_fd "${::errorInfo}"
118    puts $runlog_fd "Failed to load the macports1.0 Tcl package: $errstr"
119    cleanup lockfile
120    terminate 1
121}
122
123# macports1.0 UI instantiation to route information/error messages wherever we want.
124# This is a custom ui_channels proc because we want to get reported information on
125# channels other than the default stdout/stderr that the macports1.0 API provides,
126# namely a log file we can later mail to people in charge if need be.
127proc ui_channels {priority} {
128    global runlog_fd
129    switch $priority {
130        debug {
131            if {[macports::ui_isset ports_debug]} {
132                return $runlog_fd
133            } else {
134                return {}
135            }
136        }
137        info {
138            if {[macports::ui_isset ports_verbose]} {
139                return $runlog_fd
140            } else {
141                return {}
142            }
143        }
144        msg {
145            if {[macports::ui_isset ports_quiet]} {
146                return $runlog_fd
147            } else {
148                return {}
149            }
150        }
151        error {
152            return $runlog_fd
153        }
154        default {
155            return {}
156        }
157    }
158}
159
160# Initialize macports1.0 and its UI, in order to find the sources.conf file
161# (which is what will point us to the PortIndex we're gonna use) and use
162# the runtime information.
163array set ui_options {ports_verbose yes}
164if {[catch {mportinit ui_options} errstr]} {
165    puts $runlog_fd "${::errorInfo}"
166    puts $runlog_fd "Failed to initialize MacPorts: $errstr"
167    cleanup lockfile
168    terminate 1
169}
170
171
172# Procedure to catch the database password from a protected file.
173proc getpasswd {passwdfile} {
174    if {[catch {open $passwdfile r} passwdfile_fd]} {
175        global lockfile lockfile_fd
176        ui_error "${::errorCode}: $passwdfile_fd"
177        cleanup lockfile
178        terminate 1
179    }
180    if {[gets $passwdfile_fd passwd] <= 0} {
181        global lockfile lockfile_fd
182        close $passwdfile_fd
183        ui_error "No password found in password file $passwdfile!"
184        cleanup lockfile
185        terminate 1
186    }
187    close $passwdfile_fd
188    return $passwd
189}
190
191# Database abstraction variables:
192set sqlfile "/tmp/portsdb.sql"
193set portsdb_host localhost
194set portsdb_name macports
195set portsdb_user macports
196set passwdfile "/opt/local/share/macports/resources/portmgr/password_file"
197set portsdb_passwd [getpasswd $passwdfile]
198set portsdb_cmd [macports::findBinary mysql5]
199
200
201# Flat text file to which sql statements are written.
202if {[catch {open $sqlfile w+} sqlfile_fd]} {
203    ui_error "${::errorCode}: $sqlfile_fd"
204    cleanup lockfile
205    terminate 1
206}
207
208
209# Call the sync procedure to make sure we always have a fresh ports tree.
210if {[catch {mportsync} errstr]} {
211    ui_error "${::errorInfo}"
212    ui_error "Failed to update the ports tree, $errstr"
213    cleanup sqlfile lockfile
214    terminate 1
215}
216
217# Load every port in the index through a search that matches everything.
218if {[catch {set ports [mportsearch ".+"]} errstr]} {
219    ui_error "${::errorInfo}"
220    ui_error "port search failed: $errstr"
221    cleanup sqlfile lockfile
222    terminate 1
223}
224
225
226# SQL string escaping.
227proc sql_escape {str} {
228    regsub -all -- {'} $str {\\'} str
229    regsub -all -- {"} $str {\\"} str
230    regsub -all -- {\n} $str {\\n} str
231    return $str
232}
233
234# Initial creation of database tables: log, portfiles, categories, maintainers, dependencies, variants and platforms.
235# Do we need any other?
236puts $sqlfile_fd "DROP TABLE IF EXISTS log;"
237puts $sqlfile_fd "CREATE TABLE log (activity VARCHAR(255), activity_time TIMESTAMP(14)) DEFAULT CHARSET=utf8;"
238
239puts $sqlfile_fd "DROP TABLE IF EXISTS portfiles;"
240puts $sqlfile_fd "CREATE TABLE portfiles (name VARCHAR(255) PRIMARY KEY NOT NULL, path VARCHAR(255), version VARCHAR(255),  description TEXT) DEFAULT CHARSET=utf8;"
241
242puts $sqlfile_fd "DROP TABLE IF EXISTS categories;"
243puts $sqlfile_fd "CREATE TABLE categories (portfile VARCHAR(255), category VARCHAR(255), is_primary INTEGER) DEFAULT CHARSET=utf8;"
244
245puts $sqlfile_fd "DROP TABLE IF EXISTS maintainers;"
246puts $sqlfile_fd "CREATE TABLE maintainers (portfile VARCHAR(255), maintainer VARCHAR(255), is_primary INTEGER) DEFAULT CHARSET=utf8;"
247
248puts $sqlfile_fd "DROP TABLE IF EXISTS dependencies;"
249puts $sqlfile_fd "CREATE TABLE dependencies (portfile VARCHAR(255), library VARCHAR(255)) DEFAULT CHARSET=utf8;"
250
251puts $sqlfile_fd "DROP TABLE IF EXISTS variants;"
252puts $sqlfile_fd "CREATE TABLE variants (portfile VARCHAR(255), variant VARCHAR(255)) DEFAULT CHARSET=utf8;"
253
254puts $sqlfile_fd "DROP TABLE IF EXISTS platforms;"
255puts $sqlfile_fd "CREATE TABLE platforms (portfile VARCHAR(255), platform VARCHAR(255)) DEFAULT CHARSET=utf8;"
256
257
258# Iterate over each matching port, extracting its information from the
259# portinfo array.
260foreach {name array} $ports {
261
262    array unset portinfo
263    array set portinfo $array
264
265    set portname [sql_escape $portinfo(name)]
266    if {[info exists portinfo(version)]} {
267        set portversion [sql_escape $portinfo(version)]
268    } else {
269        set portversion ""
270    }
271    set portdir [sql_escape $portinfo(portdir)]
272    if {[info exists portinfo(description)]} {
273        set description [sql_escape $portinfo(description)]
274    } else {
275        set description ""
276    }
277    if {[info exists portinfo(categories)]} {
278        set categories $portinfo(categories)
279    } else {
280        set categories ""
281    }
282    if {[info exists portinfo(maintainers)]} {
283        set maintainers $portinfo(maintainers)
284    } else {
285        set maintainers ""
286    }
287    if {[info exists portinfo(variants)]} {
288        set variants $portinfo(variants)
289    } else {
290        set variants ""
291    }
292    if {[info exists portinfo(depends_fetch)]} {
293        set depends_fetch $portinfo(depends_fetch)
294    } else {
295        set depends_fetch ""
296    }
297    if {[info exists portinfo(depends_extract)]} {
298        set depends_extract $portinfo(depends_extract)
299    } else {
300        set depends_extract ""
301    }
302    if {[info exists portinfo(depends_build)]} {
303        set depends_build $portinfo(depends_build)
304    } else {
305        set depends_build ""
306    }
307    if {[info exists portinfo(depends_lib)]} {
308        set depends_lib $portinfo(depends_lib)
309    } else {
310        set depends_lib ""
311    }
312    if {[info exists portinfo(depends_run)]} {
313        set depends_run $portinfo(depends_run)
314    } else {
315        set depends_run ""
316    }
317    if {[info exists portinfo(platforms)]} {
318        set platforms $portinfo(platforms)
319    } else {
320        set platforms ""
321    }
322
323    puts $sqlfile_fd "INSERT INTO portfiles VALUES ('$portname', '$portdir', '$portversion', '$description');"
324
325    set primary 1
326    foreach category $categories {
327        set category [sql_escape $category]
328        puts $sqlfile_fd "INSERT INTO categories VALUES ('$portname', '$category', $primary);"
329        set primary 0
330    }
331   
332    set primary 1
333    foreach maintainer $maintainers {
334        set maintainer [sql_escape $maintainer]
335        puts $sqlfile_fd "INSERT INTO maintainers VALUES ('$portname', '$maintainer', $primary);"
336        set primary 0
337    }
338
339    foreach fetch_dep $depends_fetch {
340        set fetch_dep [sql_escape $fetch_dep]
341        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$fetch_dep');"
342    }
343   
344    foreach extract_dep $depends_extract {
345        set extract_dep [sql_escape $extract_dep]
346        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$extract_dep');"
347    }
348
349    foreach build_dep $depends_build {
350        set build_dep [sql_escape $build_dep]
351        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$build_dep');"
352    }
353
354    foreach lib $depends_lib {
355        set lib [sql_escape $lib]
356        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$lib');"
357    }
358
359    foreach run_dep $depends_run {
360        set run_dep [sql_escape $run_dep]
361        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$run_dep');"
362    }
363
364    foreach variant $variants {
365        set variant [sql_escape $variant]
366        puts $sqlfile_fd "INSERT INTO variants VALUES ('$portname', '$variant');"
367    }
368
369    foreach platform $platforms {
370        set platform [sql_escape $platform]
371        puts $sqlfile_fd "INSERT INTO platforms VALUES ('$portname', '$platform');"
372    }
373
374}
375
376# Mark the db regen as done only once we're done processing all ports:
377puts $sqlfile_fd "INSERT INTO log VALUES ('update', NOW());"
378
379
380# Pipe the contents of the generated sql file to the database command,
381# reading from the file descriptor for the raw sql file to assure completeness.
382if {[catch {seek $sqlfile_fd 0 start} errstr]} {
383    ui_error "${::errorCode}: $errstr"
384    cleanup sqlfile lockfile
385    terminate 1
386}
387if {[catch {exec -- $portsdb_cmd --host=$portsdb_host --user=$portsdb_user --password=$portsdb_passwd --database=$portsdb_name <@ $sqlfile_fd} errstr]} {
388    ui_error "${::errorCode}: $errstr"
389    cleanup sqlfile lockfile
390    terminate 1
391}
392
393
394# And we're done regen'ing the MacPorts dabase! Cleanup and exit successfully.
395cleanup sqlfile lockfile
396terminate 0
Note: See TracBrowser for help on using the repository browser.