source: trunk/base/portmgr/jobs/PortIndex2MySQL.tcl @ 30447

Last change on this file since 30447 was 30447, checked in by jmpp@…, 12 years ago

Move all server side jobs to a new portmgr/jobs dir, in order to keep easier track of what we need to setup to have all our services running.
Add placeholder launchd plists for those jobs that don't already have an uploaded one (all but PortIndex2MySQL); I hope to fill these with
the appropriate information as I gather it.

PS: James, do you want me to move autosubmit in here too?

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 12.1 KB
Line 
1#!/usr/bin/env tclsh
2# -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:filetype=tcl:et:sw=4:ts=4:sts=4
3#
4# PortIndex2MySQL.tcl
5# Kevin Van Vechten | kevin@opendarwin.org
6# 3-Oct-2002
7# Juan Manuel Palacios | jmpp@macports.org
8# 30-Jul-2007
9# $Id: PortIndex2MySQL.tcl 30447 2007-10-28 05:58:31Z jmpp@macports.org $
10#
11# Copyright (c) 2007 Juan Manuel Palacios, MacPorts Team.
12# Copyright (c) 2003 Apple Computer, Inc.
13# Copyright (c) 2002 Kevin Van Vechten.
14# All rights reserved.
15#
16# Redistribution and use in source and binary forms, with or without
17# modification, are permitted provided that the following conditions
18# are met:
19# 1. Redistributions of source code must retain the above copyright
20#    notice, this list of conditions and the following disclaimer.
21# 2. Redistributions in binary form must reproduce the above copyright
22#    notice, this list of conditions and the following disclaimer in the
23#    documentation and/or other materials provided with the distribution.
24# 3. Neither the name of Apple Computer, Inc. nor the names of its contributors
25#    may be used to endorse or promote products derived from this software
26#    without specific prior written permission.
27#
28# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
32# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38# POSSIBILITY OF SUCH DAMAGE.
39
40#####
41# The PortIndex2MySQL script populates a database with key information extracted
42# from the Portfiles in the ports tree pointed to by the sources.conf file in a
43# MacPorts installation, found by loading its macports1.0 tcl package and initializing
44# it with 'mportinit' below. Main use of the resulting database is providing live
45# information to the ports.php page, a client tailored to poll it. For this very reason,
46# information fed to the database always has to be kept up to date in order to remain
47# meaningful, which is accomplished simply by calling 'macports::selfupdate' (which
48# updates the ports tree in use) and by installing the script on cron/launchd to be run
49# on a timely schedule (not any more frequent than the run of the mprsyncup script on
50# the MacPorts server, which is every half hour).
51#
52# Remaining requirement to successfully run this script is performing the necessary
53# MySQL admin tasks on the host box to create the database in the first place and the
54# MySQL user that will be given enough privileges to alter it. Values in the database
55# related variables provided below have to be adapted accordingly to match the chosen
56# setup.
57#####
58
59
60# Load macports1.0 so that we can use some of its procs and the portinfo array.
61catch {source \
62    [file join "@TCL_PACKAGE_DIR@" macports1.0 macports_fastload.tcl]}
63package require macports
64
65
66# Runtime information log file and reciepient.
67set runlog "/tmp/portsdb.log"
68set runlog_fd [open $runlog w+]
69set lockfile "/tmp/portsdb.lock"
70set mailprog "/usr/sbin/sendmail"
71set DATE [clock format [clock seconds] -format "%A %Y-%m-%d at %T"]
72set subject "PortIndex2MySQL run failure on $DATE"
73set SPAM_LOVERS macports-dev@lists.macosforge.org
74
75# House keeping on exit.
76proc cleanup {args} {
77    foreach file_to_clean $args {
78        upvar $file_to_clean up_file_to_clean
79        upvar ${file_to_clean}_fd up_file_to_clean_fd
80        close $up_file_to_clean_fd
81        file delete -force $up_file_to_clean
82    }
83}
84
85# What to do when terminating execution, depending on the $exit_status condition.
86proc terminate {exit_status} {
87    global runlog runlog_fd
88    if {$exit_status} {
89        global subject SPAM_LOVERS mailprog
90        seek $runlog_fd 0 start
91        exec -- $mailprog $SPAM_LOVERS <@ $runlog_fd
92    }
93    cleanup runlog
94    exit $exit_status
95}
96
97
98# UI instantiation to route information/error messages wherever we want.
99proc ui_channels {priority} {
100    global runlog_fd
101    switch $priority {
102        debug {
103            if {[macports::ui_isset ports_debug]} {
104                return $runlog_fd
105            } else {
106                return {}
107            }
108        }
109        info {
110            if {[macports::ui_isset ports_verbose]} {
111                return $runlog_fd
112            } else {
113                return {}
114            }
115        }
116        msg {
117            if {[macports::ui_isset ports_quiet]} {
118                return $runlog_fd
119            } else {
120                return {}
121            }
122        }
123        error {
124            return $runlog_fd
125        }
126        default {
127            return {}
128        }
129    }
130}
131
132
133# We first initialize the runlog with a proper mail subject:
134puts $runlog_fd "Subject: $subject"
135
136# Check if there are any stray sibling jobs before moving on, bail in such case.
137if {[file exists $lockfile]} {
138    ui_error "PortIndex2MySQL lock file found, is another job running?"
139    terminate 1
140} else {
141    set lockfile_fd [open $lockfile a]
142}
143
144
145# Initialize macports1.0 and its UI, in order to find the sources.conf file
146# (which is what will point us to the PortIndex we're gonna use) and use
147# the runtime information.
148array set ui_options {ports_verbose yes}
149if {[catch {mportinit ui_options} errstr]} {
150    ui_error "${::errorInfo}"
151    ui_error "Failed to initialize MacPorts, $errstr"
152    cleanup lockfile
153    terminate 1
154}
155
156# Call the selfupdate procedure to make sure the MacPorts installation
157# is up-to-date and with a fresh ports tree.
158if {[catch {macports::selfupdate} errstr]} {
159    ui_error "${::errorInfo}"
160    ui_error "Failed to update the ports tree, $errstr"
161    cleanup lockfile
162    terminate 1
163}
164
165
166# Procedure to catch the database password from a protected file.
167proc getpasswd {passwdfile} {
168    if {[catch {open $passwdfile r} passwdfile_fd]} {
169        global lockfile lockfile_fd
170        ui_error "${::errorCode}: $passwdfile_fd"
171        cleanup lockfile
172        terminate 1
173    }
174    if {[gets $passwdfile_fd passwd] <= 0} {
175        global lockfile lockfile_fd
176        close $passwdfile_fd
177        ui_error "No password found in password file $passwdfile!"
178        cleanup lockfile
179        terminate 1
180    }
181    close $passwdfile_fd
182    return $passwd
183}
184
185# Database abstraction variables:
186set sqlfile "/tmp/portsdb.sql"
187set dbcmd [macports::findBinary mysql5]
188set dbhost 127.0.0.1
189set dbuser macports
190set passwdfile "./password_file"
191set dbpasswd [getpasswd $passwdfile]
192set dbname macports
193
194
195# Flat text file to which sql statements are written.
196if {[catch {open $sqlfile w+} sqlfile_fd]} {
197    ui_error "${::errorCode}: $sqlfile_fd"
198    cleanup lockfile
199    terminate 1
200}
201
202# SQL string escaping.
203proc sql_escape {str} {
204    regsub -all -- {'} $str {\\'} str
205    regsub -all -- {"} $str {\\"} str
206    regsub -all -- {\n} $str {\\n} str
207    return $str
208}
209
210# Initial creation of database tables: log, portfiles, categories, maintainers, dependencies, variants and platforms.
211# Do we need any other?
212puts $sqlfile_fd "DROP TABLE IF EXISTS log;"
213puts $sqlfile_fd "CREATE TABLE IF NOT EXISTS log (activity VARCHAR(255), activity_time TIMESTAMP(14));"
214puts $sqlfile_fd "INSERT INTO log VALUES ('update', NOW());"
215
216puts $sqlfile_fd "DROP TABLE IF EXISTS portfiles;"
217puts $sqlfile_fd "CREATE TABLE portfiles (name VARCHAR(255) PRIMARY KEY NOT NULL, path VARCHAR(255), version VARCHAR(255),  description TEXT);"
218
219puts $sqlfile_fd "DROP TABLE IF EXISTS categories;"
220puts $sqlfile_fd "CREATE TABLE categories (portfile VARCHAR(255), category VARCHAR(255), is_primary INTEGER);"
221
222puts $sqlfile_fd "DROP TABLE IF EXISTS maintainers;"
223puts $sqlfile_fd "CREATE TABLE maintainers (portfile VARCHAR(255), maintainer VARCHAR(255), is_primary INTEGER);"
224
225puts $sqlfile_fd "DROP TABLE IF EXISTS dependencies;"
226puts $sqlfile_fd "CREATE TABLE dependencies (portfile VARCHAR(255), library VARCHAR(255));"
227
228puts $sqlfile_fd "DROP TABLE IF EXISTS variants;"
229puts $sqlfile_fd "CREATE TABLE variants (portfile VARCHAR(255), variant VARCHAR(255));"
230
231puts $sqlfile_fd "DROP TABLE IF EXISTS platforms;"
232puts $sqlfile_fd "CREATE TABLE platforms (portfile VARCHAR(255), platform VARCHAR(255));"
233
234
235# Load every port in the index through a search that matches everything.
236if {[catch {set ports [mportsearch ".+"]} errstr]} {
237    ui_error "${::errorInfo}"
238    ui_error "port search failed: $errstr"
239    cleanup sqlfile lockfile
240    terminate 1
241}
242
243# Iterate over each matching port, extracting its information from the
244# portinfo array.
245foreach {name array} $ports {
246
247    array unset portinfo
248    array set portinfo $array
249
250    set portname [sql_escape $portinfo(name)]
251    if {[info exists portinfo(version)]} {
252        set portversion [sql_escape $portinfo(version)]
253    } else {
254        set portversion ""
255    }
256    set portdir [sql_escape $portinfo(portdir)]
257    if {[info exists portinfo(description)]} {
258        set description [sql_escape $portinfo(description)]
259    } else {
260        set description ""
261    }
262    if {[info exists portinfo(categories)]} {
263        set categories $portinfo(categories)
264    } else {
265        set categories ""
266    }
267    if {[info exists portinfo(maintainers)]} {
268        set maintainers $portinfo(maintainers)
269    } else {
270        set maintainers ""
271    }
272    if {[info exists portinfo(variants)]} {
273        set variants $portinfo(variants)
274    } else {
275        set variants ""
276    }
277    if {[info exists portinfo(depends_build)]} {
278        set depends_build $portinfo(depends_build)
279    } else {
280        set depends_build ""
281    }
282    if {[info exists portinfo(depends_lib)]} {
283        set depends_lib $portinfo(depends_lib)
284    } else {
285        set depends_lib ""
286    }
287    if {[info exists portinfo(depends_run)]} {
288        set depends_run $portinfo(depends_run)
289    } else {
290        set depends_run ""
291    }
292    if {[info exists portinfo(platforms)]} {
293        set platforms $portinfo(platforms)
294    } else {
295        set platforms ""
296    }
297
298    puts $sqlfile_fd "INSERT INTO portfiles VALUES ('$portname', '$portdir', '$portversion', '$description');"
299
300    set primary 1
301    foreach category $categories {
302        set category [sql_escape $category]
303        puts $sqlfile_fd "INSERT INTO categories VALUES ('$portname', '$category', $primary);"
304        incr primary
305    }
306   
307    set primary 1
308    foreach maintainer $maintainers {
309        set maintainer [sql_escape $maintainer]
310        puts $sqlfile_fd "INSERT INTO maintainers VALUES ('$portname', '$maintainer', $primary);"
311        incr primary
312    }
313
314    foreach build_dep $depends_build {
315        set build_dep [sql_escape $build_dep]
316        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$build_dep');"
317    }
318
319    foreach lib $depends_lib {
320        set lib [sql_escape $lib]
321        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$lib');"
322    }
323
324    foreach run_dep $depends_run {
325        set run_dep [sql_escape $run_dep]
326        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$run_dep');"
327    }
328
329    foreach variant $variants {
330        set variant [sql_escape $variant]
331        puts $sqlfile_fd "INSERT INTO variants VALUES ('$portname', '$variant');"
332    }
333
334    foreach platform $platforms {
335        set platform [sql_escape $platform]
336        puts $sqlfile_fd "INSERT INTO platforms VALUES ('$portname', '$platform');"
337    }
338
339}
340
341
342# Pipe the contents of the generated sql file to the database command,
343# reading from the file descriptor for the raw sql file to assure completeness.
344if {[catch {seek $sqlfile_fd 0 start} errstr]} {
345    ui_error "${::errorCode}: $errstr"
346    cleanup sqlfile lockfile
347    terminate 1
348}
349if {[catch {exec -- $dbcmd --host=$dbhost --user=$dbuser --password=$dbpasswd --database=$dbname <@ $sqlfile_fd} errstr]} {
350    ui_error "${::errorCode}: $errstr"
351    cleanup sqlfile lockfile
352    terminate 1
353}
354
355
356# And we're done regen'ing the MacPorts dabase! Cleanup and exit successfully.
357cleanup sqlfile lockfile
358terminate 0
Note: See TracBrowser for help on using the repository browser.