source: trunk/base/portmgr/jobs/PortIndex2MySQL.tcl

Last change on this file was 151423, checked in by cal@…, 3 years ago

portmgr: Fix PortIndex2MySQL #!, drop Makefile

The shebang of PortIndex2MySQL.tcl was hardcoded to /opt/local/bin/port-tclsh.
Use /usr/bin/env port-tclsh to make the script portable and obsolete the
Makefile (which didn't do anything other than copying the file for a while
now). Create a symlink so that use cases that expect the script to be available
as "PortIndex2MySQL" don't break.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 15.0 KB
Line 
1#!/usr/bin/env port-tclsh
2# -*- coding: utf-8; mode: tcl; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- vim:fenc=utf-8:filetype=tcl:et:sw=4:ts=4:sts=4
3#
4# PortIndex2MySQL.tcl
5# Kevin Van Vechten | kevin@opendarwin.org
6# 3-Oct-2002
7# Juan Manuel Palacios | jmpp@macports.org
8# 22-Nov-2007
9# $Id: PortIndex2MySQL.tcl 151423 2016-08-15 23:36:55Z cal@macports.org $
10#
11# Copyright (c) 2007 Juan Manuel Palacios, The MacPorts Project.
12# Copyright (c) 2003 Apple Inc.
13# Copyright (c) 2002 Kevin Van Vechten.
14# All rights reserved.
15#
16# Redistribution and use in source and binary forms, with or without
17# modification, are permitted provided that the following conditions
18# are met:
19# 1. Redistributions of source code must retain the above copyright
20#    notice, this list of conditions and the following disclaimer.
21# 2. Redistributions in binary form must reproduce the above copyright
22#    notice, this list of conditions and the following disclaimer in the
23#    documentation and/or other materials provided with the distribution.
24# 3. Neither the name of Apple Inc. nor the names of its contributors
25#    may be used to endorse or promote products derived from this software
26#    without specific prior written permission.
27#
28# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
32# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38# POSSIBILITY OF SUCH DAMAGE.
39
40
41#####
42# The PortIndex2MySQL script populates a database with key information extracted
43# from the Portfiles in the ports tree pointed to by the sources.conf file in a
44# MacPorts installation, found by loading its macports1.0 tcl package and initializing
45# it with 'mportinit' below. Main use of the resulting database is providing live
46# information to the ports.php page, a client tailored to poll it. For this very reason,
47# information fed to the database always has to be kept up to date in order to remain
48# meaningful, which is accomplished simply by calling the 'mportsync' proc in macports1.0
49# (which updates the ports tree in use) and by installing the script on cron/launchd to be
50# run on a timely schedule (not any more frequent than the run of the PortIndexRegen.sh
51# script on that creates a new PortIndex file).
52#
53# Remaining requirement to successfully run this script is performing the necessary
54# MySQL admin tasks on the host box to create the database in the first place and the
55# MySQL user that will be given enough privileges to alter it. Values in the database
56# related variables provided below have to be adapted accordingly to match the chosen
57# setup.
58#####
59
60
61# Configuration variables. Fill in the blank strings.
62#
63# Email addresses that get failure notifications - e.g. "admin@macosforge.org"
64set SPAM_LOVERS ""
65# File to read the db password from - e.g. "/var/macports/script_data"
66set passwdfile ""
67# Database abstraction variables:
68# name of the mysql executable
69set mysql_exe "mysql"
70# path where the mysql executable is located (only needed if not in the default PATH)
71set mysql_exe_path ""
72set sqlfile "/tmp/portsdb.sql"
73set portsdb_host localhost
74set portsdb_name macports
75set portsdb_user macports
76
77# Runtime information log file and recipient.
78set runlog "/tmp/portsdb.log"
79set runlog_fd [open $runlog w+]
80set lockfile "/tmp/portsdb.lock"
81set mailprog "/usr/sbin/sendmail"
82set DATE [clock format [clock seconds] -format "%A %Y-%m-%d at %T"]
83
84set SUBJECT "PortIndex2MySQL run failure on $DATE"
85set FROM noreply@macports.org
86set HEADERS "To: $SPAM_LOVERS\r\nFrom: $FROM\r\nSubject: $SUBJECT\r\n\r\n"
87
88# handle command line arguments
89set create_tables true
90if {[llength $argv]} {
91    if {[lindex $argv 0] eq "--create-tables"} {
92        set create_tables true
93    }
94}
95
96# House keeping on exit.
97proc cleanup {args} {
98    foreach file_to_clean $args {
99        upvar $file_to_clean up_file_to_clean
100        upvar ${file_to_clean}_fd up_file_to_clean_fd
101        close $up_file_to_clean_fd
102        file delete -force $up_file_to_clean
103    }
104}
105
106# What to do when terminating execution, depending on the $exit_status condition.
107proc terminate {exit_status} {
108    global runlog runlog_fd
109    if {$exit_status} {
110        global subject SPAM_LOVERS mailprog
111        seek $runlog_fd 0 start
112        exec -- $mailprog $SPAM_LOVERS <@ $runlog_fd
113    }
114    cleanup runlog
115    exit $exit_status
116}
117
118# macports1.0 UI instantiation to route information/error messages wherever we want.
119# This is a custom ui_channels proc because we want to get reported information on
120# channels other than the default stdout/stderr that the macports1.0 API provides,
121# namely a log file we can later mail to people in charge if need be.
122proc ui_channels {priority} {
123    global runlog_fd
124    switch $priority {
125        debug {
126            if {[macports::ui_isset ports_debug]} {
127                return $runlog_fd
128            } else {
129                return {}
130            }
131        }
132        info {
133            if {[macports::ui_isset ports_verbose]} {
134                return $runlog_fd
135            } else {
136                return {}
137            }
138        }
139        msg {
140            if {[macports::ui_isset ports_quiet]} {
141                return $runlog_fd
142            } else {
143                return {}
144            }
145        }
146        error {
147            return $runlog_fd
148        }
149        default {
150            return {}
151        }
152    }
153}
154
155# Procedure to catch the database password from a protected file.
156proc getpasswd {passwdfile} {
157    if {$passwdfile eq ""} {
158        global lockfile lockfile_fd
159        ui_error "passwdfile is empty, did you forget to set it?"
160        cleanup lockfile
161        terminate 1
162    }
163    if {[catch {open $passwdfile r} passwdfile_fd]} {
164        global lockfile lockfile_fd
165        ui_error "${::errorCode}: $passwdfile_fd"
166        cleanup lockfile
167        terminate 1
168    }
169    if {[gets $passwdfile_fd passwd] <= 0} {
170        global lockfile lockfile_fd
171        close $passwdfile_fd
172        ui_error "No password found in password file $passwdfile!"
173        cleanup lockfile
174        terminate 1
175    }
176    close $passwdfile_fd
177    return $passwd
178}
179
180# SQL string escaping.
181proc sql_escape {str} {
182    regsub -all -- {'} $str {\\'} str
183    regsub -all -- {"} $str {\\"} str
184    regsub -all -- {\n} $str {\\n} str
185    return $str
186}
187
188# We first initialize the runlog with proper mail headers
189puts $runlog_fd $HEADERS
190
191# Check if there are any stray sibling jobs before moving on, bail in such case.
192if {[file exists $lockfile]} {
193    puts $runlog_fd "PortIndex2MySQL lock file found, is another job running?" 
194    terminate 1
195} else {
196    set lockfile_fd [open $lockfile a]
197}
198
199# Load macports1.0 so that we can use some of its procs and the portinfo array.
200if {[catch { package require macports } errstr]} {
201    puts $runlog_fd "${::errorInfo}"
202    puts $runlog_fd "Failed to load the macports1.0 Tcl package: $errstr"
203    cleanup lockfile
204    terminate 1
205}
206
207# Initialize macports1.0 and its UI, in order to find the sources.conf file
208# (which is what will point us to the PortIndex we're gonna use) and use
209# the runtime information.
210array set ui_options {ports_verbose yes}
211if {[catch {mportinit ui_options} errstr]} {
212    puts $runlog_fd "${::errorInfo}"
213    puts $runlog_fd "Failed to initialize MacPorts: $errstr"
214    cleanup lockfile
215    terminate 1
216}
217
218
219set portsdb_passwd [getpasswd $passwdfile]
220set portsdb_cmd [macports::findBinary $mysql_exe $mysql_exe_path]
221
222
223# Flat text file to which sql statements are written.
224if {[catch {open $sqlfile w+} sqlfile_fd]} {
225    ui_error "${::errorCode}: $sqlfile_fd"
226    cleanup lockfile
227    terminate 1
228}
229
230
231# Call the sync procedure to make sure we always have a fresh ports tree.
232if {[catch {mportsync} errstr]} {
233    ui_error "${::errorInfo}"
234    ui_error "Failed to update the ports tree, $errstr"
235    cleanup sqlfile lockfile
236    terminate 1
237}
238
239# Load every port in the index through a search that matches everything.
240if {[catch {set ports [mportlistall]} errstr]} {
241    ui_error "${::errorInfo}"
242    ui_error "port search failed: $errstr"
243    cleanup sqlfile lockfile
244    terminate 1
245}
246
247if {$create_tables} {
248    # Initial creation of database tables: log, portfiles, categories, maintainers, dependencies, variants and platforms.
249    # Do we need any other?
250    puts $sqlfile_fd "DROP TABLE IF EXISTS log;"
251    puts $sqlfile_fd "CREATE TABLE log (activity VARCHAR(255), activity_time TIMESTAMP(14)) DEFAULT CHARSET=utf8;"
252   
253    puts $sqlfile_fd "DROP TABLE IF EXISTS portfiles;"
254    puts $sqlfile_fd "CREATE TABLE portfiles (name VARCHAR(255) PRIMARY KEY NOT NULL, path VARCHAR(255), version VARCHAR(255),  description TEXT) DEFAULT CHARSET=utf8;"
255   
256    puts $sqlfile_fd "DROP TABLE IF EXISTS categories;"
257    puts $sqlfile_fd "CREATE TABLE categories (portfile VARCHAR(255), category VARCHAR(255), is_primary INTEGER) DEFAULT CHARSET=utf8;"
258   
259    puts $sqlfile_fd "DROP TABLE IF EXISTS maintainers;"
260    puts $sqlfile_fd "CREATE TABLE maintainers (portfile VARCHAR(255), maintainer VARCHAR(255), is_primary INTEGER) DEFAULT CHARSET=utf8;"
261   
262    puts $sqlfile_fd "DROP TABLE IF EXISTS dependencies;"
263    puts $sqlfile_fd "CREATE TABLE dependencies (portfile VARCHAR(255), library VARCHAR(255)) DEFAULT CHARSET=utf8;"
264   
265    puts $sqlfile_fd "DROP TABLE IF EXISTS variants;"
266    puts $sqlfile_fd "CREATE TABLE variants (portfile VARCHAR(255), variant VARCHAR(255)) DEFAULT CHARSET=utf8;"
267   
268    puts $sqlfile_fd "DROP TABLE IF EXISTS platforms;"
269    puts $sqlfile_fd "CREATE TABLE platforms (portfile VARCHAR(255), platform VARCHAR(255)) DEFAULT CHARSET=utf8;"
270
271    puts $sqlfile_fd "DROP TABLE IF EXISTS licenses;"
272    puts $sqlfile_fd "CREATE TABLE licenses (portfile VARCHAR(255), license VARCHAR(255)) DEFAULT CHARSET=utf8;"
273} else {
274    # if we are not creating tables from scratch, remove the old data
275    puts $sqlfile_fd "TRUNCATE log;"
276    puts $sqlfile_fd "TRUNCATE portfiles;"
277    puts $sqlfile_fd "TRUNCATE categories;"
278    puts $sqlfile_fd "TRUNCATE maintainers;"
279    puts $sqlfile_fd "TRUNCATE dependencies;"
280    puts $sqlfile_fd "TRUNCATE variants;"
281    puts $sqlfile_fd "TRUNCATE platforms;"
282    puts $sqlfile_fd "TRUNCATE licenses;"
283}
284 
285# Iterate over each matching port, extracting its information from the
286# portinfo array.
287foreach {name array} $ports {
288
289    array unset portinfo
290    array set portinfo $array
291
292    set portname [sql_escape $portinfo(name)]
293    if {[info exists portinfo(version)]} {
294        set portversion [sql_escape $portinfo(version)]
295    } else {
296        set portversion ""
297    }
298    set portdir [sql_escape $portinfo(portdir)]
299    if {[info exists portinfo(description)]} {
300        set description [sql_escape $portinfo(description)]
301    } else {
302        set description ""
303    }
304    if {[info exists portinfo(categories)]} {
305        set categories $portinfo(categories)
306    } else {
307        set categories ""
308    }
309    if {[info exists portinfo(maintainers)]} {
310        set maintainers $portinfo(maintainers)
311    } else {
312        set maintainers ""
313    }
314    if {[info exists portinfo(variants)]} {
315        set variants $portinfo(variants)
316    } else {
317        set variants ""
318    }
319    if {[info exists portinfo(depends_fetch)]} {
320        set depends_fetch $portinfo(depends_fetch)
321    } else {
322        set depends_fetch ""
323    }
324    if {[info exists portinfo(depends_extract)]} {
325        set depends_extract $portinfo(depends_extract)
326    } else {
327        set depends_extract ""
328    }
329    if {[info exists portinfo(depends_build)]} {
330        set depends_build $portinfo(depends_build)
331    } else {
332        set depends_build ""
333    }
334    if {[info exists portinfo(depends_lib)]} {
335        set depends_lib $portinfo(depends_lib)
336    } else {
337        set depends_lib ""
338    }
339    if {[info exists portinfo(depends_run)]} {
340        set depends_run $portinfo(depends_run)
341    } else {
342        set depends_run ""
343    }
344    if {[info exists portinfo(platforms)]} {
345        set platforms $portinfo(platforms)
346    } else {
347        set platforms ""
348    }
349    if {[info exists portinfo(license)]} {
350        set licenses $portinfo(license)
351    } else {
352        set licenses ""
353    }
354
355    puts $sqlfile_fd "INSERT INTO portfiles VALUES ('$portname', '$portdir', '$portversion', '$description');"
356
357    set primary 1
358    foreach category $categories {
359        set category [sql_escape $category]
360        puts $sqlfile_fd "INSERT INTO categories VALUES ('$portname', '$category', $primary);"
361        set primary 0
362    }
363   
364    set primary 1
365    foreach maintainer $maintainers {
366        set maintainer [sql_escape $maintainer]
367        puts $sqlfile_fd "INSERT INTO maintainers VALUES ('$portname', '$maintainer', $primary);"
368        set primary 0
369    }
370
371    foreach fetch_dep $depends_fetch {
372        set fetch_dep [sql_escape $fetch_dep]
373        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$fetch_dep');"
374    }
375   
376    foreach extract_dep $depends_extract {
377        set extract_dep [sql_escape $extract_dep]
378        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$extract_dep');"
379    }
380
381    foreach build_dep $depends_build {
382        set build_dep [sql_escape $build_dep]
383        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$build_dep');"
384    }
385
386    foreach lib $depends_lib {
387        set lib [sql_escape $lib]
388        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$lib');"
389    }
390
391    foreach run_dep $depends_run {
392        set run_dep [sql_escape $run_dep]
393        puts $sqlfile_fd "INSERT INTO dependencies VALUES ('$portname', '$run_dep');"
394    }
395
396    foreach variant $variants {
397        set variant [sql_escape $variant]
398        puts $sqlfile_fd "INSERT INTO variants VALUES ('$portname', '$variant');"
399    }
400
401    foreach platform $platforms {
402        set platform [sql_escape $platform]
403        puts $sqlfile_fd "INSERT INTO platforms VALUES ('$portname', '$platform');"
404    }
405
406    foreach license $licenses {
407        set license [sql_escape $license]
408        puts $sqlfile_fd "INSERT INTO licenses VALUES ('$portname', '$license');"
409    }
410
411}
412
413# Mark the db regen as done only once we're done processing all ports:
414puts $sqlfile_fd "INSERT INTO log VALUES ('update', NOW());"
415
416# Pipe the contents of the generated sql file to the database command,
417# reading from the file descriptor for the raw sql file to assure completeness.
418if {[catch {seek $sqlfile_fd 0 start} errstr]} {
419    ui_error "${::errorCode}: $errstr"
420    cleanup sqlfile lockfile
421    terminate 1
422}
423
424if {[catch {exec -- $portsdb_cmd --host=$portsdb_host --user=$portsdb_user --password=$portsdb_passwd --database=$portsdb_name <@ $sqlfile_fd} errstr]} {
425    ui_error "${::errorCode}: $errstr"
426    cleanup sqlfile lockfile
427    terminate 1
428}
429
430# done regenerating the database. Cleanup and exit successfully.
431cleanup sqlfile lockfile
432terminate 0
Note: See TracBrowser for help on using the repository browser.