#!/usr/bin/env python import subprocess import portage import sqlite3 import sys import argparse def package_id_by_name(c, catpkg): cat, pkg = catpkg.split('/') c.execute('SELECT id FROM packages WHERE category = ? AND package = ?', (cat, pkg)) row = c.fetchone() if row: package_id = row[0] else: c.execute('INSERT OR IGNORE INTO packages (category, package) VALUES (?,?)', (cat, pkg)) package_id = c.lastrowid print '[e] %s/%s' % (cat, pkg) return package_id def store_package(c, cpv, slot): catpkg, ver, rev = portage.pkgsplit(cpv) cat, pkg = catpkg.split('/') package_id = -1 package_id = package_id_by_name(c, catpkg) sql = 'INSERT OR IGNORE INTO versions (package_id, slot, revision, version, packaged) VALUES (?, ?, ?, ?, 1)' c.execute(sql, (package_id, slot, rev, ver)) if c.lastrowid: print '[v] %s:%s' % (cpv, slot) def portage_scan(db, package=None): c = db.cursor() cmd = ['eix', '--format', '', '--pure-packages', '-x'] if package: cmd.append(package) output1 = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] cmd = ['eix', '--format', '', '--pure-packages', '-x'] if package: cmd.append(package) output2 = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] output1 = output1.split('\n') output2 = output2.split('\n') for i in range(0, len(output1)): if not output1[i]: continue cpv = output1[i] slot = output2[i].split(':')[1] store_package(c, cpv, slot) db.commit() def herd_id_by_name(cursor, herd): cursor.execute('SELECT id FROM herds WHERE herd = ?', (herd,)) row = cursor.fetchone() if row: herd_id = row[0] else: cursor.execute('INSERT INTO herds (herd) VALUES (?)', (herd,)) herd_id = cursor.lastrowid print '[h] %s' % (herd) return herd_id def maintainer_id_by_name(cursor, maintainer): cursor.execute('SELECT id FROM maintainers WHERE maintainer = ?', (maintainer,)) row = cursor.fetchone() if row: maintainer_id = row[0] else: cursor.execute('INSERT INTO maintainers (maintainer) VALUES (?)', (maintainer,)) maintainer_id = cursor.lastrowid print '[m] %s' % (maintainer) return maintainer_id def store_herd(cursor, catpkg, herd): if herd == 'no-herd': return package_id = package_id_by_name(cursor, catpkg) herd_id = herd_id_by_name(cursor, herd) print catpkg, herd cursor.execute('INSERT OR IGNORE INTO package_herds (herd_id, package_id) VALUES (?, ?)', (herd_id, package_id)) def store_maintainer(cursor, catpkg, maintainer): if maintainer == 'None specified': return package_id = package_id_by_name(cursor, catpkg) maintainer_id = maintainer_id_by_name(cursor, maintainer) print catpkg, maintainer cursor.execute('INSERT OR IGNORE INTO package_maintainers (maintainer_id, package_id) VALUES (?, ?)', (maintainer_id, package_id)) def metadata_scan_some(c, packages): cmd = ['epkginfo'] cmd.extend(packages[:-1]) output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] output = output.split('\n\n') for infos in output: infos = infos.split('\n') package = packages.pop(0) for line in infos: if line.startswith('Herd:'): line = line.replace('Herd:', '').strip() store_herd(c, package, line) if line.startswith('Maintainer:'): line = line.replace('Maintainer:', '').strip() store_maintainer(c, package, line) def metadata_scan(db, package=None): c = db.cursor() cmd = ['eix', '--only-names'] if package: cmd.append(package) output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] packages = output.split('\n') tmp = [] for package in packages: tmp.append(package) if len(tmp) > 10: metadata_scan_some(c, tmp) tmp = [] metadata_scan_some(c, tmp) db.commit() def version_id_by_version(cursor, package_id, version): cursor.execute('SELECT id, packaged FROM versions WHERE package_id = ? AND version = ?', (package_id, version)) row = cursor.fetchone() if row: version_id = row[0] packaged = row[1] else: cursor.execute('INSERT INTO versions (package_id, slot, revision, version, packaged) VALUES (?, ?, ?, ?, 0)', (package_id, '', 'r0', version)) version_id = cursor.lastrowid packaged = 0 return version_id, packaged def upstream_scan(db, package): c = db.cursor() cmd = ['eix', '--format', '', '--pure-packages'] if package: cmd.append(package) output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] packages = output.split('\n') for package in packages: if not package.strip(): continue catpkg, ver, rev = portage.pkgsplit(package) cmd = ['../euscan', package] output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] output = output.split('\n') for line in output: if not line.startswith('New Upstream Version: '): continue line = line.replace('New Upstream Version: ', '').split(' ') ver = line[0] urls = line[1:] package_id = package_id_by_name(c, catpkg) version_id, packaged= version_id_by_version(c, package_id, ver) if packaged: continue print '[u] %s %s' % (ver, ' '.join(urls)) for url in urls: c.execute('INSERT OR REPLACE INTO upstream_urls (version_id, url) VALUES (?, ?)', (version_id, url)) db.commit() def main(): parser = argparse.ArgumentParser(description='Update euscan database.') parser.add_argument('--skip-portage', action='store_true', help='Skip portage scan.') parser.add_argument('--skip-metadata', action='store_true', help='Skip metadata scan.') parser.add_argument('package', nargs='*', help='Only check updates for these packages') args = parser.parse_args() db = sqlite3.connect('euscan.db') if not args.package: args.package = [None] for package in args.package: if not args.skip_portage: portage_scan(db, package) if not args.skip_metadata: metadata_scan(db, package) upstream_scan(db, package) db.close() if __name__ == '__main__': main()