euscan-ng/web/euscan-update

#!/usr/bin/env python

import subprocess
import portage
import sqlite3
import sys
import argparse

def package_id_by_name(c, catpkg):
	cat, pkg = catpkg.split('/')

	c.execute('SELECT id FROM packages WHERE category = ? AND package = ?', (cat, pkg))

	row = c.fetchone()

	if row:
		package_id = row[0]
	else:
		c.execute('INSERT OR IGNORE INTO packages (category, package) VALUES (?,?)', (cat, pkg))
		package_id = c.lastrowid
		print '[e] %s/%s' % (cat, pkg)

	return package_id

def store_package(c, cpv, slot):
	catpkg, ver, rev = portage.pkgsplit(cpv)
	cat, pkg = catpkg.split('/')
	package_id = -1

	package_id = package_id_by_name(c, catpkg)

	sql = 'INSERT OR IGNORE INTO versions (package_id, slot, revision, version, packaged) VALUES (?, ?, ?, ?, 1)'
	c.execute(sql, (package_id, slot, rev, ver))

	if c.lastrowid:
		print '[v] %s:%s' % (cpv, slot)

def portage_scan(db, package=None):
	c = db.cursor()

	cmd = ['eix', '--format', '<availableversions:NAMEVERSION>', '--pure-packages', '-x']
	if package:
		cmd.append(package)

	output1 = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]

	cmd = ['eix', '--format', '<availableversions:NAMEASLOT>', '--pure-packages', '-x']
	if package:
		cmd.append(package)
	output2 = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]

	output1 = output1.split('\n')
	output2 = output2.split('\n')

	for i in range(0, len(output1)):
		if not output1[i]:
			continue

		cpv = output1[i]
		slot = output2[i].split(':')[1]
		store_package(c, cpv, slot)

	db.commit()

def herd_id_by_name(cursor, herd):
	cursor.execute('SELECT id FROM herds WHERE herd = ?', (herd,))

	row = cursor.fetchone()

	if row:
		herd_id = row[0]
	else:
		cursor.execute('INSERT INTO herds (herd) VALUES (?)', (herd,))
		herd_id = cursor.lastrowid
		print '[h] %s' % (herd)

	return herd_id

def maintainer_id_by_name(cursor, maintainer):

	cursor.execute('SELECT id FROM maintainers WHERE maintainer = ?', (maintainer,))

	row = cursor.fetchone()

	if row:
		maintainer_id = row[0]
	else:
		cursor.execute('INSERT INTO maintainers (maintainer) VALUES (?)', (maintainer,))
		maintainer_id = cursor.lastrowid
		print '[m] %s' % (maintainer)

	return maintainer_id


def store_herd(cursor, catpkg, herd):
	if herd == 'no-herd':
		return
	package_id = package_id_by_name(cursor, catpkg)
	herd_id = herd_id_by_name(cursor, herd)
	print catpkg, herd
	cursor.execute('INSERT OR IGNORE INTO package_herds (herd_id, package_id) VALUES (?, ?)', (herd_id, package_id))

def store_maintainer(cursor, catpkg, maintainer):
	if maintainer == 'None specified':
		return
	package_id = package_id_by_name(cursor, catpkg)
	maintainer_id = maintainer_id_by_name(cursor, maintainer)
	print catpkg, maintainer
	cursor.execute('INSERT OR IGNORE INTO package_maintainers (maintainer_id, package_id) VALUES (?, ?)', (maintainer_id, package_id))

def metadata_scan_some(c, packages):
	cmd = ['epkginfo']
	cmd.extend(packages[:-1])
	output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
	output = output.split('\n\n')

	for infos in output:
		infos = infos.split('\n')

		package = packages.pop(0)

		for line in infos:
			if line.startswith('Herd:'):
				line = line.replace('Herd:', '').strip()
				store_herd(c, package, line)
			if line.startswith('Maintainer:'):
				line = line.replace('Maintainer:', '').strip()
				store_maintainer(c, package, line)

def metadata_scan(db, package=None):
	c = db.cursor()

	cmd = ['eix', '--only-names']

	if package:
		cmd.append(package)
	output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
        packages = output.split('\n')

	tmp = []
	for package in packages:
		tmp.append(package)
		if len(tmp) > 10:
			metadata_scan_some(c, tmp)
			tmp = []

	metadata_scan_some(c, tmp)

	db.commit()

def version_id_by_version(cursor, package_id, version):

	cursor.execute('SELECT id, packaged FROM versions WHERE package_id = ? AND version = ?', (package_id, version))

	row = cursor.fetchone()

	if row:
		version_id = row[0]
		packaged = row[1]
	else:
		cursor.execute('INSERT INTO versions (package_id, slot, revision, version, packaged) VALUES (?, ?, ?, ?, 0)',
			       (package_id, '', 'r0', version))
		version_id = cursor.lastrowid
		packaged = 0

	return version_id, packaged

def upstream_scan(db, package):
	c = db.cursor()

	cmd = ['eix', '--format', '<bestversion*:NAMEVERSION>', '--pure-packages']
	if package:
		cmd.append(package)
	output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
        packages = output.split('\n')

	for package in packages:
		if not package.strip():
			       continue
		catpkg, ver, rev = portage.pkgsplit(package)

		cmd = ['../euscan', package]
		output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
		output = output.split('\n')
		for line in output:
			if not line.startswith('New Upstream Version: '):
				continue
			line = line.replace('New Upstream Version: ', '').split(' ')
			ver = line[0]
			urls = line[1:]

			package_id = package_id_by_name(c, catpkg)
			version_id, packaged= version_id_by_version(c, package_id, ver)

			if packaged:
				continue

			print '[u] %s %s' % (ver, ' '.join(urls))

			for url in urls:
				c.execute('INSERT OR REPLACE INTO upstream_urls (version_id, url) VALUES (?, ?)', (version_id, url))
		db.commit()


def main():
	parser = argparse.ArgumentParser(description='Update euscan database.')
	parser.add_argument('--skip-portage', action='store_true', help='Skip portage scan.')
	parser.add_argument('--skip-metadata', action='store_true', help='Skip metadata scan.')
	parser.add_argument('package', nargs='*', help='Only check updates for these packages')

	args = parser.parse_args()

	db = sqlite3.connect('euscan.db')

	if not args.package:
		args.package = [None]

	for package in args.package:
		if not args.skip_portage:
			portage_scan(db, package)
		if not args.skip_metadata:
			metadata_scan(db, package)

		upstream_scan(db, package)

	db.close()

if __name__ == '__main__':
	main()