euscan-ng/web/euscan-update

230 lines
5.9 KiB
Plaintext
Raw Normal View History

#!/usr/bin/env python
import subprocess
import portage
import sqlite3
import sys
import argparse
def package_id_by_name(c, catpkg):
cat, pkg = catpkg.split('/')
c.execute('SELECT id FROM packages WHERE category = ? AND package = ?', (cat, pkg))
row = c.fetchone()
if row:
package_id = row[0]
else:
c.execute('INSERT OR IGNORE INTO packages (category, package) VALUES (?,?)', (cat, pkg))
package_id = c.lastrowid
print '[e] %s/%s' % (cat, pkg)
return package_id
def store_package(c, cpv, slot):
catpkg, ver, rev = portage.pkgsplit(cpv)
cat, pkg = catpkg.split('/')
package_id = -1
package_id = package_id_by_name(c, catpkg)
sql = 'INSERT OR IGNORE INTO versions (package_id, slot, revision, version, packaged) VALUES (?, ?, ?, ?, 1)'
c.execute(sql, (package_id, slot, rev, ver))
if c.lastrowid:
print '[v] %s:%s' % (cpv, slot)
def portage_scan(db, package=None):
c = db.cursor()
cmd = ['eix', '--format', '<availableversions:NAMEVERSION>', '--pure-packages', '-x']
if package:
cmd.append(package)
output1 = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
cmd = ['eix', '--format', '<availableversions:NAMEASLOT>', '--pure-packages', '-x']
if package:
cmd.append(package)
output2 = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
output1 = output1.split('\n')
output2 = output2.split('\n')
for i in range(0, len(output1)):
if not output1[i]:
continue
cpv = output1[i]
slot = output2[i].split(':')[1]
store_package(c, cpv, slot)
db.commit()
def herd_id_by_name(cursor, herd):
cursor.execute('SELECT id FROM herds WHERE herd = ?', (herd,))
row = cursor.fetchone()
if row:
herd_id = row[0]
else:
cursor.execute('INSERT INTO herds (herd) VALUES (?)', (herd,))
herd_id = cursor.lastrowid
print '[h] %s' % (herd)
return herd_id
def maintainer_id_by_name(cursor, maintainer):
cursor.execute('SELECT id FROM maintainers WHERE maintainer = ?', (maintainer,))
row = cursor.fetchone()
if row:
maintainer_id = row[0]
else:
cursor.execute('INSERT INTO maintainers (maintainer) VALUES (?)', (maintainer,))
maintainer_id = cursor.lastrowid
print '[m] %s' % (maintainer)
return maintainer_id
def store_herd(cursor, catpkg, herd):
if herd == 'no-herd':
return
package_id = package_id_by_name(cursor, catpkg)
herd_id = herd_id_by_name(cursor, herd)
print catpkg, herd
cursor.execute('INSERT OR IGNORE INTO package_herds (herd_id, package_id) VALUES (?, ?)', (herd_id, package_id))
def store_maintainer(cursor, catpkg, maintainer):
if maintainer == 'None specified':
return
package_id = package_id_by_name(cursor, catpkg)
maintainer_id = maintainer_id_by_name(cursor, maintainer)
print catpkg, maintainer
cursor.execute('INSERT OR IGNORE INTO package_maintainers (maintainer_id, package_id) VALUES (?, ?)', (maintainer_id, package_id))
def metadata_scan_some(c, packages):
cmd = ['epkginfo']
cmd.extend(packages[:-1])
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
output = output.split('\n\n')
for infos in output:
infos = infos.split('\n')
package = packages.pop(0)
for line in infos:
if line.startswith('Herd:'):
line = line.replace('Herd:', '').strip()
store_herd(c, package, line)
if line.startswith('Maintainer:'):
line = line.replace('Maintainer:', '').strip()
store_maintainer(c, package, line)
def metadata_scan(db, package=None):
c = db.cursor()
cmd = ['eix', '--only-names']
if package:
cmd.append(package)
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
packages = output.split('\n')
tmp = []
for package in packages:
tmp.append(package)
if len(tmp) > 10:
metadata_scan_some(c, tmp)
tmp = []
metadata_scan_some(c, tmp)
db.commit()
def version_id_by_version(cursor, package_id, version):
cursor.execute('SELECT id, packaged FROM versions WHERE package_id = ? AND version = ?', (package_id, version))
row = cursor.fetchone()
if row:
version_id = row[0]
packaged = row[1]
else:
cursor.execute('INSERT INTO versions (package_id, slot, revision, version, packaged) VALUES (?, ?, ?, ?, 0)',
(package_id, '', 'r0', version))
version_id = cursor.lastrowid
packaged = 0
return version_id, packaged
def upstream_scan(db, package):
c = db.cursor()
cmd = ['eix', '--format', '<bestversion*:NAMEVERSION>', '--pure-packages']
if package:
cmd.append(package)
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
packages = output.split('\n')
for package in packages:
if not package.strip():
continue
catpkg, ver, rev = portage.pkgsplit(package)
cmd = ['../euscan', package]
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
output = output.split('\n')
for line in output:
if not line.startswith('New Upstream Version: '):
continue
line = line.replace('New Upstream Version: ', '').split(' ')
ver = line[0]
urls = line[1:]
package_id = package_id_by_name(c, catpkg)
version_id, packaged= version_id_by_version(c, package_id, ver)
if packaged:
continue
print '[u] %s %s' % (ver, ' '.join(urls))
for url in urls:
c.execute('INSERT OR REPLACE INTO upstream_urls (version_id, url) VALUES (?, ?)', (version_id, url))
db.commit()
def main():
parser = argparse.ArgumentParser(description='Update euscan database.')
parser.add_argument('--skip-portage', action='store_true', help='Skip portage scan.')
parser.add_argument('--skip-metadata', action='store_true', help='Skip metadata scan.')
parser.add_argument('package', nargs='*', help='Only check updates for these packages')
args = parser.parse_args()
db = sqlite3.connect('euscan.db')
if not args.package:
args.package = [None]
for package in args.package:
if not args.skip_portage:
portage_scan(db, package)
if not args.skip_metadata:
metadata_scan(db, package)
upstream_scan(db, package)
db.close()
if __name__ == '__main__':
main()