Corentin Chary
229
web/euscan-update
Executable file
229
web/euscan-update
Executable file
@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import subprocess
|
||||
import portage
|
||||
import sqlite3
|
||||
import sys
|
||||
import argparse
|
||||
|
||||
def package_id_by_name(c, catpkg):
|
||||
cat, pkg = catpkg.split('/')
|
||||
|
||||
c.execute('SELECT id FROM packages WHERE category = ? AND package = ?', (cat, pkg))
|
||||
|
||||
row = c.fetchone()
|
||||
|
||||
if row:
|
||||
package_id = row[0]
|
||||
else:
|
||||
c.execute('INSERT OR IGNORE INTO packages (category, package) VALUES (?,?)', (cat, pkg))
|
||||
package_id = c.lastrowid
|
||||
print '[e] %s/%s' % (cat, pkg)
|
||||
|
||||
return package_id
|
||||
|
||||
def store_package(c, cpv, slot):
|
||||
catpkg, ver, rev = portage.pkgsplit(cpv)
|
||||
cat, pkg = catpkg.split('/')
|
||||
package_id = -1
|
||||
|
||||
package_id = package_id_by_name(c, catpkg)
|
||||
|
||||
sql = 'INSERT OR IGNORE INTO versions (package_id, slot, revision, version, packaged) VALUES (?, ?, ?, ?, 1)'
|
||||
c.execute(sql, (package_id, slot, rev, ver))
|
||||
|
||||
if c.lastrowid:
|
||||
print '[v] %s:%s' % (cpv, slot)
|
||||
|
||||
def portage_scan(db, package=None):
|
||||
c = db.cursor()
|
||||
|
||||
cmd = ['eix', '--format', '<availableversions:NAMEVERSION>', '--pure-packages', '-x']
|
||||
if package:
|
||||
cmd.append(package)
|
||||
|
||||
output1 = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
|
||||
|
||||
cmd = ['eix', '--format', '<availableversions:NAMEASLOT>', '--pure-packages', '-x']
|
||||
if package:
|
||||
cmd.append(package)
|
||||
output2 = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
|
||||
|
||||
output1 = output1.split('\n')
|
||||
output2 = output2.split('\n')
|
||||
|
||||
for i in range(0, len(output1)):
|
||||
if not output1[i]:
|
||||
continue
|
||||
|
||||
cpv = output1[i]
|
||||
slot = output2[i].split(':')[1]
|
||||
store_package(c, cpv, slot)
|
||||
|
||||
db.commit()
|
||||
|
||||
def herd_id_by_name(cursor, herd):
|
||||
cursor.execute('SELECT id FROM herds WHERE herd = ?', (herd,))
|
||||
|
||||
row = cursor.fetchone()
|
||||
|
||||
if row:
|
||||
herd_id = row[0]
|
||||
else:
|
||||
cursor.execute('INSERT INTO herds (herd) VALUES (?)', (herd,))
|
||||
herd_id = cursor.lastrowid
|
||||
print '[h] %s' % (herd)
|
||||
|
||||
return herd_id
|
||||
|
||||
def maintainer_id_by_name(cursor, maintainer):
|
||||
|
||||
cursor.execute('SELECT id FROM maintainers WHERE maintainer = ?', (maintainer,))
|
||||
|
||||
row = cursor.fetchone()
|
||||
|
||||
if row:
|
||||
maintainer_id = row[0]
|
||||
else:
|
||||
cursor.execute('INSERT INTO maintainers (maintainer) VALUES (?)', (maintainer,))
|
||||
maintainer_id = cursor.lastrowid
|
||||
print '[m] %s' % (maintainer)
|
||||
|
||||
return maintainer_id
|
||||
|
||||
|
||||
def store_herd(cursor, catpkg, herd):
|
||||
if herd == 'no-herd':
|
||||
return
|
||||
package_id = package_id_by_name(cursor, catpkg)
|
||||
herd_id = herd_id_by_name(cursor, herd)
|
||||
print catpkg, herd
|
||||
cursor.execute('INSERT OR IGNORE INTO package_herds (herd_id, package_id) VALUES (?, ?)', (herd_id, package_id))
|
||||
|
||||
def store_maintainer(cursor, catpkg, maintainer):
|
||||
if maintainer == 'None specified':
|
||||
return
|
||||
package_id = package_id_by_name(cursor, catpkg)
|
||||
maintainer_id = maintainer_id_by_name(cursor, maintainer)
|
||||
print catpkg, maintainer
|
||||
cursor.execute('INSERT OR IGNORE INTO package_maintainers (maintainer_id, package_id) VALUES (?, ?)', (maintainer_id, package_id))
|
||||
|
||||
def metadata_scan_some(c, packages):
|
||||
cmd = ['epkginfo']
|
||||
cmd.extend(packages[:-1])
|
||||
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
|
||||
output = output.split('\n\n')
|
||||
|
||||
for infos in output:
|
||||
infos = infos.split('\n')
|
||||
|
||||
package = packages.pop(0)
|
||||
|
||||
for line in infos:
|
||||
if line.startswith('Herd:'):
|
||||
line = line.replace('Herd:', '').strip()
|
||||
store_herd(c, package, line)
|
||||
if line.startswith('Maintainer:'):
|
||||
line = line.replace('Maintainer:', '').strip()
|
||||
store_maintainer(c, package, line)
|
||||
|
||||
def metadata_scan(db, package=None):
|
||||
c = db.cursor()
|
||||
|
||||
cmd = ['eix', '--only-names']
|
||||
|
||||
if package:
|
||||
cmd.append(package)
|
||||
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
|
||||
packages = output.split('\n')
|
||||
|
||||
tmp = []
|
||||
for package in packages:
|
||||
tmp.append(package)
|
||||
if len(tmp) > 10:
|
||||
metadata_scan_some(c, tmp)
|
||||
tmp = []
|
||||
|
||||
metadata_scan_some(c, tmp)
|
||||
|
||||
db.commit()
|
||||
|
||||
def version_id_by_version(cursor, package_id, version):
|
||||
|
||||
cursor.execute('SELECT id, packaged FROM versions WHERE package_id = ? AND version = ?', (package_id, version))
|
||||
|
||||
row = cursor.fetchone()
|
||||
|
||||
if row:
|
||||
version_id = row[0]
|
||||
packaged = row[1]
|
||||
else:
|
||||
cursor.execute('INSERT INTO versions (package_id, slot, revision, version, packaged) VALUES (?, ?, ?, ?, 0)',
|
||||
(package_id, '', 'r0', version))
|
||||
version_id = cursor.lastrowid
|
||||
packaged = 0
|
||||
|
||||
return version_id, packaged
|
||||
|
||||
def upstream_scan(db, package):
|
||||
c = db.cursor()
|
||||
|
||||
cmd = ['eix', '--format', '<bestversion*:NAMEVERSION>', '--pure-packages']
|
||||
if package:
|
||||
cmd.append(package)
|
||||
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
|
||||
packages = output.split('\n')
|
||||
|
||||
for package in packages:
|
||||
if not package.strip():
|
||||
continue
|
||||
catpkg, ver, rev = portage.pkgsplit(package)
|
||||
|
||||
cmd = ['../euscan', package]
|
||||
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
|
||||
output = output.split('\n')
|
||||
for line in output:
|
||||
if not line.startswith('New Upstream Version: '):
|
||||
continue
|
||||
line = line.replace('New Upstream Version: ', '').split(' ')
|
||||
ver = line[0]
|
||||
urls = line[1:]
|
||||
|
||||
package_id = package_id_by_name(c, catpkg)
|
||||
version_id, packaged= version_id_by_version(c, package_id, ver)
|
||||
|
||||
if packaged:
|
||||
continue
|
||||
|
||||
print '[u] %s %s' % (ver, ' '.join(urls))
|
||||
|
||||
for url in urls:
|
||||
c.execute('INSERT OR REPLACE INTO upstream_urls (version_id, url) VALUES (?, ?)', (version_id, url))
|
||||
db.commit()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Update euscan database.')
|
||||
parser.add_argument('--skip-portage', action='store_true', help='Skip portage scan.')
|
||||
parser.add_argument('--skip-metadata', action='store_true', help='Skip metadata scan.')
|
||||
parser.add_argument('package', nargs='*', help='Only check updates for these packages')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
db = sqlite3.connect('euscan.db')
|
||||
|
||||
if not args.package:
|
||||
args.package = [None]
|
||||
|
||||
for package in args.package:
|
||||
if not args.skip_portage:
|
||||
portage_scan(db, package)
|
||||
if not args.skip_metadata:
|
||||
metadata_scan(db, package)
|
||||
|
||||
upstream_scan(db, package)
|
||||
|
||||
db.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user