euscanwww/scan_portage: fixes and add --category

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>
This commit is contained in:
Corentin Chary 2012-07-02 09:43:31 +02:00
parent ad5e399096
commit 093cb4da47
2 changed files with 104 additions and 92 deletions

View File

@ -19,6 +19,11 @@ class Command(BaseCommand):
dest='all',
default=False,
help='Scan all packages'),
make_option('--category',
action='store',
dest='category',
default=None,
help='Scan only this category'),
make_option('--purge-packages',
action='store_true',
dest='purge-packages',
@ -47,7 +52,7 @@ class Command(BaseCommand):
def handle(self, *args, **options):
set_verbosity_level(logger, options.get("verbosity", 1))
if options['all']:
if options['all'] or options['category']:
packages = None
elif len(args):
packages = [pkg for pkg in args]
@ -56,6 +61,7 @@ class Command(BaseCommand):
scan_portage(
packages=packages,
category=options['category'],
no_log=options["no-log"],
purge_packages=options["purge-packages"],
purge_versions=options["purge-versions"],

View File

@ -56,11 +56,78 @@ class ScanPortage(object):
)
self._cache['versions'][key] = version
def scan(self, query=None):
def scan_eix_xml(self, query, category=None):
cmd = ['eix', '--xml']
if query:
cmd.extend(['--exact', query])
if category:
cmd.extend(['-C', category])
sub = subprocess.Popen(cmd, stdout=subprocess.PIPE)
output = sub.stdout
try:
parser = iterparse(output, ["start", "end"])
parser.next() # read root tag just for testing output
except ParseError:
if query:
msg = "Unknown package '%s'" % query
else:
msg = "No packages."
self.logger.error(self.style.ERROR(msg))
return
package = {'versions' : []}
category = ""
for event, elem in parser:
if event == "start": # on tag opening
if elem.tag == "category":
category = elem.attrib["name"]
elif elem.tag == "package":
package["package"] = elem.attrib["name"]
package["category"] = category
elif elem.tag in ["description", "homepage"]:
package[elem.tag] = elem.text or ""
elif elem.tag == "version":
# append version data to versions
cpv = "%s/%s-%s" % \
(package["category"], package["package"], elem.attrib["id"])
slot = elem.attrib.get("slot", "")
overlay = elem.attrib.get("repository", "gentoo")
package["versions"].append((cpv, slot, overlay))
elif event == "end": # on tag closing
if elem.tag == "package":
# clean old data
yield package
package = {"versions" : []}
if elem.tag == "category":
# clean old data
category = ""
elem.clear()
def prepare_purge_versions(self, packages, query=None, category=None):
if not self.purge_versions:
return
# Set all versions dead, then set found versions alive and
# delete old versions
if not query:
# Optimisation for --all or --category
self.logger.info('Killing existing versions...')
qs = Version.objects.filter(packaged=True)
if category:
qs.filter(package__category=category)
qs.update(alive=False)
self.logger.info('done')
else:
for package in packages:
Version.objects.filter(package=package, packaged=True).\
update(alive=False)
def scan(self, query=None, category=None):
if not query:
current_packages = Package.objects.all()
elif '/' in query:
@ -68,78 +135,22 @@ class ScanPortage(object):
current_packages = Package.objects.filter(category=cat, name=pkg)
else:
current_packages = Package.objects.filter(name=query)
if category:
current_packages = current_packages.filter(category=category)
if self.purge_versions:
if not query:
self.logger.info('Killing existing versions...')
Version.objects.filter(packaged=True).update(alive=False)
self.logger.info('done')
else:
for package in current_packages:
Version.objects.filter(package=package, packaged=True).\
update(alive=False)
self.prepare_purge_versions(current_packages, query, category)
sub = subprocess.Popen(cmd, stdout=subprocess.PIPE)
output = sub.stdout
try:
parser = iterparse(output, ["start", "end"])
parser.next() # read root tag just for testing output
except ParseError:
self.logger.error(
self.style.ERROR(
"Unknown package '%s'" % query
)
)
else:
cat, pkg, homepage, desc = ("", "", "", "")
versions = []
packages_alive = set()
for event, elem in parser:
if event == "start": # on tag opening
if elem.tag == "category":
cat = elem.attrib["name"]
if elem.tag == "package":
pkg = elem.attrib["name"]
if elem.tag == "description":
desc = elem.text or ""
if elem.tag == "homepage":
homepage = elem.text or ""
if elem.tag == "version":
# append version data to versions
cpv = "%s/%s-%s" % (cat, pkg, elem.attrib["id"])
slot = elem.attrib.get("slot", "")
overlay = elem.attrib.get("overlay", "")
versions.append((cpv, slot, overlay))
elif event == "end": # on tag closing
if elem.tag == "package":
# package tag has been closed, saving everything!
package = self.store_package(cat, pkg, homepage,
desc)
packages_alive.add('%s/%s' % (cat, pkg))
for cpv, slot, overlay in versions:
for data in self.scan_eix_xml(query, category):
cat, pkg = data['category'], data['package']
package = self.store_package(cat, pkg, data['homepage'], data['description'])
packages_alive.add("%s/%s" % (cat, pkg))
for cpv, slot, overlay in data['versions']:
self.store_version(package, cpv, slot, overlay)
# clean old data
pkg, homepage, desc = ("", "", "")
versions = []
if elem.tag == "category":
# clean old data
cat = ""
elem.clear()
if self.purge_packages:
for package in current_packages:
cp = "%s/%s" % (package.category, package.name)
if cp not in packages_alive:
self.logger.info('- [p] %s' % (package))
package.delete()
if self.purge_versions:
self.purge_old_versions(current_packages)
self.purge_old_packages(current_packages, packages_alive)
self.purge_old_versions()
def store_package(self, cat, pkg, homepage, description):
created = False
@ -157,14 +168,6 @@ class ScanPortage(object):
if created:
self.logger.info('+ [p] %s/%s' % (cat, pkg))
# Set all versions dead, then set found versions alive and
# delete old versions
if not self.purge_versions:
Version.objects.filter(
package=obj,
packaged=True
).update(alive=False)
return obj
def store_version(self, package, cpv, slot, overlay):
@ -223,18 +226,21 @@ class ScanPortage(object):
overlay=obj.overlay
)
def purge_old_versions(self, packages):
# For each dead versions
if packages:
versions = []
for package in packages:
qs = Version.objects.filter(package=package, packaged=True,
alive=False)
for version in qs:
versions.append(version)
else:
versions = Version.objects.filter(packaged=True, alive=False)
def purge_old_packages(self, packages, alive):
if not self.purge_packages:
return
for package in packages:
cp = "%s/%s" % (package.category, package.name)
if cp not in alive:
self.logger.info('- [p] %s' % (package))
package.delete()
def purge_old_versions(self):
if not self.purge_versions:
return
versions = Version.objects.filter(packaged=True, alive=False)
for version in versions:
if version.overlay == 'gentoo':
version.package.n_packaged -= 1
@ -257,11 +263,11 @@ class ScanPortage(object):
overlay=version.overlay
)
Version.objects.filter(packaged=True, alive=False).delete()
versions.delete()
@commit_on_success
def scan_portage(packages=None, no_log=False, purge_packages=False,
def scan_portage(packages=None, category=None, no_log=False, purge_packages=False,
purge_versions=False, prefetch=False, logger=None):
logger = logger or FakeLogger()
@ -287,7 +293,7 @@ def scan_portage(packages=None, no_log=False, purge_packages=False,
logger.info('done')
if not packages:
scan_handler.scan()
scan_handler.scan(category=category)
else:
for pkg in packages:
if isinstance(pkg, Package):