diff --git a/TODO b/TODO index da0f38f..e93e468 100644 --- a/TODO +++ b/TODO @@ -34,7 +34,7 @@ euscan - Convert subtree - Commit metadata.xml changes to tree -packages: +### packages: - MySQL: should use http://downloads.mysql.com/archives/ - mariadb: should use http://downloads.askmonty.org/MariaDB/+releases/ @@ -101,9 +101,6 @@ The scan process should be faster than the one using euscan. gentoo-packages hav ### Scanning - Remove alive field -- Find a way to rescan upstream of packages recently changed in portage: - - make scan_portage/scan_upstream return a list of packages that were updated (or list of versions ?) - - use that list in tasks.py as scan_upstream (should be easy using chains and subtasks) ### API (LOW-PRIORITY) diff --git a/euscanwww/djeuscan/management/commands/scan_portage.py b/euscanwww/djeuscan/management/commands/scan_portage.py index e40778c..e70affa 100644 --- a/euscanwww/djeuscan/management/commands/scan_portage.py +++ b/euscanwww/djeuscan/management/commands/scan_portage.py @@ -34,6 +34,11 @@ class Command(BaseCommand): dest='purge-versions', default=False, help='Purge old versions'), + make_option('--upstream', + action='store_true', + dest='upstream', + default=False, + help='Scan upstream for updated packages in Portage'), make_option('--no-log', action='store_true', dest='no-log', @@ -66,5 +71,6 @@ class Command(BaseCommand): purge_packages=options["purge-packages"], purge_versions=options["purge-versions"], prefetch=options["prefetch"], + upstream=options["upstream"], logger=logger, ) diff --git a/euscanwww/djeuscan/processing/scan/scan_portage.py b/euscanwww/djeuscan/processing/scan/scan_portage.py index fb450c2..1ce7197 100644 --- a/euscanwww/djeuscan/processing/scan/scan_portage.py +++ b/euscanwww/djeuscan/processing/scan/scan_portage.py @@ -13,16 +13,18 @@ from django.core.management.color import color_style from euscan.version import get_version_type from djeuscan.processing import FakeLogger +from djeuscan.processing.scan.scan_upstream import scan_upstream from djeuscan.models import Package, Version, VersionLog, Category, Overlay class ScanPortage(object): def __init__(self, logger=None, no_log=False, purge_packages=False, - purge_versions=False): + purge_versions=False, upstream=False): self.logger = logger or FakeLogger() self.no_log = no_log self.purge_packages = purge_packages self.purge_versions = purge_versions + self.upstream = upstream self.style = color_style() @@ -196,8 +198,15 @@ class ScanPortage(object): cat, pkg, data['homepage'], data['description'] ) packages_alive.add("%s/%s" % (cat, pkg)) + new_version = False for cpv, slot, overlay, overlay_path in data['versions']: - self.store_version(package, cpv, slot, overlay, overlay_path) + new_version = new_version or self.store_version( + package, cpv, slot, overlay, overlay_path + ) + + # If the package has at least one new version scan upstream for it + if new_version and self.upstream: + scan_upstream([package], self.purge_versions, self.logger) self.purge_old_packages(current_packages, packages_alive) self.purge_old_versions() @@ -260,8 +269,9 @@ class ScanPortage(object): # nothing to do (note: it can't be an upstream version because # overlay can't be empty here) if not created: - return + return False + # New version created self.logger.info('+ [v] %s' % (obj)) if overlay == 'gentoo': @@ -282,6 +292,8 @@ class ScanPortage(object): vtype=obj.vtype, ) + return True + def purge_old_packages(self, packages, alive): if not self.purge_packages: return @@ -324,7 +336,7 @@ class ScanPortage(object): @commit_on_success -def scan_portage(packages=None, category=None, no_log=False, +def scan_portage(packages=None, category=None, no_log=False, upstream=False, purge_packages=False, purge_versions=False, prefetch=False, logger=None): @@ -338,6 +350,7 @@ def scan_portage(packages=None, category=None, no_log=False, no_log=no_log, purge_packages=purge_packages, purge_versions=purge_versions, + upstream=upstream ) logger.info('Scanning portage tree...') diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py index b9aa92b..d3c1b5b 100644 --- a/euscanwww/djeuscan/tasks.py +++ b/euscanwww/djeuscan/tasks.py @@ -114,7 +114,7 @@ def scan_metadata(packages=[], category=None, populate=False): @task -def scan_portage(packages=[], category=None, +def scan_portage(packages=[], category=None, upstream=True, no_log=False, purge_packages=False, purge_versions=False, prefetch=False): """ @@ -134,6 +134,7 @@ def scan_portage(packages=[], category=None, scan.scan_portage( packages=packages, category=category, + upstream=upstream, no_log=no_log, purge_packages=purge_packages, purge_versions=purge_versions,