euscanwww: Adding an option to run upstream scanning after an update in portage

Signed-off-by: volpino <fox91@anche.no>

TODO

@@ -34,7 +34,7 @@ euscan
 - Convert subtree
 - Commit metadata.xml changes to tree
 
-packages:
+### packages:
 - MySQL: should use http://downloads.mysql.com/archives/
 - mariadb: should use http://downloads.askmonty.org/MariaDB/+releases/
 
@@ -101,9 +101,6 @@ The scan process should be faster than the one using euscan. gentoo-packages hav
 ### Scanning
 
 - Remove alive field
-- Find a way to rescan upstream of packages recently changed in portage:
-  - make scan_portage/scan_upstream return a list of packages that were updated (or list of versions ?)
-  - use that list in tasks.py as scan_upstream (should be easy using chains and subtasks)
 
 ### API (LOW-PRIORITY)
 

euscanwww/djeuscan/management/commands/scan_portage.py

@@ -34,6 +34,11 @@ class Command(BaseCommand):
             dest='purge-versions',
             default=False,
             help='Purge old versions'),
+        make_option('--upstream',
+            action='store_true',
+            dest='upstream',
+            default=False,
+            help='Scan upstream for updated packages in Portage'),
         make_option('--no-log',
             action='store_true',
             dest='no-log',
@@ -66,5 +71,6 @@ class Command(BaseCommand):
             purge_packages=options["purge-packages"],
             purge_versions=options["purge-versions"],
             prefetch=options["prefetch"],
+            upstream=options["upstream"],
             logger=logger,
         )

euscanwww/djeuscan/processing/scan/scan_portage.py

@@ -13,16 +13,18 @@ from django.core.management.color import color_style
 from euscan.version import get_version_type
 
 from djeuscan.processing import FakeLogger
+from djeuscan.processing.scan.scan_upstream import scan_upstream
 from djeuscan.models import Package, Version, VersionLog, Category, Overlay
 
 
 class ScanPortage(object):
     def __init__(self, logger=None, no_log=False, purge_packages=False,
-                 purge_versions=False):
+                 purge_versions=False, upstream=False):
         self.logger = logger or FakeLogger()
         self.no_log = no_log
         self.purge_packages = purge_packages
         self.purge_versions = purge_versions
+        self.upstream = upstream
 
         self.style = color_style()
 
@@ -196,8 +198,15 @@ class ScanPortage(object):
                 cat, pkg, data['homepage'], data['description']
             )
             packages_alive.add("%s/%s" % (cat, pkg))
+            new_version = False
             for cpv, slot, overlay, overlay_path in data['versions']:
-                self.store_version(package, cpv, slot, overlay, overlay_path)
+                new_version = new_version or self.store_version(
+                    package, cpv, slot, overlay, overlay_path
+                )
+
+            # If the package has at least one new version scan upstream for it
+            if new_version and self.upstream:
+                scan_upstream([package], self.purge_versions, self.logger)
 
         self.purge_old_packages(current_packages, packages_alive)
         self.purge_old_versions()
@@ -260,8 +269,9 @@ class ScanPortage(object):
         # nothing to do (note: it can't be an upstream version because
         # overlay can't be empty here)
         if not created:
-            return
+            return False
 
+        # New version created
         self.logger.info('+ [v] %s' % (obj))
 
         if overlay == 'gentoo':
@@ -282,6 +292,8 @@ class ScanPortage(object):
                 vtype=obj.vtype,
             )
 
+        return True
+
     def purge_old_packages(self, packages, alive):
         if not self.purge_packages:
             return
@@ -324,7 +336,7 @@ class ScanPortage(object):
 
 
 @commit_on_success
-def scan_portage(packages=None, category=None, no_log=False,
+def scan_portage(packages=None, category=None, no_log=False, upstream=False,
                  purge_packages=False, purge_versions=False, prefetch=False,
                  logger=None):
 
@@ -338,6 +350,7 @@ def scan_portage(packages=None, category=None, no_log=False,
         no_log=no_log,
         purge_packages=purge_packages,
         purge_versions=purge_versions,
+        upstream=upstream
     )
 
     logger.info('Scanning portage tree...')

euscanwww/djeuscan/tasks.py

@@ -113,7 +113,7 @@ def scan_metadata(packages=[], category=None, populate=False):
 
 
 @task
-def scan_portage(packages=[], category=None,
+def scan_portage(packages=[], category=None, upstream=True,
                  no_log=False, purge_packages=False,
                  purge_versions=False, prefetch=False):
     """
@@ -133,6 +133,7 @@ def scan_portage(packages=[], category=None,
     scan.scan_portage(
         packages=packages,
         category=category,
+        upstream=upstream,
         no_log=no_log,
         purge_packages=purge_packages,
         purge_versions=purge_versions,