djeuscan/processing: fix scan_upstream after scan_portage

Signed-off-by: Corentin Chary <corentin.chary@gmail.com>

euscanwww/djeuscan/management/commands/scan_portage.py

@@ -34,11 +34,6 @@ class Command(BaseCommand):
             dest='purge-versions',
             default=False,
             help='Purge old versions'),
-        make_option('--upstream',
-            action='store_true',
-            dest='upstream',
-            default=False,
-            help='Scan upstream for updated packages in Portage'),
         make_option('--no-log',
             action='store_true',
             dest='no-log',
@@ -71,6 +66,5 @@ class Command(BaseCommand):
             purge_packages=options["purge-packages"],
             purge_versions=options["purge-versions"],
             prefetch=options["prefetch"],
-            upstream=options["upstream"],
             logger=logger,
         )

euscanwww/djeuscan/processing/misc/update_counters.py

@@ -114,7 +114,6 @@ def update_counters(fast=False, nolog=False, logger=None):
             package.last_version_upstream_id = last_versions_upstream.get(
                 package.id, default
             )['id']
-
             package.save()
 
         n_packages_gentoo = int(package.n_packaged == package.n_versions)

euscanwww/djeuscan/processing/scan/scan_portage.py

@@ -13,23 +13,25 @@ from django.core.management.color import color_style
 from euscan.version import get_version_type
 
 from djeuscan.processing import FakeLogger
-from djeuscan.processing.scan.scan_upstream import scan_upstream
 from djeuscan.models import Package, Version, VersionLog, Category, Overlay
 
 
 class ScanPortage(object):
     def __init__(self, logger=None, no_log=False, purge_packages=False,
-                 purge_versions=False, upstream=False):
+                 purge_versions=False):
         self.logger = logger or FakeLogger()
         self.no_log = no_log
         self.purge_packages = purge_packages
         self.purge_versions = purge_versions
-        self.upstream = upstream
 
         self.style = color_style()
 
         self._cache = {'packages': {}, 'versions': {}}
         self._overlays = None
+        self._updated_packages = set()
+
+    def updated_packages(self):
+        return list(self._updated_packages)
 
     def cache_hash_package(self, category, name):
         return '%s/%s' % (category, name)
@@ -200,13 +202,14 @@ class ScanPortage(object):
             packages_alive.add("%s/%s" % (cat, pkg))
             new_version = False
             for cpv, slot, overlay, overlay_path in data['versions']:
-                new_version = new_version or self.store_version(
+                obj, created = self.store_version(
                     package, cpv, slot, overlay, overlay_path
                 )
+                new_version = created or new_version
 
             # If the package has at least one new version scan upstream for it
-            if new_version and self.upstream:
+            if new_version:
-                scan_upstream([package], self.purge_versions, self.logger)
+                self._updated_packages.add(package)
 
         self.purge_old_packages(current_packages, packages_alive)
         self.purge_old_versions()
@@ -269,7 +272,7 @@ class ScanPortage(object):
         # nothing to do (note: it can't be an upstream version because
         # overlay can't be empty here)
         if not created:
-            return False
+            return obj, created
 
         # New version created
         self.logger.info('+ [v] %s' % (obj))
@@ -292,7 +295,7 @@ class ScanPortage(object):
                 vtype=obj.vtype,
             )
 
-        return True
+        return obj, created
 
     def purge_old_packages(self, packages, alive):
         if not self.purge_packages:
@@ -336,7 +339,7 @@ class ScanPortage(object):
 
 
 @commit_on_success
-def scan_portage(packages=None, category=None, no_log=False, upstream=False,
+def scan_portage(packages=None, category=None, no_log=False,
                  purge_packages=False, purge_versions=False, prefetch=False,
                  logger=None):
 
@@ -349,8 +352,7 @@ def scan_portage(packages=None, category=None, no_log=False, upstream=False,
         logger=logger,
         no_log=no_log,
         purge_packages=purge_packages,
-        purge_versions=purge_versions,
+        purge_versions=purge_versions
-        upstream=upstream
     )
 
     logger.info('Scanning portage tree...')
@@ -368,8 +370,10 @@ def scan_portage(packages=None, category=None, no_log=False, upstream=False,
             scan_handler.cache_store_version(version)
         logger.info('done')
 
-    if not packages:
+    if not packages and category:
         scan_handler.scan(category=category)
+    elif not packages:
+        scan_handler.scan()
     else:
         for pkg in packages:
             if isinstance(pkg, Package):
@@ -394,3 +398,4 @@ def scan_portage(packages=None, category=None, no_log=False, upstream=False,
             logger.info("+ [o] %s", overlay["overlay"])
 
     logger.info('Done.')
+    return scan_handler.updated_packages()

euscanwww/djeuscan/processing/scan/scan_upstream.py

@@ -34,7 +34,7 @@ class ScanUpstream(object):
                 "Error while scanning upstream for package %s!",
                 package
             )
-            self.logger.info(
+            self.logger.debug(
                 "Error %s",
                 out_json
             )

euscanwww/djeuscan/tasks.py

@@ -113,7 +113,7 @@ def scan_metadata(packages=[], category=None, populate=False):
 
 
 @task
-def scan_portage(packages=[], category=None, upstream=True,
+def scan_portage(packages=None, category=None,
                  no_log=False, purge_packages=False,
                  purge_versions=False, prefetch=False):
     """
@@ -130,17 +130,15 @@ def scan_portage(packages=[], category=None, upstream=True,
     else:
         logger.info("Starting portage scan...")
 
-    scan.scan_portage(
+    return scan.scan_portage(
         packages=packages,
         category=category,
-        upstream=upstream,
         no_log=no_log,
         purge_packages=purge_packages,
         purge_versions=purge_versions,
         prefetch=prefetch,
         logger=logger,
     )
-    return True
 
 
 @task
@@ -180,9 +178,17 @@ def update_portage(packages=None):
 
     # Workaround for celery bug when chaining groups
     update_portage_trees()
-    scan_portage(packages=[], purge_packages=True, purge_versions=True,
+    updated_packages = scan_portage(
-                 prefetch=True)
+        packages=None,
-    scan_metadata(packages=[], populate=True)
+        purge_packages=True,
+        purge_versions=True,
+        prefetch=True
+    )
+    scan_metadata(packages=None, populate=True)
+    if updated_packages:
+            group_chunks(scan_upstream, updated_packages,
+                         settings.TASKS_UPSTREAM_GROUPS,
+                         purge_versions=True)()
     update_counters(fast=False)
 
     """ Currently broken