euscanwww: Fixed tasks to work with processing.scan and processing.misc

Fixed tasks to work with processing.* changes.
Moved periodic tasks config to settings.py using CELERYBEAT_SCHEDULE

Signed-off-by: volpino <fox91@anche.no>

euscanwww/djeuscan/processing/scan/scan_metadata.py

@@ -1,7 +1,4 @@
-import os.path
-
 from gentoolkit.query import Query
-from gentoolkit.errors import GentoolkitFatalError
 
 from django.db.transaction import commit_on_success
 from django.core.management.color import color_style
@@ -10,6 +7,7 @@ from django.core.exceptions import ValidationError
 from djeuscan.models import Package, Herd, Maintainer
 from djeuscan.processing import FakeLogger
 
+
 class ScanMetadata(object):
     def __init__(self, logger=None):
         self.style = color_style()
@@ -61,8 +59,10 @@ class ScanMetadata(object):
         old_herds = set(existing_herds).difference(herds.keys())
 
         existing_maintainers = [m.email for m in obj.maintainers.all()]
-        new_maintainers = set(maintainers.keys()).difference(existing_maintainers)
-        old_maintainers = set(existing_maintainers).difference(maintainers.keys())
+        new_maintainers = set(maintainers.keys()).\
+                          difference(existing_maintainers)
+        old_maintainers = set(existing_maintainers).\
+                          difference(maintainers.keys())
 
         for herd in obj.herds.all():
             if herd.herd in old_herds:
@@ -125,6 +125,7 @@ class ScanMetadata(object):
             )
         return maintainer
 
+
 @commit_on_success
 def scan_metadata(packages=None, category=None, logger=None):
     scan_handler = ScanMetadata(logger=logger)
@@ -139,4 +140,3 @@ def scan_metadata(packages=None, category=None, logger=None):
             scan_handler.scan('%s/%s' % (pkg.category, pkg.name), pkg)
         else:
             scan_handler.scan(pkg)
-

euscanwww/djeuscan/processing/scan/scan_portage.py

@@ -83,7 +83,9 @@ class ScanPortage(object):
                 package['homepage'] = ' '.join(p.homepages)
                 package['description'] = p.description
                 package['versions'] = []
-            package['versions'].append((p._cpv, p.slot, p.repository or 'gentoo'))
+            package['versions'].append(
+                (p._cpv, p.slot, p.repository or 'gentoo')
+            )
 
         if package_name:
             yield package
@@ -109,7 +111,7 @@ class ScanPortage(object):
             self.logger.error(self.style.ERROR(msg))
             return
 
-        package = {'versions' : []}
+        package = {'versions': []}
         category = ""
 
         for event, elem in parser:
@@ -123,8 +125,11 @@ class ScanPortage(object):
                     package[elem.tag] = elem.text or ""
                 elif elem.tag == "version":
                     # append version data to versions
-                    cpv = "%s/%s-%s" % \
-                        (package["category"], package["package"], elem.attrib["id"])
+                    cpv = "%s/%s-%s" % (
+                        package["category"],
+                        package["package"],
+                        elem.attrib["id"]
+                    )
                     slot = elem.attrib.get("slot", "0")
                     overlay = elem.attrib.get("repository", "gentoo")
                     package["versions"].append((cpv, slot, overlay))
@@ -133,7 +138,7 @@ class ScanPortage(object):
                 if elem.tag == "package":
                     # clean old data
                     yield package
-                    package = {"versions" : []}
+                    package = {"versions": []}
 
                 if elem.tag == "category":
                     # clean old data
@@ -177,7 +182,9 @@ class ScanPortage(object):
         for data in self.scan_eix_xml(query, category):
         #for data in self.scan_gentoopm(query, category):
             cat, pkg = data['category'], data['package']
-            package = self.store_package(cat, pkg, data['homepage'], data['description'])
+            package = self.store_package(
+                cat, pkg, data['homepage'], data['description']
+            )
             packages_alive.add("%s/%s" % (cat, pkg))
             for cpv, slot, overlay in data['versions']:
                 self.store_version(package, cpv, slot, overlay)
@@ -300,8 +307,9 @@ class ScanPortage(object):
 
 
 @commit_on_success
-def scan_portage(packages=None, category=None, no_log=False, purge_packages=False,
-                 purge_versions=False, prefetch=False, logger=None):
+def scan_portage(packages=None, category=None, no_log=False,
+                 purge_packages=False, purge_versions=False, prefetch=False,
+                 logger=None):
 
     logger = logger or FakeLogger()
 
@@ -335,4 +343,3 @@ def scan_portage(packages=None, category=None, no_log=False, purge_packages=Fals
                 scan_handler.scan(pkg)
 
     logger.info('Done.')
-

euscanwww/djeuscan/processing/scan/scan_upstream.py

@@ -69,7 +69,9 @@ class ScanUpstream(object):
         # Set all versions dead, then set found versions alive and
         # delete old versions
         if self.purge_versions:
-            Version.objects.filter(package=obj, packaged=False).update(alive=False)
+            Version.objects.filter(
+                package=obj, packaged=False
+            ).update(alive=False)
 
         return obj
 
@@ -109,7 +111,6 @@ class ScanUpstream(object):
         package.n_versions += 1
         package.save()
 
-
     def purge_old_versions(self):
         if not self.purge_versions:
             return
@@ -132,6 +133,7 @@ class ScanUpstream(object):
 
         versions.delete()
 
+
 @commit_on_success
 def scan_upstream(packages=None, purge_versions=False,
                   logger=None):
@@ -144,8 +146,6 @@ def scan_upstream(packages=None, purge_versions=False,
     if not packages:
         packages = Package.objects.all()
 
-    result = True
-
     for pkg in packages:
         if isinstance(pkg, Package):
             scan_handler.scan('%s/%s' % (pkg.category, pkg.name))