Merge branch 'master' of https://github.com/iksaif/euscan

bin/euscan

@@ -308,6 +308,9 @@ def main():
                 "for more informations"
             )
 
+        if not (CONFIG['format'] or CONFIG['quiet']) and len(queries) > 1:
+            print("")
+
     if CONFIG['progress']:
         on_progress_gen.next()
         print("\n", file=sys.stderr)

euscanwww/djeuscan/helpers.py

@@ -105,3 +105,4 @@ def get_account_maintainers(user):
 
     ids = [obj.pk for obj in get_profile(user).maintainers.all()]
     return Package.objects.maintainers(ids=ids)
+

euscanwww/djeuscan/management/commands/scan_metadata.py

@@ -25,9 +25,9 @@ class Command(BaseCommand):
             default=None,
             help='Scan only this category'),
         make_option('--populate',
-            action='store',
+            action='store_true',
             dest='populate',
-            default=None,
+            default=False,
             help='Populate herds and maintainers from herds.xml'),
         )
     args = '<package package ...>'

euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py

@@ -4,34 +4,38 @@ from south.v2 import DataMigration
 
 class Migration(DataMigration):
 
+    depends_on = (
+        ("djcelery", "0001_initial"),
+    )
+
     def forwards(self, orm):
-        every_minute = orm["djcelery.IntervalSchedule"].objects.create(
+        every_day = orm["djcelery.CrontabSchedule"].objects.create(
-            every=1, period="minutes"
+            minute = "00",
+            hour = "01",
+            day_of_week = "*",
+            day_of_month = "*",
+            month_of_year = "*"
         )
-        every_day = orm["djcelery.IntervalSchedule"].objects.create(
+        every_week = orm["djcelery.CrontabSchedule"].objects.create(
-            every=1, period="days"
+            minute = "00",
-        )
+            hour = "03",
-        every_week = orm["djcelery.IntervalSchedule"].objects.create(
+            day_of_week = "1",
-            every=7, period="days"
+            day_of_month = "*",
-        )
+            month_of_year = "*"
-        orm["djcelery.PeriodicTask"].objects.create(
-            name="Refresh package queries",
-            task="djeuscan.tasks.consume_refresh_package_request",
-            interval=every_minute
         )
         orm["djcelery.PeriodicTask"].objects.create(
             name="Daily portage update",
             task="djeuscan.tasks.update_portage",
-            interval=every_day
+            crontab=every_day
         )
         orm["djcelery.PeriodicTask"].objects.create(
             name="Weekly upstream update",
             task="djeuscan.tasks.update_upstream",
-            interval=every_week
+            crontab=every_week
         )
 
     def backwards(self, orm):
-        orm["djcelery.IntervalSchedule"].objects.all().delete()
+        orm["djcelery.CrontabSchedule"].objects.all().delete()
         orm["djcelery.PeriodicTask"].objects.all().delete()
 
 

euscanwww/djeuscan/processing/scan/scan_metadata.py

@@ -49,8 +49,14 @@ class ScanMetadata(object):
         if created:
             self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name))
 
+        try:
             if not pkg.metadata:
                 return
+        except Exception as e:
+            self.logger.error(
+                self.style.ERROR('%s/%s: %s' % (pkg.category, pkg.name, str(e)))
+            )
+            return
 
         herds = dict(
             [(herd[0], herd) for herd in pkg.metadata.herds(True)]

pym/euscan/__init__.py

@@ -70,6 +70,7 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [
     '(.*)chromium.org(.*)',
     '(.*)nodejs.org(.*)',
     '(.*)download.mono-project.com(.*)',
+    '(.*)fedorahosted.org(.*)',
 ]
 
 from out import EuscanOutput

pym/euscan/out.py

@@ -37,8 +37,10 @@ class ProgressHandler(object):
 
 def progress_bar():
     on_progress = None
+    try:
+        progress_bar = TermProgressBar(fd=sys.stderr, title="euscan")
+    except TypeError:
         progress_bar = TermProgressBar(title="euscan")
-    progress_bar.file = sys.stderr
 
     progress_handler = ProgressHandler(progress_bar)
     on_progress = progress_handler.on_progress

pym/euscan/scan.py

@@ -169,8 +169,7 @@ def scan_upstream(query, on_progress=None):
     is_current_version_stable = is_version_stable(ver)
     if len(result) > 0:
         if not (CONFIG['format'] or CONFIG['quiet']):
-            print("\n", file=sys.stderr)
+            print("")
-
         for cp, url, version, handler, confidence in result:
             if CONFIG["ignore-pre-release"]:
                 if not is_version_stable(version):
@@ -179,6 +178,8 @@ def scan_upstream(query, on_progress=None):
                 if is_current_version_stable and \
                    not is_version_stable(version):
                     continue
+            if CONFIG['progress']:
+               print ("", file=sys.stderr)
             output.result(cp, version, url, handler, confidence)
 
     return result