diff --git a/bin/euscan b/bin/euscan index 441d479..4e2e10f 100755 --- a/bin/euscan +++ b/bin/euscan @@ -308,6 +308,9 @@ def main(): "for more informations" ) + if not (CONFIG['format'] or CONFIG['quiet']) and len(queries) > 1: + print("") + if CONFIG['progress']: on_progress_gen.next() print("\n", file=sys.stderr) diff --git a/euscanwww/djeuscan/helpers.py b/euscanwww/djeuscan/helpers.py index 2a5f878..c05daaf 100644 --- a/euscanwww/djeuscan/helpers.py +++ b/euscanwww/djeuscan/helpers.py @@ -105,3 +105,4 @@ def get_account_maintainers(user): ids = [obj.pk for obj in get_profile(user).maintainers.all()] return Package.objects.maintainers(ids=ids) + diff --git a/euscanwww/djeuscan/management/commands/scan_metadata.py b/euscanwww/djeuscan/management/commands/scan_metadata.py index 48f2c3c..7fb5b52 100644 --- a/euscanwww/djeuscan/management/commands/scan_metadata.py +++ b/euscanwww/djeuscan/management/commands/scan_metadata.py @@ -25,9 +25,9 @@ class Command(BaseCommand): default=None, help='Scan only this category'), make_option('--populate', - action='store', + action='store_true', dest='populate', - default=None, + default=False, help='Populate herds and maintainers from herds.xml'), ) args = '' diff --git a/euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py b/euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py index a5b3bed..d86eca9 100644 --- a/euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py +++ b/euscanwww/djeuscan/migrations/0015_initial_celery_periodictasks.py @@ -4,34 +4,38 @@ from south.v2 import DataMigration class Migration(DataMigration): + depends_on = ( + ("djcelery", "0001_initial"), + ) + def forwards(self, orm): - every_minute = orm["djcelery.IntervalSchedule"].objects.create( - every=1, period="minutes" + every_day = orm["djcelery.CrontabSchedule"].objects.create( + minute = "00", + hour = "01", + day_of_week = "*", + day_of_month = "*", + month_of_year = "*" ) - every_day = orm["djcelery.IntervalSchedule"].objects.create( - every=1, period="days" - ) - every_week = orm["djcelery.IntervalSchedule"].objects.create( - every=7, period="days" - ) - orm["djcelery.PeriodicTask"].objects.create( - name="Refresh package queries", - task="djeuscan.tasks.consume_refresh_package_request", - interval=every_minute + every_week = orm["djcelery.CrontabSchedule"].objects.create( + minute = "00", + hour = "03", + day_of_week = "1", + day_of_month = "*", + month_of_year = "*" ) orm["djcelery.PeriodicTask"].objects.create( name="Daily portage update", task="djeuscan.tasks.update_portage", - interval=every_day + crontab=every_day ) orm["djcelery.PeriodicTask"].objects.create( name="Weekly upstream update", task="djeuscan.tasks.update_upstream", - interval=every_week + crontab=every_week ) def backwards(self, orm): - orm["djcelery.IntervalSchedule"].objects.all().delete() + orm["djcelery.CrontabSchedule"].objects.all().delete() orm["djcelery.PeriodicTask"].objects.all().delete() diff --git a/euscanwww/djeuscan/processing/scan/scan_metadata.py b/euscanwww/djeuscan/processing/scan/scan_metadata.py index 54dd9c4..329c226 100644 --- a/euscanwww/djeuscan/processing/scan/scan_metadata.py +++ b/euscanwww/djeuscan/processing/scan/scan_metadata.py @@ -49,7 +49,13 @@ class ScanMetadata(object): if created: self.logger.info('+ [p] %s/%s' % (pkg.category, pkg.name)) - if not pkg.metadata: + try: + if not pkg.metadata: + return + except Exception as e: + self.logger.error( + self.style.ERROR('%s/%s: %s' % (pkg.category, pkg.name, str(e))) + ) return herds = dict( diff --git a/pym/euscan/__init__.py b/pym/euscan/__init__.py index 8efdce4..dad6742 100644 --- a/pym/euscan/__init__.py +++ b/pym/euscan/__init__.py @@ -70,6 +70,7 @@ ROBOTS_TXT_BLACKLIST_DOMAINS = [ '(.*)chromium.org(.*)', '(.*)nodejs.org(.*)', '(.*)download.mono-project.com(.*)', + '(.*)fedorahosted.org(.*)', ] from out import EuscanOutput diff --git a/pym/euscan/out.py b/pym/euscan/out.py index 8a505fc..65947ee 100644 --- a/pym/euscan/out.py +++ b/pym/euscan/out.py @@ -37,8 +37,10 @@ class ProgressHandler(object): def progress_bar(): on_progress = None - progress_bar = TermProgressBar(title="euscan") - progress_bar.file = sys.stderr + try: + progress_bar = TermProgressBar(fd=sys.stderr, title="euscan") + except TypeError: + progress_bar = TermProgressBar(title="euscan") progress_handler = ProgressHandler(progress_bar) on_progress = progress_handler.on_progress diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py index 5d9fd06..86574ae 100644 --- a/pym/euscan/scan.py +++ b/pym/euscan/scan.py @@ -169,8 +169,7 @@ def scan_upstream(query, on_progress=None): is_current_version_stable = is_version_stable(ver) if len(result) > 0: if not (CONFIG['format'] or CONFIG['quiet']): - print("\n", file=sys.stderr) - + print("") for cp, url, version, handler, confidence in result: if CONFIG["ignore-pre-release"]: if not is_version_stable(version): @@ -179,6 +178,8 @@ def scan_upstream(query, on_progress=None): if is_current_version_stable and \ not is_version_stable(version): continue + if CONFIG['progress']: + print ("", file=sys.stderr) output.result(cp, version, url, handler, confidence) return result