euscanwww: PEP8, Cleaning unused imports and removing useless stuff
* Removed useless use of chords * populate in scan_metadata is off by default * fixed group_one and scan_metadata launch * removed useless utils.py Signed-off-by: volpino <fox91@anche.no>
This commit is contained in:
parent
3d8ea27ecc
commit
7dc322348d
@ -24,6 +24,11 @@ class Command(BaseCommand):
|
|||||||
dest='category',
|
dest='category',
|
||||||
default=None,
|
default=None,
|
||||||
help='Scan only this category'),
|
help='Scan only this category'),
|
||||||
|
make_option('--populate',
|
||||||
|
action='store',
|
||||||
|
dest='populate',
|
||||||
|
default=None,
|
||||||
|
help='Populate herds and maintainers from herds.xml'),
|
||||||
)
|
)
|
||||||
args = '<package package ...>'
|
args = '<package package ...>'
|
||||||
help = 'Scans metadata and fills database'
|
help = 'Scans metadata and fills database'
|
||||||
@ -42,5 +47,6 @@ class Command(BaseCommand):
|
|||||||
scan_metadata(
|
scan_metadata(
|
||||||
packages=packages,
|
packages=packages,
|
||||||
category=options['category'],
|
category=options['category'],
|
||||||
logger=logger
|
logger=logger,
|
||||||
|
populate=options['populate'],
|
||||||
)
|
)
|
||||||
|
@ -164,7 +164,7 @@ class ScanMetadata(object):
|
|||||||
|
|
||||||
|
|
||||||
@commit_on_success
|
@commit_on_success
|
||||||
def scan_metadata(packages=None, category=None, logger=None, populate=True):
|
def scan_metadata(packages=None, category=None, logger=None, populate=False):
|
||||||
scan_handler = ScanMetadata(logger=logger)
|
scan_handler = ScanMetadata(logger=logger)
|
||||||
|
|
||||||
if category:
|
if category:
|
||||||
|
@ -15,6 +15,7 @@ from djeuscan.models import Package, Version, VersionLog
|
|||||||
|
|
||||||
PORTDB = None
|
PORTDB = None
|
||||||
|
|
||||||
|
|
||||||
class ScanPortage(object):
|
class ScanPortage(object):
|
||||||
def __init__(self, logger=None, no_log=False, purge_packages=False,
|
def __init__(self, logger=None, no_log=False, purge_packages=False,
|
||||||
purge_versions=False):
|
purge_versions=False):
|
||||||
@ -23,8 +24,8 @@ class ScanPortage(object):
|
|||||||
self.purge_packages = purge_packages
|
self.purge_packages = purge_packages
|
||||||
self.purge_versions = purge_versions
|
self.purge_versions = purge_versions
|
||||||
|
|
||||||
if not PORTDB: # Lazy loading for portdb
|
global PORTDB
|
||||||
global PORTDB
|
if not PORTDB: # Lazy loading for portdb
|
||||||
PORTDB = portage.db[portage.root]["porttree"].dbapi
|
PORTDB = portage.db[portage.root]["porttree"].dbapi
|
||||||
|
|
||||||
self.style = color_style()
|
self.style = color_style()
|
||||||
|
@ -2,15 +2,15 @@
|
|||||||
Celery tasks for djeuscan
|
Celery tasks for djeuscan
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from itertools import islice
|
from celery.task import task, group
|
||||||
|
|
||||||
from celery.task import task, group, chord
|
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
|
|
||||||
|
import portage
|
||||||
|
|
||||||
from djeuscan.models import Package, RefreshPackageQuery
|
from djeuscan.models import Package, RefreshPackageQuery
|
||||||
from djeuscan.processing import scan, misc
|
from djeuscan.processing import scan, misc
|
||||||
from djeuscan.utils import queryset_iterator
|
|
||||||
|
|
||||||
class TaskFailedException(Exception):
|
class TaskFailedException(Exception):
|
||||||
"""
|
"""
|
||||||
@ -18,24 +18,35 @@ class TaskFailedException(Exception):
|
|||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def group_one(task, seq, *args, **kwargs):
|
def group_one(task, seq, *args, **kwargs):
|
||||||
"""
|
"""
|
||||||
Create a group of tasks, each task handle one element of seq
|
Create a group of tasks, each task handle one element of seq
|
||||||
"""
|
"""
|
||||||
tasks = []
|
tasks = []
|
||||||
for i in seq:
|
|
||||||
tasks.append(task.subtask(args=[seq[i]] + list(args), kwargs=kwargs))
|
for elem in seq:
|
||||||
|
if "attr_name" in kwargs:
|
||||||
|
kwargs[kwargs["attr_name"]] = elem
|
||||||
|
del kwargs["attr_name"]
|
||||||
|
tasks.append(task.subtask(args=args, kwargs=kwargs))
|
||||||
|
else:
|
||||||
|
tasks.append(task.subtask(args=[elem] + list(args), kwargs=kwargs))
|
||||||
return group(tasks)
|
return group(tasks)
|
||||||
|
|
||||||
|
|
||||||
def group_chunks(task, seq, n, *args, **kwargs):
|
def group_chunks(task, seq, n, *args, **kwargs):
|
||||||
"""
|
"""
|
||||||
Creates a group of tasks, each subtask has <n> elements to handle
|
Creates a group of tasks, each subtask has <n> elements to handle
|
||||||
"""
|
"""
|
||||||
tasks = []
|
tasks = []
|
||||||
for i in xrange(0, len(seq), n):
|
for i in xrange(0, len(seq), n):
|
||||||
tasks.append(task.subtask(args=[seq[i:i+n]] + list(args), kwargs=kwargs))
|
tasks.append(
|
||||||
|
task.subtask(args=[seq[i:i + n]] + list(args), kwargs=kwargs)
|
||||||
|
)
|
||||||
return group(tasks)
|
return group(tasks)
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def regen_rrds():
|
def regen_rrds():
|
||||||
"""
|
"""
|
||||||
@ -44,6 +55,7 @@ def regen_rrds():
|
|||||||
misc.regen_rrds()
|
misc.regen_rrds()
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def update_counters(fast=False):
|
def update_counters(fast=False):
|
||||||
"""
|
"""
|
||||||
@ -55,8 +67,9 @@ def update_counters(fast=False):
|
|||||||
logger.info("Done")
|
logger.info("Done")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def scan_metadata(packages=[], category=None):
|
def scan_metadata(packages=[], category=None, populate=False):
|
||||||
"""
|
"""
|
||||||
Scans metadata for the given set of packages
|
Scans metadata for the given set of packages
|
||||||
"""
|
"""
|
||||||
@ -75,9 +88,11 @@ def scan_metadata(packages=[], category=None):
|
|||||||
packages=packages,
|
packages=packages,
|
||||||
category=category,
|
category=category,
|
||||||
logger=logger,
|
logger=logger,
|
||||||
|
populate=populate,
|
||||||
)
|
)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def scan_portage(packages=[], category=None,
|
def scan_portage(packages=[], category=None,
|
||||||
no_log=False, purge_packages=False,
|
no_log=False, purge_packages=False,
|
||||||
@ -107,6 +122,7 @@ def scan_portage(packages=[], category=None,
|
|||||||
)
|
)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def scan_upstream(packages=[], purge_versions=False):
|
def scan_upstream(packages=[], purge_versions=False):
|
||||||
"""
|
"""
|
||||||
@ -128,6 +144,7 @@ def scan_upstream(packages=[], purge_versions=False):
|
|||||||
)
|
)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def update_portage_trees():
|
def update_portage_trees():
|
||||||
"""
|
"""
|
||||||
@ -137,17 +154,19 @@ def update_portage_trees():
|
|||||||
misc.update_portage_trees(logger=logger)
|
misc.update_portage_trees(logger=logger)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def update_portage(packages=None):
|
def update_portage(packages=None):
|
||||||
|
update_portage_trees()
|
||||||
|
scan_portage(purge_packages=True, purge_versions=True, prefetch=True)
|
||||||
(
|
(
|
||||||
update_portage_trees.s() |
|
group_one(scan_metadata, portage.settings.categories,
|
||||||
scan_portage.si(purge_packages=True, purge_versions=True, prefetch=True) |
|
attr_name="category") |
|
||||||
#scan_metadata.si() |
|
|
||||||
group_one(scan_metadata, portage.settings.categories) |
|
|
||||||
update_counters.si(fast=False)
|
update_counters.si(fast=False)
|
||||||
)()
|
)()
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def update_upstream():
|
def update_upstream():
|
||||||
if settings.TASKS_UPSTREAM_GROUPS >= 1:
|
if settings.TASKS_UPSTREAM_GROUPS >= 1:
|
||||||
@ -165,6 +184,7 @@ def update_upstream():
|
|||||||
)()
|
)()
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def scan_package(package):
|
def scan_package(package):
|
||||||
scan_portage([package], purge_packages=True, purge_versions=True)
|
scan_portage([package], purge_packages=True, purge_versions=True)
|
||||||
@ -172,11 +192,13 @@ def scan_package(package):
|
|||||||
scan_upstream([package])
|
scan_upstream([package])
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@task(rate_limit="1/m")
|
@task(rate_limit="1/m")
|
||||||
def scan_package_user(package):
|
def scan_package_user(package):
|
||||||
scan_package(package)
|
scan_package(package)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def consume_refresh_package_request():
|
def consume_refresh_package_request():
|
||||||
"""
|
"""
|
||||||
@ -191,6 +213,7 @@ def consume_refresh_package_request():
|
|||||||
query.delete()
|
query.delete()
|
||||||
scan_package_user.delay(pkg)
|
scan_package_user.delay(pkg)
|
||||||
|
|
||||||
|
|
||||||
admin_tasks = [
|
admin_tasks = [
|
||||||
regen_rrds,
|
regen_rrds,
|
||||||
update_counters,
|
update_counters,
|
||||||
@ -203,7 +226,8 @@ admin_tasks = [
|
|||||||
scan_package,
|
scan_package,
|
||||||
]
|
]
|
||||||
|
|
||||||
""" Chunk helpers (chunks can't use keyword arguments) """
|
|
||||||
|
# Chunk helpers (chunks can't use keyword arguments)
|
||||||
@task
|
@task
|
||||||
def scan_metadata_category(category):
|
def scan_metadata_category(category):
|
||||||
"""
|
"""
|
||||||
@ -212,6 +236,7 @@ def scan_metadata_category(category):
|
|||||||
scan_metadata(category=category)
|
scan_metadata(category=category)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@task
|
@task
|
||||||
def scan_upstream_purge(*packages):
|
def scan_upstream_purge(*packages):
|
||||||
"""
|
"""
|
||||||
|
@ -1,22 +0,0 @@
|
|||||||
|
|
||||||
def queryset_iterator(queryset, chunksize=1000):
|
|
||||||
'''''
|
|
||||||
Iterate over a Django Queryset ordered by the primary key
|
|
||||||
|
|
||||||
This method loads a maximum of chunksize (default: 1000) rows in it's
|
|
||||||
memory at the same time while django normally would load all rows in it's
|
|
||||||
memory. Using the iterator() method only causes it to not preload all the
|
|
||||||
classes.
|
|
||||||
|
|
||||||
Note that the implementation of the iterator does not support ordered query sets.
|
|
||||||
'''
|
|
||||||
import gc
|
|
||||||
|
|
||||||
pk = 0
|
|
||||||
last_pk = queryset.order_by('-pk')[0].pk
|
|
||||||
queryset = queryset.order_by('pk')
|
|
||||||
while pk < last_pk:
|
|
||||||
for row in queryset.filter(pk__gt=pk)[:chunksize]:
|
|
||||||
pk = row.pk
|
|
||||||
yield row
|
|
||||||
gc.collect()
|
|
Loading…
Reference in New Issue
Block a user