From 6f9e575bbc5d57a228367f72b739395d8a674b60 Mon Sep 17 00:00:00 2001 From: Corentin Chary Date: Fri, 20 Jul 2012 08:27:24 +0200 Subject: [PATCH] euscanwww: add missing file Signed-off-by: Corentin Chary --- euscanwww/djeuscan/utils.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 euscanwww/djeuscan/utils.py diff --git a/euscanwww/djeuscan/utils.py b/euscanwww/djeuscan/utils.py new file mode 100644 index 0000000..19fa00d --- /dev/null +++ b/euscanwww/djeuscan/utils.py @@ -0,0 +1,22 @@ + +def queryset_iterator(queryset, chunksize=1000): + ''''' + Iterate over a Django Queryset ordered by the primary key + + This method loads a maximum of chunksize (default: 1000) rows in it's + memory at the same time while django normally would load all rows in it's + memory. Using the iterator() method only causes it to not preload all the + classes. + + Note that the implementation of the iterator does not support ordered query sets. + ''' + import gc + + pk = 0 + last_pk = queryset.order_by('-pk')[0].pk + queryset = queryset.order_by('pk') + while pk < last_pk: + for row in queryset.filter(pk__gt=pk)[:chunksize]: + pk = row.pk + yield row + gc.collect()