23 lines
789 B
Python
23 lines
789 B
Python
|
|
def queryset_iterator(queryset, chunksize=1000):
|
|
'''''
|
|
Iterate over a Django Queryset ordered by the primary key
|
|
|
|
This method loads a maximum of chunksize (default: 1000) rows in it's
|
|
memory at the same time while django normally would load all rows in it's
|
|
memory. Using the iterator() method only causes it to not preload all the
|
|
classes.
|
|
|
|
Note that the implementation of the iterator does not support ordered query sets.
|
|
'''
|
|
import gc
|
|
|
|
pk = 0
|
|
last_pk = queryset.order_by('-pk')[0].pk
|
|
queryset = queryset.order_by('pk')
|
|
while pk < last_pk:
|
|
for row in queryset.filter(pk__gt=pk)[:chunksize]:
|
|
pk = row.pk
|
|
yield row
|
|
gc.collect()
|