Login

Prefetch generic relations

Author:
jpic
Posted:
July 19, 2011
Language:
Python
Version:
1.3
Score:
2 (after 2 ratings)

See the function docstring.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic

def prefetch_relations(weak_queryset):
    """
    Consider such a model class::

        class Action(models.Model):
            actor_content_type = models.ForeignKey(ContentType,related_name='actor')
            actor_object_id = models.PositiveIntegerField() 
            actor = generic.GenericForeignKey('actor_content_type','actor_object_id')
 
    And dataset::
        
        Action(actor=user1).save()
        Action(actor=user2).save()
    
    This will hit the user table once for each action::

        [a.actor for a in Action.objects.all()]

    Whereas this will hit the user table once::

        [a.actor for a in prefetch_relations(Action.objects.all())]

    Actually, the example above will hit the database N+1 times,  where N is
    the number of actions. But with prefetch_relations(), the database will be
    hit N+1 times where N is the number of distinct content types.

    Note that prefetch_relations() is recursive.

    Here an example, making a list with prefetch_relations(), and then without prefetch_relations(). See the number of database hits after each test.
    
        In [1]: from django import db; from prefetch_relations import prefetch_relations
        
        In [2]: db.reset_queries()
        
        In [3]: x = [(a.actor, a.action_object, a.target) for a in prefetch_relations(Action.objects.all().order_by('-pk'))]
        
        In [4]: print len(db.connection.queries)
        34
        
        In [5]: db.reset_queries()
        
        In [6]: print len(db.connection.queries)
        0
                
        In [7]: x = [(a.actor, a.action_object, a.target) for a in Action.objects.all().order_by('-pk')]
        
        In [8]: print len(db.connection.queries)
        396
    """
    weak_queryset = weak_queryset.select_related()

    # reverse model's generic foreign keys into a dict:
    # { 'field_name': generic.GenericForeignKey instance, ... }
    gfks = {}
    for name, gfk in weak_queryset.model.__dict__.items():
        if not isinstance(gfk, generic.GenericForeignKey):
            continue
        gfks[name] = gfk

    data = {}
    for weak_model in weak_queryset:
        for gfk_name, gfk_field in gfks.items():
            related_content_type_id = getattr(weak_model, gfk_field.model._meta.get_field_by_name(gfk_field.ct_field)[0].get_attname())
            if not related_content_type_id:
                continue
            related_content_type = ContentType.objects.get_for_id(related_content_type_id)
            related_object_id = int(getattr(weak_model, gfk_field.fk_field))

            if related_content_type not in data.keys():
                data[related_content_type] = []
            data[related_content_type].append(related_object_id)

    for content_type, object_ids in data.items():
        model_class = content_type.model_class()
        models = prefetch_relations(model_class.objects.filter(pk__in=object_ids))
        for model in models:
            for weak_model in weak_queryset:
                for gfk_name, gfk_field in gfks.items():
                    related_content_type_id = getattr(weak_model, gfk_field.model._meta.get_field_by_name(gfk_field.ct_field)[0].get_attname())
                    if not related_content_type_id:
                        continue
                    related_content_type = ContentType.objects.get_for_id(related_content_type_id)
                    related_object_id = int(getattr(weak_model, gfk_field.fk_field))
                    
                    if related_object_id != model.pk:
                        continue
                    if related_content_type != content_type:
                        continue

                    setattr(weak_model, gfk_name, model)

    return weak_queryset

More like this

  1. Template tag - list punctuation for a list of items by shapiromatron 2 months, 2 weeks ago
  2. JSONRequestMiddleware adds a .json() method to your HttpRequests by cdcarter 2 months, 3 weeks ago
  3. Serializer factory with Django Rest Framework by julio 9 months, 3 weeks ago
  4. Image compression before saving the new model / work with JPG, PNG by Schleidens 10 months, 1 week ago
  5. Help text hyperlinks by sa2812 11 months ago

Comments

Please login first before commenting.