# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
from temba.utils import chunk_list
from redis_cache import get_redis_connection
import time

HIGHPOINT_KEY = 'flowstep_backfill_highpoint'


def populate_flowsteps_for_broadcast(RelatedBroadcast, RelatedMsg, MsgManager, broadcast, batch):
    msg_ids = MsgManager.filter(broadcast=broadcast.id).values_list('id', flat=True)
    start_count = len(batch)

    for msg_id_batch in chunk_list(set(msg_ids), 1000):
        # get all the flowsteps that reference these msgs
        RelatedMsg.objects.filter(msg_id__in=msg_id_batch).values_list('flowstep_id', flat=True)

        broadcast_batch = [RelatedBroadcast(flowstep_id=mid, broadcast_id=broadcast.id) for mid in msg_id_batch]
        batch += broadcast_batch

    return len(batch) - start_count


def backfill_flowsteps(FlowStep, Broadcast, MsgManager):
    # we keep track of our completed broadcasts so we can pick up where we left off if interrupted
    r = get_redis_connection()
    highpoint = r.get(HIGHPOINT_KEY)
    if highpoint is None:
        highpoint = 0

    RelatedBroadcast = FlowStep.broadcasts.through
    RelatedMsg = FlowStep.messages.through

    broadcast_ids = Broadcast.objects.filter(id__gt=highpoint).order_by('id').values_list('id', flat=True)
    start = time.time()
    batch = []
    i = 0

    for broadcast_id_batch in chunk_list(broadcast_ids, 1000):
        broadcasts = Broadcast.objects.filter(id__in=broadcast_id_batch).order_by('id').only('id')
        for broadcast in broadcasts:
            i += 1

            # clear any current relations on this broadcast
            RelatedBroadcast.objects.filter(broadcast_id=broadcast.id).delete()

            populate_flowsteps_for_broadcast(RelatedBroadcast, RelatedMsg, MsgManager, broadcast, batch)
            if len(batch) > 1000:
                for broadcast_batch in chunk_list(batch, 1000):
                    RelatedBroadcast.objects.bulk_create(broadcast_batch)
                print "%d (%d of %d) in %d" % (broadcast.id, i, len(broadcast_ids)-1, int(time.time() - start))
                r.set(HIGHPOINT_KEY, broadcast.id)
                batch = []

    for broadcast_batch in chunk_list(batch, 1000):
        RelatedBroadcast.objects.bulk_create(broadcast_batch)

    # we finished, no need to track any more status
    r.delete(HIGHPOINT_KEY)


def migration_backfill_flowsteps(apps, schema):
    backfill_flowsteps(apps.get_model('flows', 'FLowStep'),
                        apps.get_model('msgs', 'Broadcast'),
                        apps.get_model('msgs', 'Msg').objects)


def noop(apps, schema):
    pass


class Migration(migrations.Migration):

    dependencies = [
        ('flows', '0054_flowstep_broadcasts'),
    ]

    operations = [
        migrations.RunPython(migration_backfill_flowsteps, noop)
    ]

@@ -521,66 +521,21 @@
     
-# get all the flowsteps that reference these msgs%0A        
+fs_ids = set(
 Rela
@@ -618,17 +618,17 @@
 at=True)
-%0A
+)
 %0A       
@@ -676,17 +676,19 @@
 step_id=
-m
+fs_
 id, broa
@@ -718,27 +718,23 @@
 for 
-m
+fs_
 id in 
-msg_id_batch
+fs_ids
 %5D%0A  
@@ -1527,16 +1527,174 @@
 dcasts:%0A
+            if i %25 1000 == 0:%0A                print %22Processed %25d / %25d (batch size %25d) in %25d%22 %25 (i, len(broadcast_ids), len(batch), int(time.time() - start))%0A
         
@@ -2127,120 +2127,8 @@
 ch)%0A
-                print %22%25d (%25d of %25d) in %25d%22 %25 (broadcast.id, i, len(broadcast_ids)-1, int(time.time() - start))%0A
     

fix data migration