from __future__ import absolute_import
from datetime import datetime

from celery.task import periodic_task
from celery.schedules import crontab

from corehq.util.datadog.gauges import datadog_counter
from corehq.blobs.models import BlobExpiration
from corehq.blobs import get_blob_db


@periodic_task(run_every=crontab(minute=0, hour='0,12'))
def delete_expired_blobs():
    blob_expirations = BlobExpiration.objects.filter(expires_on__lt=_utcnow(), deleted=False)

    db = get_blob_db()
    paths = []
    bytes_deleted = 0
    for blob_expiration in blob_expirations:
        paths.append(db.get_path(blob_expiration.identifier, blob_expiration.bucket))
        bytes_deleted += blob_expiration.length

    db.bulk_delete(paths)
    blob_expirations.update(deleted=True)
    datadog_counter(
        'commcare.temp_blobs.bytes_deleted',
        value=bytes_deleted,
    )
    return bytes_deleted


def _utcnow():
    return datetime.utcnow()

@@ -491,19 +491,117 @@
     
-paths = %5B%5D%0A
+total_bytes_deleted = 0%0A    while blob_expirations.exists():%0A        paths = %5B%5D%0A        deleted_ids = %5B%5D%0A    
     
@@ -614,24 +614,28 @@
 deleted = 0%0A
+    
     for blob
@@ -665,18 +665,29 @@
 irations
-:%0A
+%5B:1000%5D:%0A    
         
@@ -764,16 +764,71 @@
 ucket))%0A
+            deleted_ids.append(blob_expiration.id)%0A    
         
@@ -868,16 +868,20 @@
 length%0A%0A
+    
     db.b
@@ -898,30 +898,33 @@
 (paths)%0A    
-blob_e
+    BlobE
 xpirations.u
@@ -920,17 +920,51 @@
 piration
-s
+.objects.filter(id__in=deleted_ids)
 .update(
@@ -981,16 +981,20 @@
 ue)%0A    
+    
 datadog_
@@ -1010,16 +1010,20 @@
         
+    
 'commcar
@@ -1059,16 +1059,20 @@
         
+    
 value=by
@@ -1088,14 +1088,64 @@
 ed,%0A
+    
     )%0A
+        total_bytes_deleted += bytes_deleted%0A%0A
     
@@ -1151,16 +1151,22 @@
  return 
+total_
 bytes_de

Delete expired blobs in batches