import logging
from django.db import transaction
from .models import Document
from . import es
from .utils import now, open_url

logger = logging.getLogger(__name__)


def index(doc):
    logger.info('indexing %s', doc)

    with open_url(doc.url) as f:
        text = f.read()

    es.index(doc.slug, {
        'text': text,
        'title': doc.title,
        'url': doc.url,
    })

    doc.indexed = True
    doc.index_time = now()
    doc.save()


def work_loop():
    while True:
        with transaction.atomic():
            doc = (
                Document.objects
                .select_for_update()
                .filter(indexed=False)
                .order_by('?')
                .first()
            )

            if doc is None:
                return

            index(doc)

@@ -42,16 +42,32 @@
 saction%0A
+import requests%0A
 from .mo
@@ -130,18 +130,8 @@
  now
-, open_url
 %0A%0Alo
@@ -163,24 +163,68 @@
 __name__)%0A%0A%0A
+class TextMissing(RuntimeError):%0A    pass%0A%0A%0A
 def index(do
@@ -272,59 +272,215 @@
     
-with open_url(doc.url) as f:%0A        text = f.read(
+resp = requests.get(doc.text_url)%0A    if resp.status_code == 404:%0A        raise TextMissing(str(doc))%0A%0A    if resp.status_code != 200:%0A        raise RuntimeError(%22failed to get text for %25s: %25r%22 %25 (doc, resp)
 )%0A%0A 
@@ -519,16 +519,21 @@
 'text': 
+resp.
 text,%0A  
@@ -989,19 +989,137 @@
         
-index(
+try:%0A                index(doc)%0A            except TextMissing:%0A                logger.warn('text missing for document %25s', 
 doc)%0A

index from text_url