#! /usr/bin/env python

# Copyright (c) 2010, Nate Stedman <natesm@gmail.com>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

# this is the fetch script for observatory.
# it should be run in cron or manually.

import sys
import os
from threading import Thread
from Queue import Queue

# set django's required paths
path = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, path)
sys.path.insert(0, os.path.abspath(os.path.join(path, '..')))
sys.path.insert(0, os.path.abspath(os.path.join(path, '..', '..')))
os.environ['DJANGO_SETTINGS_MODULE'] = 'observatory.settings'

# now we can import from observatory's stuff
from dashboard.models import Project
from observatory.settings import FETCH_THREAD_COUNT

def fetcher():
  while True:
    project = queue.get()
    project.fetch()
    queue.task_done()

# build a queue
queue = Queue()
for project in Project.objects.all():
  queue.put(project)

# run the threads
for i in range(FETCH_THREAD_COUNT):
  thread = Thread(target = fetcher)
  thread.daemon = True
  thread.start()

# wait until we're finished
queue.join()

# rank the projects
rank = 1
for project in Project.objects.order_by('score'):
  project.rank = rank
  project.save()
  rank += 1

@@ -1414,16 +1414,27 @@
 e True:%0A
+    try:%0A  
     proj
@@ -1451,24 +1451,26 @@
 e.get()%0A    
+ 
+ 
 project.fetc
@@ -1473,16 +1473,18 @@
 fetch()%0A
+  
     queu
@@ -1496,16 +1496,64 @@
 k_done()
+%0A    except:%0A      queue.task_done()%0A      raise
 %0A%0A# buil

Fix fetch.py indefinite hang on exceptions