#### encoding: cp1255 ####

import urllib2
import re
import logging
logger = logging.getLogger("open-knesset.parse_future_committee_meetings")

from knesset.mks.models import Member
from knesset.committees.models import Committee 
from django.core.management.base import BaseCommand

import csv
spamWriter = csv.writer(open('eggs.csv', 'wb'))

class Command(BaseCommand):

    args = ''
    help = 'Parses commitee members from the Knesset website'
        
    def parse_future_committee_meetings(self):
        retval = []
        
        url = 'http://knesset.gov.il/agenda/heb/CommitteesByDate.asp'

        data = urllib2.urlopen(url).read()
        
        committee_re = re.compile('<td class="Day" bgcolor="#990000" >\s+\xf1\xe3\xf8 \xe4\xe9\xe5\xed \xec.+, <span style=color:#c0c0c0>')
        committee_name = re.compile('<td class="Day" bgcolor="#990000" >\s+\xf1\xe3\xf8 \xe4\xe9\xe5\xed \xec(.+), <span style=color:#c0c0c0>')
        date_re = re.compile("<nobr>\((\d+)/(\d+)/(\d+)\)</nobr>")
        meeting_title_re = re.compile('TitleCommittee')
        meeting_agenda_re = re.compile('class="Agenda"')
        meeting_agenda_text_re = re.compile('<[Tt]d class=AgendaText>([^<]+)</[Tt]d>')
        
        datas = committee_re.split( data )[1:]

        committee_names = committee_name.findall( data )
        committee_names = [ name.decode('cp1255') for name in committee_names ]
        
        committee_data = zip( committee_names, datas )
        for name, data in committee_data:
            date = date_re.findall(data)[0]
            year, month, day = int(date[2]), int(date[1]), int(date[0])
            
            meeting_datas = meeting_title_re.split(data)[1:]
            
            for meeting_data in meeting_datas:
                meeting_agenda_data = meeting_agenda_re.split(meeting_data)[1] 
                titles = meeting_agenda_text_re.findall( meeting_agenda_data )
                titles = [ title.decode('cp1255').strip() for title in titles ]
                title = " ".join( titles )
    
                retval.append( [ name, year, month, day, title ] )
                spamWriter.writerow( [ name.encode('utf8'), year, month, day, title.encode('utf8') ] )
 
        return retval

    def update_future_committee_meetings_db(self,r):
        for row in r:
            ev, created = Event.objects.get_or_create( when = datetime.datetime( year=row[1], month=row[2], day=row[3] ),
                                                       what = u"%s: %s" % (row[0], row[3]) )
            if created:
                ev.save()
                logger.debug("created %s" % ev)
        
    def handle(self, *args, **options):
        r = self.parse_future_committee_meetings()
        logger.debug(r)
        self.update_future_committee_meetings_db(r)
    
@@ -65,234 +65,349 @@
 ing%0A
-logger = logging.getLogger(%22open-knesset.parse_future_committee_meetings%22)%0A%0Afrom knesset.mks.models import Member%0Afrom knesset.committees.models import Committee %0Afrom django.core.management.base import BaseCommand%0A%0Aimport csv
+import csv%0Aimport datetime%0Afrom django.core.management.base import BaseCommand%0Afrom django.contrib.contenttypes.models import ContentType%0A%0Afrom knesset.mks.models import Member%0Afrom knesset.committees.models import Committee %0Afrom knesset.events.models import Event%0A%0Alogger = logging.getLogger(%22open-knesset.parse_future_committee_meetings%22)
 %0Aspa
@@ -557,28 +557,84 @@
 ebsite'%0A    
-    
+committee_ct = ContentType.objects.get_for_model(Committee)%0A
 %0A    def par
@@ -2479,16 +2479,100 @@
 w in r:%0A
+            try:%0A                committee = Committee.objects.get(name=row%5B0%5D)%0A    
         
@@ -2744,95 +2744,777 @@
     
+    
 what = 
-u%22%25s:
+row%5B4%5D,%0A                                                           which_pk = committee.id,%0A                                                           which_type = self.committee_ct,%0A                                                           )%0A            except Committee.DoesNotExist:%0A                logger.debug(%22couldn't find committee 
  %25s%22 %25 
-(
 row%5B0%5D
-, row%5B3%5D) )%0A            if created:%0A                ev.save()
+)%0A                try:%0A                    ev, created = Event.objects.get_or_create(when = datetime.datetime( year=row%5B1%5D, month=row%5B2%5D, day=row%5B3%5D ),%0A                                                               what = row%5B4%5D,%0A                                                           )%0A                except Event.MultipleObjectsReturned:%0A                    created = False%0A            if created:
 %0A   
@@ -3730,16 +3730,12 @@
 tings_db(r)%0A
-    

parsing future meetings into events