import sys
from BeautifulSoup import BeautifulSoup, Tag
from utils import fetch_content_from_url, make_soup_from_html_content, remove_text_formatting_markup







def extract_frontpage_title_and_url(link):
    title = ''.join([remove_text_formatting_markup(c.strip()) for c in link.contents])
    return title, link.get('href')



def extract_headlines_from_module(module_container):
    body = module_container.find('div', {'class':'ModuleElementBody'})
    
    headline_lists = body.findAll('div', {'class':'tot_widget-ul'})
    print len(headline_lists)



def extract_headlines_from_modules(maincontent):
    module_zone = maincontent.findAll('div', {'id':'hp-zone-list'})

    all_modules = module_zone.findAll('div', {'class':'ModuleElement'})



def extract_small_articles(maincontent):
    small_articles_container = maincontent.find('div', {'id':'SmallArticles'})

    small_articles = small_articles_container.findAll('div', {'class':'SmallArticles-Items'})
    small_articles.extend(small_articles_container.findAll('div', {'class':'SmallArticles-Items SmallArticles-Lasts'}))

    return [extract_frontpage_title_and_url(item.h5.a) for item in small_articles ]


def extract_first_articles(maincontent):
    first_article = maincontent.find('div', {'id':'FirstArticlesLeft'})
    titles_and_urls = list()

    titles_and_urls.append(extract_frontpage_title_and_url(first_article.h1.a))

    articles_right = maincontent.findAll('div', {'class':'ArticlesRight '})
    articles_right.extend(maincontent.findAll('div', {'class':'ArticlesRight ArticlesRight-First'}))
    articles_right.extend(maincontent.findAll('div', {'class':'ArticlesRight ArticlesRight-Last'}))
    
    titles_and_urls.extend(extract_frontpage_title_and_url(article.h5.a) for article in articles_right)

    return titles_and_urls



def make_full_url((title, url)):
    if url.startswith('http://'):
        return title, url
    else:
        return title, 'http://www.rtl.be{0}'.format(url)


def get_frontpage_toc():
    url = 'http://www.rtl.be/info/'
    html_content = fetch_content_from_url(url)
    soup = make_soup_from_html_content(html_content)

    maincontent = soup.find('div', {'class':'mainContent'})

    first_articles = extract_first_articles(maincontent)
    small_articles = extract_small_articles(maincontent)

    all_articles = first_articles + small_articles

    return [make_full_url(title_and_url) for title_and_url in all_articles]


if __name__=='__main__':
    toc = get_frontpage_toc()
    for title, url in toc:
        print title, url
@@ -1,60 +1,4 @@
-import sys%0Afrom BeautifulSoup import BeautifulSoup, Tag%0A
 from
@@ -97,20 +97,16 @@
 rkup%0A%0A%0A%0A
-%0A%0A%0A%0A
 def extr
@@ -318,16 +318,37 @@
 ainer):%0A
+    %22%22%22%0A    %0A    %22%22%22%0A
     body
@@ -410,21 +410,34 @@
 Body'%7D)%0A
+%0A
     
-%0A
+if body:%0A    
     head
@@ -500,38 +500,297 @@
 '%7D)%0A
+%0A
     
-print len(headline_lists)%0A
+    titles_and_urls = list()%0A        for l in headline_lists:%0A            for tag_name in ('h3', 'h5', 'li'):%0A                titles_and_urls.extend(%5Bextract_frontpage_title_and_url(h.a) for h in l.findAll(tag_name) if h.a%5D)%0A%0A        return titles_and_urls%0A    else:%0A        return %5B%5D
 %0A%0A%0Ad
@@ -863,35 +863,32 @@
 maincontent.find
-All
 ('div', %7B'id':'h
@@ -976,16 +976,200 @@
 ent'%7D)%0A%0A
+    headlines = list()%0A    for module in all_modules:%0A        module_headlines = extract_headlines_from_module(module)%0A        headlines.extend(module_headlines)%0A%0A    return headlines%0A
 %0A%0Adef ex
@@ -2387,16 +2387,270 @@
 (url)%0A%0A%0A
+def separate_news_and_blogposts(titles_and_urls):%0A    all_items = set(titles_and_urls)%0A    blogposts = set(%5B(title, url) for title, url in all_items if not url.startswith('/')%5D)%0A    news_items = all_items - blogposts%0A%0A    return news_items, blogposts%0A%0A%0A%0A
 def get_
@@ -2982,59 +2982,216 @@
 nt)%0A
-%0A    all_articles = first_articles + small_articles
+    modules_articles = extract_headlines_from_modules(maincontent)%0A%0A    all_articles = first_articles + small_articles + modules_articles%0A%0A    news_items, blogposts = separate_news_and_blogposts(all_articles)
 %0A%0A  
@@ -3254,21 +3254,30 @@
  in 
-all_articles%5D
+news_items%5D, blogposts
 %0A%0A%0Ai
@@ -3307,16 +3307,23 @@
 %0A    toc
+, blogs
  = get_f
@@ -3336,16 +3336,36 @@
 ge_toc()
+%0A%0A    print len(toc)
 %0A    for
@@ -3400,16 +3400,18 @@
 print title, url
+%0A%0A

extract headlines in "modules". Separate blogposts and actual news tems