from splinter import Browser
from bs4 import BeautifulSoup
import time

url = "http://egazette.nic.in"
browser = Browser()
browser.visit(url)
browser.click_link_by_text("Digital Directory")

assert browser.is_element_present_by_id("ContentPlaceHolder1_ddlcat1", wait=10)
elem = browser.find_by_id("ContentPlaceHolder1_ddlcat1").first
elem.select("Extra Ordinary")

time.sleep(1)
elem = browser.find_by_id("ContentPlaceHolder1_ddlyear1").first
elem.select("2014")

time.sleep(2)
search_button = browser.find_by_id("ContentPlaceHolder1_Button1").first
search_button.click()

# TODO: will have to add wait between all of these actions

soup = BeautifulSoup(browser.html, 'html.parser')

# TODO: Wait till "scroll to top" appears

# ContentPlaceHolder1_rptmmain_imgbtndownload_10

# ContentPlaceHolder1_rptmmain_imgbtndownload_13

# x = browser.find_by_xpath("//input[@src='images/pdf_icon.png']")
# I'm able to get links to all pdfs using this but I also want the metadata.

assert browser.is_text_present("Scroll to Top", wait=10)
metadata = list()
rows = list(browser.find_by_tag('tr'))
for row in rows[:20]:
    if 'input' not in row.html:
        continue

    row_data = [z.text for z in list(row.find_by_tag('td'))[1:-1]]
    # link = row.find_by_xpath("//input[@src='images/pdf_icon.png']").first
    link = row.find_by_tag("input").first
    link.click()

    # Two sends wait is not enough
    start_time = time.time()
    end_time = time.time()
    url = browser.windows[-1].url
    while 'pdf' not in url or end_time - start_time > 120:
        time.sleep(1)
        url = browser.windows[-1].url
        end_time = time.time()

    print(url)
    if 'pdf' in url:
        browser.windows[-1].close()
    row_data.append(url)
    metadata.append(row_data)

@@ -1,12 +1,167 @@
+# I downloaded many pdfs by observing the url pattern of the pdf%0A# %60seq 160000 # 169999 %7C parallel wget http://egazette.nic.in/Write%5ECadData/2016/%7B%7D.pdf%60%0A%0A
 from splinte
@@ -218,16 +218,28 @@
 ort time
+%0Aimport json
 %0A%0Aurl = 
@@ -422,24 +422,29 @@
 lcat1%22, wait
+_time
 =10)%0Aelem = 
@@ -742,262 +742,8 @@
 ()%0A%0A
-# TODO: will have to add wait between all of these actions%0A%0Asoup = BeautifulSoup(browser.html, 'html.parser')%0A%0A# TODO: Wait till %22scroll to top%22 appears%0A%0A# ContentPlaceHolder1_rptmmain_imgbtndownload_10%0A%0A# ContentPlaceHolder1_rptmmain_imgbtndownload_13%0A%0A
 # x 
@@ -883,23 +883,16 @@
 adata.%0A%0A
-assert 
 browser.
@@ -932,91 +932,392 @@
 wait
-=10)%0Ametadata = list()%0Arows = list(browser.find_by_tag('tr'))%0Afor row in
+_time=60)%0A# The code breaks at this and browser stops loading%0A%0Aprint(%222014 page opened%22)%0A%0Ametadata = list()%0Arows = list(browser.find_by_tag('tr'))%0Afor i in range(len(rows)):%0A    # The page is refereshed each time the link is clicked and rows object we%0A    # obtained is detached from the dom, hence we need to revaluate it%0A    rows = list(browser.find_by_tag('tr'))%0A    row =
  rows%5B
-:20%5D:
+i%5D%0A
 %0A   
@@ -1576,36 +1576,80 @@
   # 
-Two sends wait is not enough
+At max wait for 30 sec till the page with pdf appears%0A    wait_time = 30
 %0A   
@@ -1791,13 +1791,19 @@
 ime 
-%3E 120
+%3C wait_time
 :%0A  
@@ -1890,16 +1890,42 @@
 e.time()
+%0A        # print(end_time)
 %0A%0A    pr
@@ -2045,8 +2045,125 @@
 w_data)%0A
+%0A    if i %25 10 == 0:%0A        with open('metadata_%7B%7D.json'.format(i), 'w') as fp:%0A            json.dump(metadata, fp)%0A

update union code