# -*- coding: utf-8 -*-
# program: 104jobdet.py
import sys
import urllib2
import re

def getDataFromURL( url ):
    try:
        response = urllib2.urlopen(urllib2.Request(url))
    except (ValueError, urllib2.URLError) as e:
        print e
        sys.exit(0)
    data = response.read() 
    # remove '\r' character
    data = data.replace('\r', '')
    return data

def getTxtFromFile( filename ):
    f = open( filename, 'r' )
    data = f.read()
    # remove '\r' character
    data = data.replace('\r', '')
    return data

def parse104(website):
    # match the strings
    # begin with "<div class=\"intro\">" or "<div class=\"interview\">"
    # end with "<div class=\"intro\">" or "<div class=\"interview\">" or "<script type"
    # (?s) means re.DOTALL
    # (?:...) means non-capturing mode
    found = re.findall("(?s)(<div class=\"(?:intro|interview)\">(?:.(?!<script type|<div class=\"(?:intro|interview)))*)", website)

    data = []
    for x in found:
        # remove html tag, space character, and statements begin with '#'
        data.append( re.sub("<[^>]*>|[ \t]+|#(?<=#).*","",str(x)) )
    return data



if len(sys.argv)!=2:
    print "agrv Error."
    print "[usage]: python [URL]"
    sys.exit(0)

#infile = sys.argv[1]
URL = sys.argv[1]
#htmlfile = getTxtFromFile(infile)
htmlfile = getDataFromURL(URL)
data = parse104(htmlfile)

# "detail" is a list of [engtitle, zhtitle, repattern, matchedstr]
detail = [ [ "update", "更新:" , "(?<=更新日期：)[\d-]+" ],
        [ "content", "工作內容:", "(?<=工作內容\n).*" ],
        [ "treatment", "工作待遇:", "(?<=工作待遇：\n).*" ],
        [ "jobtype", "工作性質:", "(?<=工作性質：\n).*" ],
        [ "location", "上班地點:", "(?<=上班地點：\n).*" ],
        [ "worktime", "上班時段:", "(?<=上班時段：\n).*" ],
        [ "leavesys", "休假制度:", "(?<=休假制度：\n).*" ],
        [ "availability", "可上班日:", "(?<=可上班日：\n).*" ],
        [ "reqnum", "需求人數:", "(?<=需求人數：\n).*" ],
        [ "acceptid", "接受身份:", "(?<=接受身份：\n).*" ],
        [ "exp", "工作經歷:", "(?<=工作經歷：\n).*" ],
        [ "education", "學歷要求:", "(?<=學歷要求：\n).*" ],
        [ "department", "科系要求:", "(?<=科系要求：\n).*" ],
        [ "language", "語文條件:", "(?<=語文條件：\n).*" ],
        [ "tool", "擅長工具:", "(?<=擅長工具：\n).*" ],
        [ "skill", "工作技能:", "(?<=工作技能：\n).*" ],
        [ "othercond", "其他條件:", "(?<=其他條件：\n).*" ],
        [ "contact", "聯絡人:", "(?<=聯&nbsp;絡&nbsp;人：\n).*" ],
        [ "emailsrc", "Email:", "(?<=E-mail：\nfun_flash_output\(\")[^\"]+" ],
        [ "inperson", "親洽:", "(?<=親　　洽：\n).*" ],
        [ "telesrc", "電洽:", "(?<=電　　洽：).*\n.*" ],
        [ "other", "其它:", "(?<=其　　他：).*" ]]

essence=""
for i in data:
    for j in re.split("\n",i):
        # ignore blank lines
        if len(j)>0:
            essence += (j+"\n")

for i in range(len(detail)):
    if "telesrc" in detail[i][0]:
        # the telephone numbers src is a link to an image which show tele numbers.
        teletmp = re.search(detail[i][2], htmlfile)
        if teletmp:
            match = re.search("(?<=src=')[^']+", teletmp.group(0))
    elif "emailsrc" in detail[i][0]:
        # the email src is a link to a flash called by "fun_flash_output"
        match = re.search(detail[i][2], essence)
    else:
        match = re.search(detail[i][2], essence)

    if match:
        detail[i].append(match.group(0))
    else:
        detail[i].append('')

'''
# dump as .psql file
print "UPDATE jobs SET",
for i in range(1,len(detail)):
    print detail[i][0] + "='" + detail[i][3] + "'",
    if i < len(detail)-1:
        print ",",
print "WHERE joburl='" + URL + "';"
'''

# columns output
for i in range(1,len(detail)):
    for j in [1,3]:
        print detail[i][j],
    print

@@ -1221,16 +1221,30 @@
 it(0)%0A%0A#
+arg as a file%0A
 infile =
@@ -1260,27 +1260,8 @@
 %5B1%5D%0A
-URL = sys.argv%5B1%5D%0A#
 html
@@ -1290,16 +1290,51 @@
 infile)%0A
+%0A#arg as an url%0AURL = sys.argv%5B1%5D%0A#
 htmlfile
@@ -1499,24 +1499,94 @@
 )%5B%5Cd-%5D+%22 %5D,%0A
+        %5B %22jobname%22, %22%E5%BF%97%E5%B7%A5%E6%9C%8D%E5%8B%99:%22, %22(?%3C=comp_name%5C%22%3E%5Cs%5Cs%5Cs%3Ch1%3E)%5B%5E%3C%5Ct%5D+%22 %5D,%0A
         %5B %22c
@@ -2839,16 +2839,102 @@
 %0A    if 
+%22jobname%22 in detail%5Bi%5D%5B0%5D:%0A        match = re.search(detail%5Bi%5D%5B2%5D, htmlfile)%0A    elif 
 %22telesrc
@@ -3814,8 +3814,16 @@
   print%0A
+'''%0A'''%0A

crawl the title of a volunteer job