@@ -0,0 +1,2176 @@
+#!/usr/bin/env python%0A#%0A# heuristic forum detection %0A# pass:%0A# (1) cow.xml file (may be gzipped)%0A# (2) optionally, an output path %0A#     (directories will be created if they%0A#      do not exist)%0A# %0A# output is gzipped, the output file name%0A# is determined on the basis of th einout file name%0A#%0A#%0A#%0Aimport sys%0Aimport re%0Aimport gzip%0Aimport os%0A%0A%0ARE_FORUM=r'.*(?:forum%7C/board/%7C/threads/%7Cshowthread%7C/archive/(?:index.php/)?t-%5B0-9%5D+%5C.html%7Cviewtopic%7Cftopic%7C/%5C?topic=%7C(?:/threads/%5BA-Za-z%5D+(?:-%5BA-Za-z%5D+)+%5C.%5B0-9%5D+/$)%7C/thread-%5B0-9%5D+%5C.html$%7C/topic/%5B0-9%5D+(?:-%5BA-Za-z%5D+)+%7C/t%5B0-9%5D+(?:-%5BA-Za-z%5D%5C+)+%5C.html)'%0A%0A%0Adef basename(filename):%0A    return(re.sub('%5C.xml(?:%5C.gz)?', '', filename).split(%22/%22)%5B-1%5D)%0A%0A%0A%0Adef main():%0A%0A    try:%0A        if sys.argv%5B1%5D.endswith(%22.gz%22):%0A            in_h = gzip.open(sys.argv%5B1%5D, 'rb')%0A        else:%0A            in_h = open(sys.argv%5B1%5D)%0A    except IOError:%0A        sys.exit(%22Cannot open %25s%5Cn%22 %25sys.argv%5B1%5D)%0A%0A     %0A    # if an output dir was specified, check if it exists:%0A    if len(sys.argv) %3E 2:%0A        outdir = sys.argv%5B2%5D.strip().rstrip(%22/%22)%0A        if not os.path.exists(outdir):%0A            os.makedirs(outdir)%0A    else:%0A        outdir = %22%22%0A%0A   %0A    # check if outfile exists:%0A    outfilename = basename(sys.argv%5B1%5D) + %22.forum.xml.gz%22%0A    outfullpath = %22/%22.join(%5Bi for i in %5Boutdir, outfilename%5D if len(i) %3E 0%5D)%0A%0A    if os.path.exists(outfullpath):%0A        sys.exit('Outfile %25s exists.%5Cn' %25outfullpath)%0A    else:%0A        out_h = gzip.open(outfullpath, 'wb')%0A        #out_h = open(outfullpath, 'w')%0A%0A%0A%0A%0A%0A    for line in in_h:%0A        line = line.decode('utf8')%0A%0A        if not line.startswith(u'%3Cdoc '):%0A            out_h.write(line.encode('utf8'))%0A%0A        else:%0A            m = re.search(u' url=%22(%5B%5E %5D+)%22', line)%0A            if m:%0A                url = m.group(1)%0A            else:%0A                url = %22%22%0A            %0A            if re.match(RE_FORUM, url, re.UNICODE):%0A                line = re.sub(u'%3E$', r' forum=%221%22%3E', line, re.UNICODE)%0A            else:%0A                line = re.sub(u'%3E$', r' forum=%220%22%3E', line, re.UNICODE)%0A%0A%0A            out_h.write(line.encode('utf8'))%0A%0A%0A    in_h.close()%0A    out_h.close()%0A%0A%0A%0Aif __name__ == %22__main__%22:%0A    main()%0A

add cow-detect-forum.py