#!/usr/bin/env python
import sys
import os
import argparse
import logging
import re
import multiprocessing as mp
import signal
import cdec.sa

extractor, prefix = None, None
online = False

def make_extractor(config, grammars, features):
    global extractor, prefix, online
    signal.signal(signal.SIGINT, signal.SIG_IGN) # Let parent process catch Ctrl+C
    load_features(features)
    extractor = cdec.sa.GrammarExtractor(config, online)
    prefix = grammars

def load_features(features):
    for featdef in features:
        logging.info('Loading additional feature definitions from %s', featdef)
        prefix = os.path.dirname(featdef)
        sys.path.append(prefix)
        __import__(os.path.basename(featdef).replace('.py', ''))
        sys.path.remove(prefix)

def extract(inp):
    global extractor, prefix, online
    i, sentence = inp
    sentence = sentence[:-1]
    fields = re.split('\s*\|\|\|\s*', sentence)
    suffix = ''
    # 3 fields for online mode, 1 for normal
    if online:
        if len(fields) < 3:
            sys.stderr.write('Error: online mode requires references and alignments.'
                    '  Not adding sentence to training data: {0}\n'.format(sentence))
            sentence = fields[0]
        else:
            sentence, reference, alignment = fields[0:3]
        if len(fields) > 3:
            suffix = ' ||| ' + ' ||| '.join(fields[3:])
    else:
        if len(fields) > 1:
            sentence = fields[0]
            suffix = ' ||| ' + ' ||| '.join(fields[1:])
    grammar_file = os.path.join(prefix, 'grammar.{0}'.format(i))
    with open(grammar_file, 'w') as output:
        for rule in extractor.grammar(sentence):
            output.write(str(rule)+'\n')
    # Add training instance _after_ extracting grammars
    if online:
        extractor.add_instance(sentence, reference, alignment)
    grammar_file = os.path.abspath(grammar_file)
    return '<seg grammar="{0}" id="{1}"> {2} </seg>{3}'.format(grammar_file, i, sentence, suffix)

def main():
    global online
    logging.basicConfig(level=logging.INFO)
    parser = argparse.ArgumentParser(description='Extract grammars from a compiled corpus.')
    parser.add_argument('-c', '--config', required=True,
                        help='extractor configuration')
    parser.add_argument('-g', '--grammars', required=True,
                        help='grammar output path')
    parser.add_argument('-j', '--jobs', type=int, default=1,
                        help='number of parallel extractors')
    parser.add_argument('-s', '--chunksize', type=int, default=10,
                        help='number of sentences / chunk')
    parser.add_argument('-f', '--features', nargs='*', default=[],
                        help='additional feature definitions')
    parser.add_argument('-o', '--online', action='store_true', default=False,
                        help='online grammar extraction')
    args = parser.parse_args()

    if not os.path.exists(args.grammars):
        os.mkdir(args.grammars)
    for featdef in args.features:
        if not featdef.endswith('.py'):
            sys.stderr.write('Error: feature definition file <{0}>'
                    ' should be a python module\n'.format(featdef))
            sys.exit(1)
    
    online = args.online
    
    if args.jobs > 1:
        logging.info('Starting %d workers; chunk size: %d', args.jobs, args.chunksize)
        pool = mp.Pool(args.jobs, make_extractor, (args.config, args.grammars, args.features))
        try:
            for output in pool.imap(extract, enumerate(sys.stdin), args.chunksize):
                print(output)
        except KeyboardInterrupt:
            pool.terminate()
    else:
        make_extractor(args.config, args.grammars, args.features)
        for output in map(extract, enumerate(sys.stdin)):
            print(output)

if __name__ == '__main__':
    main()

@@ -36,16 +36,38 @@
 port os%0A
+import re%0Aimport gzip%0A
 import a
@@ -88,34 +88,38 @@
  logging%0Aimport 
-re
+signal
 %0Aimport multipro
@@ -132,30 +132,16 @@
 g as mp%0A
-import signal%0A
 import c
@@ -185,18 +185,35 @@
 e%0Aonline
- =
+, compress = False,
  False%0A%0A
@@ -235,33 +235,11 @@
 tor(
-config, grammars, feature
+arg
 s):%0A
@@ -270,24 +270,34 @@
 efix, online
+, compress
 %0A    signal.
@@ -378,32 +378,37 @@
   load_features(
+args.
 features)%0A    ex
@@ -442,16 +442,21 @@
 tractor(
+args.
 config, 
@@ -476,24 +476,29 @@
 refix = 
+args.
 grammars
 %0A%0Adef lo
@@ -489,16 +489,70 @@
 grammars
+%0A    online = args.online%0A    compress = args.compress
 %0A%0Adef lo
@@ -905,24 +905,34 @@
 efix, online
+, compress
 %0A    i, sent
@@ -1271,17 +1271,16 @@
  data: %7B
-0
 %7D%5Cn'.for
@@ -1605,24 +1605,25 @@
 fields%5B1:%5D)%0A
+%0A
     grammar_
@@ -1663,37 +1663,99 @@
 mar.
-%7B0%7D'.format(i))%0A    with open
+'+str(i))%0A    if compress: grammar_file += '.gz'%0A    with (gzip.open if compress else open)
 (gra
@@ -2085,17 +2085,16 @@
 r=%22%7B
-0
 %7D%22 id=%22%7B
 1%7D%22%3E
@@ -2093,25 +2093,20 @@
 d=%22%7B
-1
 %7D%22%3E
- %7B2%7D 
+%7B%7D
 %3C/seg%3E%7B
-3
 %7D'.f
@@ -2163,26 +2163,8 @@
 ():%0A
-    global online%0A
     
@@ -2966,23 +2966,8 @@
 ue',
- default=False,
 %0A   
@@ -3017,24 +3017,149 @@
 xtraction')%0A
+    parser.add_argument('-z', '--compress', action='store_true',%0A                        help='compress grammars with gzip')%0A
     args = p
@@ -3389,17 +3389,16 @@
  file %3C%7B
-0
 %7D%3E'%0A    
@@ -3494,38 +3494,8 @@
    %0A
-    online = args.online%0A    %0A
     
@@ -3658,45 +3658,9 @@
 args
-.config, args.grammars, args.features
+,
 ))%0A 
@@ -3889,45 +3889,8 @@
 args
-.config, args.grammars, args.features
 )%0A  

Add compression option to grammar extractor