#!/usr/bin/env python
import click
import os
import codecs
import json

from xtas.tasks._frog import parse_frog, frog_to_saf

from nlppln.utils import create_dirs, out_file_name


@click.command()
@click.argument('input_files', nargs=-1, type=click.Path(exists=True))
@click.argument('output_dir', nargs=1, type=click.Path())
def frog2saf(input_files, output_dir):
    create_dirs(output_dir)

    for fi in input_files:
        with codecs.open(fi) as f:
            lines = f.readlines()
            lines = [line.strip() for line in lines]
        saf_data = frog_to_saf(parse_frog(lines))

        head, tail = os.path.split(fi)
        fname = tail.replace(os.path.splitext(tail)[1], '')

        out_file = os.path.join(output_dir, out_file_name(output_dir, fname,
                                                          'json'))
        with codecs.open(out_file, 'wb', encoding='utf-8') as f:
            json.dump(saf_data, f, indent=4)


if __name__ == '__main__':
    frog2saf()

@@ -68,114 +68,1855 @@
 son%0A
-%0Afrom xtas.tasks._frog import parse_frog, frog_to_saf%0A%0Afrom nlppln.utils import create_dirs, out_file_name
+import datetime%0A%0Afrom nlppln.utils import create_dirs, out_file_name%0A%0A%0A_POSMAP = %7B%22VZ%22: %22P%22,%0A           %22N%22: %22N%22,%0A           %22ADJ%22: %22A%22,%0A           %22LET%22: %22.%22,%0A           %22VNW%22: %22O%22,%0A           %22LID%22: %22D%22,%0A           %22SPEC%22: %22M%22,%0A           %22TW%22: %22Q%22,%0A           %22WW%22: %22V%22,%0A           %22BW%22: %22B%22,%0A           %22VG%22: %22C%22,%0A           %22TSW%22: %22I%22,%0A           %22MWU%22: %22U%22,%0A           %22%22: %22?%22,%0A           %7D%0A%0A%0Adef parse_frog(lines):%0A    %22%22%22%0A    Interpret the output of the frog parser.%0A    Input should be an iterable of lines (i.e. the output of call_frog)%0A    Result is a sequence of dicts representing the tokens%0A    %22%22%22%0A    sid = 0%0A    for i, line in enumerate(lines):%0A        if not line:%0A            # end of sentence marker%0A            sid += 1%0A        else:%0A            parts = line.split(%22%5Ct%22)%0A            tid, token, lemma, morph, pos, conf, ne, _, parent, rel = parts%0A            if rel:%0A                rel = (rel, int(parent) - 1)%0A            result = dict(id=i, sentence=sid, word=token, lemma=lemma,%0A                          pos=pos, pos_confidence=float(conf),%0A                          rel=rel)%0A            if ne != 'O':%0A                # NER label from BIO tags%0A                result%5B%22ne%22%5D = ne.split('_', 1)%5B0%5D%5B2:%5D%0A            yield result%0A%0A%0Adef _add_pos1(token):%0A    %22%22%22%0A    Adds a 'pos1' element to a frog token.%0A    %22%22%22%0A    result = token.copy()%0A    result%5B'pos1'%5D = _POSMAP%5Btoken%5B'pos'%5D.split(%22(%22)%5B0%5D%5D%0A    return result%0A%0A%0Adef frog_to_saf(tokens):%0A    %22%22%22%0A    Convert frog tokens into a new SAF document%0A    %22%22%22%0A    tokens = %5B_add_pos1(token) for token in tokens%5D%0A    module = %7B'module': %22frog%22,%0A              %22started%22: datetime.datetime.now().isoformat()%7D%0A    return %7B%22header%22: %7B'format': %22SAF%22,%0A                       'format-version': %220.0%22,%0A                       'processed': %5Bmodule%5D%0A                       %7D, %22tokens%22: tokens%7D
 %0A%0A%0A@

Remove dependency on xtas for frog_to_saf