@@ -0,0 +1,1284 @@
+# coding: utf8%0Afrom __future__ import unicode_literals%0A%0Afrom ...compat import json_dumps, path2str%0Afrom ...util import prints%0A%0A%0Adef iob2json(input_path, output_path, n_sents=10, *a, **k):%0A    %22%22%22%0A    Convert IOB files into JSON format for use with train cli.%0A    %22%22%22%0A    # TODO: This isn't complete yet -- need to map from IOB to%0A    # BILUO%0A    with input_path.open() as file_:%0A        docs = read_iob(file_)%0A%0A    output_filename = input_path.parts%5B-1%5D.replace(%22.iob%22, %22.json%22)%0A    output_file = output_path / output_filename%0A    with output_file.open('w', encoding='utf-8') as f:%0A        f.write(json_dumps(docs))%0A    prints(%22Created %25d documents%22 %25 len(docs),%0A           title=%22Generated output file %25s%22 %25 path2str(output_file))%0A%0A%0Adef read_iob(file_):%0A    sentences = %5B%5D%0A    for line in file_:%0A        if not line.strip():%0A            continue%0A        tokens = %5Bt.rsplit('%7C', 2) for t in line.split()%5D%0A        words, pos, iob = zip(*tokens)%0A        sentences.append(%5B%0A            %7B'orth': w, 'tag': p, 'ner': ent%7D%0A            for (w, p, ent) in zip(words, pos, iob)%0A        %5D)%0A    sentences = %5B%7B'tokens': sent%7D for sent in sentences%5D%0A    paragraphs = %5B%7B'sentences': %5Bsent%5D%7D for sent in sentences%5D%0A    docs = %5B%7B'id': 0, 'paragraphs': %5Bpara%5D%7D for para in paragraphs%5D%0A    return docs%0A

Add incomplete iob converter