# coding: utf8
from __future__ import unicode_literals

import cytoolz

from ...gold import iob_to_biluo


def iob2json(input_data, n_sents=10, *args, **kwargs):
    """
    Convert IOB files into JSON format for use with train cli.
    """
    docs = []
    for group in cytoolz.partition_all(n_sents, docs):
        group = list(group)
        first = group.pop(0)
        to_extend = first["paragraphs"][0]["sentences"]
        for sent in group[1:]:
            to_extend.extend(sent["paragraphs"][0]["sentences"])
        docs.append(first)
    return docs


def read_iob(raw_sents):
    sentences = []
    for line in raw_sents:
        if not line.strip():
            continue
        tokens = [t.split("|") for t in line.split()]
        if len(tokens[0]) == 3:
            words, pos, iob = zip(*tokens)
        else:
            words, iob = zip(*tokens)
            pos = ["-"] * len(words)
        biluo = iob_to_biluo(iob)
        sentences.append(
            [
                {"orth": w, "tag": p, "ner": ent}
                for (w, p, ent) in zip(words, pos, biluo)
            ]
        )
    sentences = [{"tokens": sent} for sent in sentences]
    paragraphs = [{"sentences": [sent]} for sent in sentences]
    docs = [{"id": 0, "paragraphs": [para]} for para in paragraphs]
    return docs

@@ -53,24 +53,8 @@
 ls%0A%0A
-import cytoolz%0A%0A
 from
@@ -82,16 +82,46 @@
 o_biluo%0A
+from ...util import minibatch%0A
 %0A%0Adef io
@@ -283,42 +283,30 @@
  in 
-cytoolz.partition_all(n_sents, doc
+minibatch(docs, n_sent
 s):%0A

Remove cytoolz usage in CLI