# -*- coding:utf-8 -*-
# Author: hankcs
# Date: 2020-06-12 20:34


def generate_words_per_line(file_path):
    with open(file_path, encoding='utf-8') as src:
        for line in src:
            cells = line.strip().split()
            if not cells:
                continue
            yield cells


def words_to_bmes(words):
    tags = []
    for w in words:
        if not w:
            raise ValueError('{} contains None or zero-length word {}'.format(str(words), w))
        if len(w) == 1:
            tags.append('S')
        else:
            tags.extend(['B'] + ['M'] * (len(w) - 2) + ['E'])
    return tags


def words_to_bi(words):
    tags = []
    for w in words:
        if not w:
            raise ValueError('{} contains None or zero-length word {}'.format(str(words), w))
        tags.extend(['B'] + ['I'] * (len(w) - 1))
    return tags


def bmes_to_words(chars, tags):
    result = []
    if len(chars) == 0:
        return result
    word = chars[0]

    for c, t in zip(chars[1:], tags[1:]):
        if t == 'B' or t == 'S':
            result.append(word)
            word = ''
        word += c
    if len(word) != 0:
        result.append(word)

    return result


def bmes_to_spans(tags):
    result = []
    offset = 0
    pre_offset = 0
    for t in tags[1:]:
        offset += 1
        if t == 'B' or t == 'S':
            result.append((pre_offset, offset))
            pre_offset = offset
    if offset != len(tags):
        result.append((pre_offset, len(tags)))

    return result


def bmes_of(sentence, segmented):
    if segmented:
        chars = []
        tags = []
        words = sentence.split()
        for w in words:
            chars.extend(list(w))
            if len(w) == 1:
                tags.append('S')
            else:
                tags.extend(['B'] + ['M'] * (len(w) - 2) + ['E'])
    else:
        chars = list(sentence)
        tags = ['S'] * len(chars)
    return chars, tags

@@ -1931,12 +1931,466 @@
 chars, tags%0A
+%0A%0Adef iobes_to_bilou(src, dst):%0A    with open(src) as src, open(dst, 'w') as out:%0A        for line in src:%0A            line = line.strip()%0A            if not line:%0A                out.write('%5Cn')%0A                continue%0A            word, tag = line.split('%5Ct')%0A            if tag.startswith('E-'):%0A                tag = 'L-' + tag%5B2:%5D%0A            elif tag.startswith('S-'):%0A                tag = 'U-' + tag%5B2:%5D%0A            out.write(f'%7Bword%7D%5Ct%7Btag%7D%5Cn')%0A

Add utility iobes_to_bilou