@@ -0,0 +1,2291 @@
+import codecs%0Aimport re%0A%0Afrom marmot.representations.representation_generator import RepresentationGenerator%0A%0A%0Aclass SegmentationRepresentationGenerator(RepresentationGenerator):%0A    '''%0A    Source, target, tags, segmentation files, one line per file, whitespace tokenized%0A    Segmentation file -- can be Moses output with phrase segmentation (with '-t' option)%0A        or just have the information on segments.%0A    Segments have to be in the form %7Ci-j%7C where i is the index of the first word in segment,%0A                                                j -- the last word in segment.%0A    Examples of acceptable segmentation formats:%0A       (1) he is %7C0-1%7C a good %7C2-3%7C ukulele %7C4-4%7C player %7C5-5%7C . %7C6-6%7C%0A       (2) %7C0-1%7C %7C2-3%7C %7C4-4%7C %7C5-5%7C %7C6-6%7C%0A    in the format (1) the words can be from the source lang (or any other), only numbers matter%0A    the format (2) is the same as (1) but with no words%0A    they are parsed in the same way: just substrings '%7Ci-j%7C' are extracted%0A    '''%0A%0A    def __init__(self, source_file, target_file, tags_file, segmentation_file):%0A        self.data = self.parse_files(source_file, target_file, tags_file, segmentation_file)%0A%0A    @staticmethod%0A    def parse_files(source_file, target_file, tags_file, segmentation_file):%0A%0A        with codecs.open(source_file, encoding='utf8') as source:%0A            source_lines = %5Bline.split() for line in source%5D%0A%0A        with codecs.open(target_file, encoding='utf8') as target:%0A            target_lines = %5Bline.split() for line in target%5D%0A%0A        with codecs.open(tags_file, encoding='utf8') as tags:%0A            tags_lines = %5Bline.split() for line in tags%5D%0A%0A        seg_regexp = re.compile(%22%5C%7C%5Cd+-%5Cd+%5C%7C%22)%0A        with codecs.open(segmentation_file, encoding='utf-8') as segmentation:%0A            segments = %5B%5D%0A            for line in segmentation:%0A                seg_strings = seg_regexp.findall(line)%0A                seg_list = %5B%5D%0A                for a_seg in seg_strings:%0A                    a_pair = a_seg.strip('%7C').split('-')%0A                    seg_list.append((int(a_pair%5B0%5D), int(a_pair%5B1%5D)+1))%0A                segments.append(seg_list)%0A%0A        return %7B'target': target_lines, 'source': source_lines, 'tags': tags_lines, 'segmentation': segments%7D%0A%0A    def generate(self, data_obj=None):%0A        return self.data%0A

generate phrase segmentation representation from Moses output