@@ -0,0 +1,2229 @@
+package uk.ac.susx.tag.classificationframework.featureextraction.tokenisation;%0A%0Aimport uk.ac.susx.tag.classificationframework.Util;%0Aimport uk.ac.susx.tag.classificationframework.datastructures.AnnotatedToken;%0Aimport uk.ac.susx.tag.classificationframework.datastructures.Document;%0Aimport uk.ac.susx.tag.classificationframework.datastructures.Instance;%0A%0A%0Aimport edu.stanford.nlp.pipeline.*;%0Aimport edu.stanford.nlp.ling.CoreAnnotations;%0Aimport edu.stanford.nlp.ling.CoreLabel;%0Aimport edu.stanford.nlp.util.StringUtils;%0A%0Aimport java.io.IOException;%0Aimport java.io.ObjectInputStream;%0Aimport java.util.*;%0A%0A%0A%0A%0Apublic class TokeniserChineseStanford implements Tokeniser %7B%0A%0A    private static final long serialVersionUID = 0L;%0A    private transient StanfordCoreNLP pipeline;%0A%0A    public TokeniserChineseStanford() throws IOException %7B%0A        loadPipeline();%0A    %7D%0A%0A    @Override%0A    public Document tokenise(Instance document) %7B%0A%0A        Document tokenised = new Document(document);%0A        if (!Util.isNullOrEmptyText(document)) %7B%0A            int end = 0;%0A%0A            Annotation annotation = new Annotation(document.text);%0A            pipeline.annotate(annotation);%0A            List%3CCoreLabel%3E tokens = annotation.get(CoreAnnotations.TokensAnnotation.class);%0A            for (CoreLabel token : tokens) %7B%0A                String word = token.get(CoreAnnotations.TextAnnotation.class);%0A                int start = document.text.indexOf(word, end);%0A                end = start + word.length();%0A                AnnotatedToken annotatedToken = new AnnotatedToken(word);%0A                annotatedToken.start(start);%0A                annotatedToken.end(end);%0A                tokenised.add(annotatedToken);%0A            %7D%0A        %7D%0A%0A        return tokenised;%0A    %7D%0A%0A    @Override%0A    public String configuration() %7B%0A        return %22%22;%0A    %7D%0A%0A    public void loadPipeline()%7B%0A        Properties props = StringUtils.argsToProperties(%22-props%22, %22StanfordCoreNLP-chinese.properties%22);%0A        props.setProperty(%22annotators%22, %22tokenize%22);%0A        pipeline = new StanfordCoreNLP(props);%0A    %7D%0A%0A    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException %7B%0A        in.defaultReadObject();%0A        loadPipeline();%0A    %7D%0A%7D%0A

add stanford chinese tokeniser