@@ -0,0 +1,2146 @@
+%22%22%22Script that converts a field in a tabular data file to cptm input files%0A%0AUsed for the CAP vragenuurtje data.%0A%0AUses frog to pos-tag and lemmatize the data.%0A%0AUsage: python tabular2cpt_input.py %3Ccsv of excel file%3E %3Cfull text field name%3E%0A%3Cdir out%3E%0A%22%22%22%0A%0Aimport pandas as pd%0Aimport logging%0Aimport sys%0Aimport argparse%0Aimport re%0A%0Afrom pynlpl.clients.frogclient import FrogClient%0A%0Afrom cptm.utils.inputgeneration import Perspective, remove_trailing_digits%0Afrom cptm.utils.dutchdata import pos_topic_words, pos_opinion_words, word_types%0A%0Alogger = logging.getLogger(__name__)%0Alogging.basicConfig(format='%25(levelname)s : %25(message)s', level=logging.DEBUG)%0A%0Aparser = argparse.ArgumentParser()%0Aparser.add_argument('in_file', help='excel or csv file containing text data')%0Aparser.add_argument('text_field', help='name of the text field')%0Aparser.add_argument('out_dir', help='directory where output should be stored')%0Aargs = parser.parse_args()%0A%0Aport = 8020%0A%0Atry:%0A    frogclient = FrogClient('localhost', port)%0Aexcept:%0A    logger.error('Cannot connect to the Frog server. '%0A                 'Is it running at port %7B%7D?'.format(port))%0A    logger.info('Start the Frog server with %22docker run ''-p 127.0.0.1:%7B%7D:%7B%7D '%0A                '-t -i proycon/lamachine frog -S %7B%7D%22'.format(port, port, port))%0A    sys.exit(1)%0A%0Aregex = re.compile(r'%5C(.*%5C)')%0A%0Aif args.in_file.endswith('.xls') or args.in_file.endswith('.xlsx'):%0A    input_data = pd.read_excel(args.in_file)%0Aelse:%0A    input_data = pd.read_csv(args.in_file)%0A%0Afor i, text in enumerate(input_data%5Bargs.text_field%5D):%0A    p = Perspective('', pos_topic_words(), pos_opinion_words())%0A    if i %25 25 == 0:%0A        logger.info('Processing text %7B%7D of %7B%7D'.format(i + 1,%0A                    len(input_data%5Bargs.text_field%5D)))%0A    if pd.notnull(text):%0A        for data in frogclient.process(text):%0A            word, lemma, morph, ext_pos = data%5B:4%5D%0A            if ext_pos:  # ext_pos can be None%0A                pos = regex.sub('', ext_pos)%0A                if pos in word_types():%0A                    p.add(pos, remove_trailing_digits(lemma))%0A        file_name = '%7B%7D.txt'.format(i)%0A        p.write2file(args.out_dir, file_name)%0A

Add script that converts a field in a tabular data file to cptm input