

// textutil -convert txt *

var parse = require('csv-parse/lib/sync'),
	csvstr = require('csv-stringify'),
	fs = require('fs'),
	promise = require('bluebird'),
	_ = require('lodash'),
	qs = require('querystring'),
	config = JSON.parse(fs.readFileSync('./munge-config.json')),
	cutils = require('./client-utils'),
	round_re = /ROUND\W*(\d+)/,
	headers,
	exit_re = /exit\W*\>|exit interview|exit questions|exit survey/i;

var loadCSV = (fname) => {
	var text = 	fs.readFileSync(fname).toString();
	console.log("Parsing file ", fname, "(", text.length, ")");
	var data = parse(text, {max_limit_on_data_read:9999999999});
	headers = data[0];
	data = data.slice(1);
	data = data.map((x) => _.zipObject(headers,x));
	return data;
}, s = (str) => (str||'').split(' ').map((x) => x.trim().toUpperCase().trim(',')).filter((x) => x.length),
  main = (fname, cols) => {
	console.info('got cols');
	var rows = loadCSV(fname),
		people = cols,
		rater_a = cols[0], rater_b = cols[1],
		tags = _.uniq(_.flatten(rows.map((r) => _.flatten(cols.map((c) => s(r[c])))))),
		header = ['id', rater_a, rater_b ],
		rowos = [header].concat(
			rows.filter((r) => {
				return r[rater_a].length && r[rater_b].length;
			}).reduce((cat, r) => {
				// [[T], [T], [T], [T]]
				return cat.concat(tags.map((T) => {
					return [ r.id + '-' + T, s(r[rater_a]).indexOf(T) >= 0 ? 'YES' : 'NO', s(r[rater_b]).indexOf(T) >= 0 ? 'YES' : 'NO' ];
				}));
			}, []));

	console.log('rowos ', rowos);

	return new Promise((acc, rej) => {
		csvstr(rowos, (err, output) => {
			if (!err) { return acc(output); }
			console.error("Error ", err);
			rej(err);
		});
	});	
};

if (require.main === module) { 
	main(process.argv[2], process.argv.slice(3)).then((data) => {
		// write it !
		console.info("Writing output ", 'tag-kappa.csv', data.length);
		fs.writeFileSync('./tag-kappa.csv', data);
		console.log('done');
	}); 
}
@@ -826,16 +826,206 @@
 ength),%0A
+%09uniquetags = (fname, rater_a, rater_b) =%3E %7B%0A%09%09var rows = loadCSV(fname),%0A%09%09%09cols = %5Brater_a, rater_b%5D;%0A%09%09return _.uniq(_.flatten(rows.map((r) =%3E _.flatten(cols.map((c) =%3E s(r%5Bc%5D))))));%0A%09%7D,%0A
   main =
@@ -1037,16 +1037,29 @@
 me, cols
+, choose_tags
 ) =%3E %7B%0A%09
@@ -1177,16 +1177,31 @@
 %09%09tags =
+ choose_tags %7C%7C
  _.uniq(
@@ -1881,16 +1881,95 @@
 ule) %7B %0A
+%09// node command fileout pers1 pers2 %0A%09if (process.argv%5B5%5D !== 'separate') %7B %0A%09
 %09main(pr
@@ -2023,16 +2023,17 @@
 a) =%3E %7B%0A
+%09
 %09%09// wri
@@ -2042,16 +2042,17 @@
  it !%0A%09%09
+%09
 console.
@@ -2076,16 +2076,28 @@
 put %22, '
+../mitm_out/
 tag-kapp
@@ -2120,16 +2120,17 @@
 gth);%0A%09%09
+%09
 fs.write
@@ -2140,16 +2140,26 @@
 eSync('.
+./mitm_out
 /tag-kap
@@ -2174,16 +2174,17 @@
  data);%0A
+%09
 %09%09consol
@@ -2202,11 +2202,471 @@
 ');%0A
+%09
 %09%7D);
- 
+%0A%09%7D else %7B%0A%09%09console.info('separate ');%0A%09%09var tags = uniquetags(process.argv%5B2%5D, process.argv%5B3%5D, process.argv%5B4%5D);%0A%09%09console.info('unique tags ', tags);%09%09%0A%09%09tags.map((t) =%3E %7B%0A%09%09%09main(process.argv%5B2%5D, process.argv.slice(3,5), %5Bt%5D).then((data) =%3E %7B%0A%09%09%09%09// write it !%0A%09%09%09%09var fname = '../mitm_out/qual-'+t+'.csv';%0A%09%09%09%09console.info(%22Writing output %22, t, ' ---%3E ', fname, data.length);%0A%09%09%09%09fs.writeFileSync(fname, data);%0A%09%09%09%09console.log('done');%0A%09%09%09%7D);%09%09%09%0A%09%09%7D);%0A%09%7D
 %0A%7D

tag unbundle can unbundle per file=