var _ = require('underscore');
var assert = require('assert');
var fs = require('fs');

var babyparse = require('babyparse');

function readCSV(filename){
  return babyparse.parse(fs.readFileSync(filename, 'utf8')).data;
};

function writeCSV(jsonCSV, filename){
  fs.writeFileSync(filename, babyparse.unparse(jsonCSV) + '\n');
}

function appendCSV(jsonCSV, filename){
  fs.appendFileSync(filename, babyparse.unparse(jsonCSV) + '\n');
}

var getRelevantLabels = function(object, tax) {
  var relevantLabels = _.keys(_.omit(tax, function(value, key, tax) {
    return !_.has(value, object);
  }));
  return relevantLabels;
};

var buildTax = function(knowledge, labels, responses) {
  // Add knowledge about labels to taxonomy
  var tax = {};

  for (var labelId = 0; labelId < labels.length; labelId++) {
    var probObj = {};
    var label = labels[labelId];
    var labelMatches = _.filter(knowledge, function(obj) {return obj.label == label;});
    _.forEach(labelMatches, function(obj) {
      probObj = _.extend(probObj, _.object([obj.response], [obj.prop]));
    });
    tax = _.extend(tax, _.object([label],[probObj]));
  }
  
  // Add knowledge about responses to taxonomy
  for (var responseId = 0; responseId < responses.length; responseId++) {
    var response = responses[responseId];
    tax = _.extend(tax, _.object([response], [_.object([response], [1])]));
  }

  return tax;
};

var buildKnowledge = function(type, domain) {
  // use appropriate knowledge
  var unifKnowledge = require("./saliencyKnowledgeUniform.json");
  var empKnowledge = require("./saliencyKnowledgeEmpirical.json");
  var filterFunc = function(obj) {
    return obj.type === type && obj.domain === domain;
  };
  var relevantUnifKnowledge = _.filter(unifKnowledge, filterFunc);
  var relevantEmpKnowledge = _.filter(empKnowledge, filterFunc);
  var relevantResponses = _.unique(_.pluck(relevantEmpKnowledge, 'response'));
  var relevantLabels =  _.unique(_.pluck(relevantEmpKnowledge, 'label'));
  var unifTax = buildTax(relevantUnifKnowledge, relevantLabels, relevantResponses);
  var empTax = buildTax(relevantEmpKnowledge, relevantLabels, relevantResponses);

  return {
    unifTaxonomy : unifTax,
    objectSpace : relevantResponses,
    labelSpace : relevantLabels
  };
};

module.exports = {
  buildKnowledge : buildKnowledge,
  getRelevantLabels : getRelevantLabels
};

@@ -440,186 +440,1166 @@
 var 
-getRelevantLabels = function(object, tax) %7B%0A  var relevantLabels = _.keys(_.omit(tax, function(value, key, tax) %7B%0A    return !_.has(value, object);%0A  %7D));%0A  return relevantLabels
+writeERP = function(erp, labels, filename, fixed) %7B%0A  var data = _.filter(erp.support().map(%0A   function(v) %7B%0A     var prob = Math.exp(erp.score(%5B%5D, v));%0A     if (prob %3E 0.0)%7B%0A      if(v.slice(-1) === %22.%22)%0A        out = butLast(v);%0A      else if (v.slice(-1) === %22?%22)%0A        out = butLast(v).split(%22Is%22)%5B1%5D.toLowerCase();%0A      else %0A        out = v%0A      return labels.concat(%5Bout, String(prob.toFixed(fixed))%5D);%0A%0A    %7D else %7B%0A      return %5B%5D;%0A    %7D%0A  %7D%0A  ), function(v) %7Breturn v.length %3E 0;%7D);%0A  appendCSV(data, filename);%0A%7D;%0A%0Avar getRelevantLabels = function(object, tax) %7B%0A  var relevantLabels = _.keys(_.omit(tax, function(value, key, tax) %7B%0A    return !_.has(value, object);%0A  %7D));%0A  return relevantLabels;%0A%7D;%0A%0Avar getWordFrequency = function(label) %7B%0A  var locParse = function(filename) %7B%0A    return babyparse.parse(fs.readFileSync(filename, 'utf8'),%0A%09%09%09   %7Bheader: true%7D).data;%0A  %7D;%0A  var frequencyData = locParse(%22../../experiments/4_numdistractors_basiclevel_newitems/%22%0A%09%09%09       + %22results/data/frequencyChart.csv%22);%0A  var relevantRow = _.filter(frequencyData, function(row) %7B%0A    return row.noun === label;%0A  %7D);%0A  return relevantRow%5B0%5D.relFreq
 ;%0A%7D;
@@ -3298,16 +3298,101 @@
 wledge,%0A
+  getWordFrequency : getWordFrequency,%0A  writeERP : writeERP,%0A  writeCSV : writeCSV,%0A
   getRel

Add utils for getting frequency, writing erps