//Gets links and writes dump of profile data to file
//
// import modules.
var fs = require('fs');
require('./ProjectClass.js');
//var p = require('./parseProjDatafromJSON.js');
var scrape = require('./profileScrape.js');
var request = require("request");

function writeToFile(data) {
    fs.writeFile('rawProfileContent2.txt', data, function () {
        console.log('file written!!!!!!!!');
    })
}
// import function to scrape profile
var GetProfileLinks = require('./GetProfileLinks_FromDump.js');
// all profile urls consist of this url
var baseURL = 'https://www.lendwithcare.org/';
// calls function from getlinksfromdump module that parses JSON object and returns array of profile links
var linksArr = GetProfileLinks.getProfileLinks;
// t termporarily stoes data for each scrape to dump to file
var t = [];

//count responses received
var responseCount = 0;
//scrape profiles
for (var i = 0; i < 40; i++) {
    //concatenate base url with profile link url to give request url
    var urlCur = baseURL + linksArr[i];

    request(urlCur, function(error, response, body){
            t.push(body);
            // increment response counter to keep track of responses received
            if(response){responseCount++}
           // this should be set to (last loop value -1) to ensure file is written after
          // all responses have been received
            if(responseCount === 39){writeToFile(t)}
        });
}



@@ -68,16 +68,46 @@
 odules.%0A
+var async = require(%22async%22);%0A
 var fs =
@@ -152,18 +152,16 @@
 s.js');%0A
-//
 var p = 
@@ -192,19 +192,16 @@
 fromJSON
-.js
 ');%0Avar 
@@ -201,21 +201,20 @@
 );%0Avar s
-crape
+plit
  = requi
@@ -223,21 +223,24 @@
 ('./
-profileScrape
+SplitTxtEachProj
 .js'
@@ -277,16 +277,43 @@
 est%22);%0A%0A
+function d(callback) %7B%0A    
 function
@@ -337,16 +337,20 @@
 ) %7B%0A    
+   
+ 
 fs.write
@@ -408,16 +408,20 @@
         
+    
 console.
@@ -457,12 +457,70 @@
     
-%7D)%0A%7D
+        //split.split();%0A        callback();%0A        %7D)%0A    %7D%0A
 %0A// 
@@ -549,24 +549,28 @@
 ape profile%0A
+    
 var GetProfi
@@ -661,16 +661,20 @@
 his url%0A
+    
 var base
@@ -818,16 +818,20 @@
 e links%0A
+    
 var link
@@ -931,16 +931,20 @@
 to file%0A
+    
 var t = 
@@ -975,16 +975,20 @@
 eceived%0A
+    
 var resp
@@ -1020,16 +1020,20 @@
 rofiles%0A
+    
 for (var
@@ -1059,16 +1059,20 @@
 ) %7B%0A    
+ 
+   
 //concat
@@ -1124,24 +1124,28 @@
 request url%0A
+    
     var urlC
@@ -1177,16 +1177,20 @@
 %5D;%0A%0A    
+  
+  
 request(
@@ -1205,16 +1205,17 @@
 function
+ 
 (error, 
@@ -1229,16 +1229,17 @@
 e, body)
+ 
 %7B%0A      
@@ -1342,32 +1342,33 @@
 d%0A            if
+ 
 (response)%7Brespo
@@ -1361,17 +1361,35 @@
 esponse)
-%7B
+ %7B%0A                
 response
@@ -1395,18 +1395,32 @@
 eCount++
+%0A            
 %7D%0A
+ 
         
@@ -1510,16 +1510,18 @@
         
+  
 // all r
@@ -1562,16 +1562,17 @@
       if
+ 
 (respons
@@ -1589,37 +1589,1095 @@
  39)
-%7BwriteToFile(t)%7D%0A        %7D);%0A%7D%0A
+ %7B%0A                writeToFile(t, callback);%0A%0A            %7D%0A        %7D);%0A    %7D%0A%0A%7D%0Aasync.series(%5B%0A        function tom(callback)%7B%0A            // get links from casper links collection and scrape each profile then dump all contents%0A            console.log('step one!');%0A            d(function()%7B%0A                console.log('ok done - now  run alis');%0A                callback(null, 'two');%7D);%0A        %7D,%0A        function alis(callback)%7B%0A            // split files from dumped contents to individual projs and write to individualprojectsRaw.json%0A            console.log('step two! - alis run');%0A            split.split ( callback(null , 'two'));%0A        %7D,%0A        function parsey(callback)%7B%0A            // get links from casper links collection and scrape each profile then dump all contents%0A            console.log('step three!');%0A           p.parsey(function()%7B%0A                console.log('ok final parse and write done');%0A                callback(null, 'two');%7D);%0A        %7D%0A    %5D,%0A// optional callback%0A    function(err, results)%7B%0A        // results is now equal to %5B'one', 'two'%5D%0A    %7D);
 %0A%0A

Update Main.js