package awesomefb;

import awesomefb.facebook.*;
import org.json.JSONArray;
import org.json.JSONObject;

import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.stream.Collectors;

/**
 * Created by earl on 5/25/2015.
 */
public class FacebookFocusedCrawler {
    private Database mDatabase;

    public FacebookFocusedCrawler() {
        mDatabase = Database.getInstance();
    }

    public void run() {
        //Facebook facebook = Facebook.getInstance();
        //facebook.login();

        // List of pages to crawl
        Queue<Page> queue = new LinkedList<Page>();
        queue.addAll(Facebook.getInstance().searchPages("iphone"));

        // List of pages crawled
        List<String> crawledPages = new ArrayList<String>();

        while (true) {
            // Get page from queue
            Page page = queue.remove();
            String pageId = page.getFacebookId();

            // If that page is not processed
            if (!crawledPages.contains(pageId)) {
                System.out.println("[awesomeFb] Processing page " + pageId);
                List<Page> pageLikes = page.getLikes();
                queue.addAll(pageLikes.stream().filter(like -> !crawledPages.contains(like.getFacebookId()))
                        .collect(Collectors.toList()));
                // Get feed as JSON array
                JSONArray feed = page.getPosts();

                if (feed != null) {
                    int count = 0;
                    for (int i = 0; i < feed.length(); i++) {
                        // Extract each post's data from feed
                        JSONObject pageObject = feed.getJSONObject(i);
                        // Skip if post does not contain message
                        if (!pageObject.has("message")) continue;

                        Post post = new Post(pageObject);
                        // Save post data to database
                        mDatabase.insertComment(post);

                        User postCreator = post.getCreator();
                        mDatabase.insertUser(postCreator);

                        // Get list of comments as JSON array
                        List<Comment> comments = post.getComments();
                        if (comments != null) {
                            for (Comment comment : comments) {
                                User commentCreator = comment.getCreator();
                                // If comment creator is page, add it to queue
                                if (!commentCreator.isPage()) {
                                    //commentCreator = facebook.updateUserDetails(commentCreator);
                                }
                                mDatabase.insertComment(comment);
                                mDatabase.insertUser(commentCreator);
                            }
                        }

                        count++;
                    }
                    System.out.println("[awesomeFb] " + count + " posts processed.");
                }

                // Mark page id as processed
                crawledPages.add(pageId);
            }

            if (queue.isEmpty()) {
                break;
            }
        }
    }
}

@@ -460,24 +460,62 @@
 oid run() %7B%0A
+        final boolean RESET = false;%0A%0A
         //Fa
@@ -580,25 +580,142 @@
 ok.login();%0A
-%0A
+        Queue%3CPage%3E queue;%0A        List%3CString%3E crawledPages;%0A        if (RESET) %7B%0A            mDatabase.drop();%0A%0A    
         // L
@@ -740,35 +740,27 @@
 awl%0A        
-Queue%3CPage%3E
+   
  queue = new
@@ -788,29 +788,45 @@
         
-queue.addAll(
+    List%3CPage%3E resultPages = 
 Facebook
@@ -861,20 +861,176 @@
 iphone%22)
-);%0A%0A
+;%0A            for (Page resultPage: resultPages) %7B%0A                queue.add(resultPage);%0A                mDatabase.insertQueue(resultPage);%0A            %7D%0A%0A    
         
@@ -1058,36 +1058,27 @@
 led%0A        
-List%3CString%3E
+   
  crawledPage
@@ -1105,16 +1105,183 @@
 ring%3E();
+%0A        %7D else %7B%0A            // Continue from last checkpoint%0A            queue = mDatabase.getQueue();%0A            crawledPages = mDatabase.getProcessed();%0A        %7D
 %0A%0A      
@@ -1368,24 +1368,69 @@
 e.remove();%0A
+            mDatabase.removeFromQueue(page);%0A
             
@@ -1716,55 +1716,65 @@
     
-queue.addAll(pageLikes.stream().filter(like -%3E 
+for (Page pageLike: pageLikes) %7B%0A                    if (
 !cra
@@ -1792,17 +1792,21 @@
 ontains(
-l
+pageL
 ike.getF
@@ -1818,16 +1818,18 @@
 okId()))
+ %7B
 %0A       
@@ -1849,39 +1849,126 @@
     
-.collect(Collectors.toList()));
+queue.add(pageLike);%0A                        mDatabase.insertQueue(pageLike);%0A                    %7D%0A                %7D%0A
 %0A   
@@ -3771,32 +3771,82 @@
 es.add(pageId);%0A
+                mDatabase.insertProcesed(pageId);%0A
             %7D%0A%0A 

Save proccessed and queue state