package de.jungblut.crawl.extraction;

import com.google.common.collect.Sets;
import de.jungblut.crawl.FetchResult;
import org.junit.Test;

import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Method;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Iterator;
import java.util.TreeSet;
import java.util.regex.Pattern;

import static org.junit.Assert.*;

public class OutlinkExtractorTest {

    private static final String HOME = "http://people.apache.org/~tjungblut/downloads/test.html";
    private static final String HOME2 = "http://people.apache.org/~tjungblut/downloads/test2.html";

    @Test
    public void testConsumeStream() throws Exception {
        String res = "<html><head><title>Tutorial:HelloWorld</title></head><body>"
                + "<h1>HelloWorldTutorial</h1></body></html>";

        String consumeStream = OutlinkExtractor.consumeStream(
                new URL(HOME).openStream()).replaceAll("\\s+", "");
        assertEquals(res, consumeStream);
    }

    @Test
    public void testGetConnection() throws IOException {
        InputStream connection = OutlinkExtractor.getConnection(HOME);
        assertNotNull(connection);
        connection.close();
    }

    @Test
    public void testExtractBaseUrl() throws IOException {
        String connection = OutlinkExtractor.extractBaseUrl(HOME);
        assertEquals("http://people.apache.org", connection);

        connection = OutlinkExtractor
                .extractBaseUrl("http://www.berliner-kurier.de/LOL123/omg");
        assertEquals("http://www.berliner-kurier.de", connection);

        connection = OutlinkExtractor
                .extractBaseUrl("http://www.spiegel.de/LOL123/omg");
        assertEquals("http://www.spiegel.de", connection);

        connection = OutlinkExtractor
                .extractBaseUrl("http://www.subdomain.google.de/LOL123/omg");
        assertEquals("http://www.subdomain.google.de", connection);
    }

    @Test
    public void testIsValid() throws Exception {
        assertTrue(OutlinkExtractor.isValid(HOME));
        assertFalse(OutlinkExtractor.isValid(HOME + ".png"));
        assertTrue(OutlinkExtractor
                .isValid("http://www.subdomain.google.de/LOL123/omg"));
        assertTrue(OutlinkExtractor
                .isValid("https://www.subdomain.google.de/LOL123/omg"));
        assertTrue(OutlinkExtractor
                .isValid("https://www.subdomain.google.de/LOL123/omg"));
        assertTrue(OutlinkExtractor
                .isValid("https://www.subdomain-google.de/LOL123/omg"));
        assertFalse(OutlinkExtractor.isValid("http://news.google.de/news/void(0)"));
        assertFalse(OutlinkExtractor.isValid("http://news.google.de/news/void()"));
        assertFalse(OutlinkExtractor.isValid("void(0)"));
        assertFalse(OutlinkExtractor.isValid("void(0);"));
        assertFalse(OutlinkExtractor
                .isValid("http://www.golem.de/news/news@golem.de"));
        assertTrue(OutlinkExtractor
                .isValid("http://handyattacke.de/cgi/websale6.cgi?shopid=etronixx&subshopid=attacke&act=load_tpl&tpl=inc_agbtext_handyattacke.htm"));

    }

    @Test
    public void testFilter() throws Exception {
        HashSet<String> set = Sets.newHashSet("  ", "\n", "\t", "asdgg");
        HashSet<String> filter = OutlinkExtractor.filter(set,
                Pattern.compile("\\s+"));
        assertEquals(3, filter.size());
    }

    @Test
    public void testExtraction() throws Exception {
        OutlinkExtractor mock = new OutlinkExtractor();
        FetchResult extract = mock.extract(HOME2);
        assertEquals(HOME2, extract.getUrl());
        assertEquals(4, extract.getOutlinks().size());
        TreeSet<String> sorted = new TreeSet<>(extract.getOutlinks());
        Iterator<String> it = sorted.iterator();
        assertEquals("http://people.apache.org/local.html", it.next());
        assertEquals("http://people.apache.org/~tjungblut/downloads/local.html",
                it.next());
        // that is the correct html expansion
        assertEquals(
                "http://people.apache.org/~tjungblut/downloads/www.testlol.de/local.html",
                it.next());
        assertEquals("http://www.logs.de/local.html", it.next());

    }

    @Test
    public void testStreamResize() throws Exception {
        // should have a look into some mocking framework :/
        Method resizeMethod = OutlinkExtractor.class.getDeclaredMethod(
                "resizeBuffer", ByteBuffer.class);
        resizeMethod.setAccessible(true);
        ByteBuffer buf = ByteBuffer.allocate(100);
        for (int i = 0; i < 80; i++) {
            buf.put((byte) i);
        }
        buf = (ByteBuffer) resizeMethod.invoke(null, buf);
        assertEquals(100, buf.limit());
        assertEquals(20, buf.remaining());

        // now fill until the 10% threshold and see if the buffer grew
        for (int i = 0; i < 11; i++) {
            buf.put((byte) i);
        }
        buf = (ByteBuffer) resizeMethod.invoke(null, buf);
        assertEquals(200, buf.limit());
        assertEquals(109, buf.remaining());
    }

}

@@ -82,118 +82,24 @@
 ort 
-de.jungblut.crawl.FetchResult;%0Aimport org.junit.Test;%0A%0Aimport java.io.IOException;%0Aimport java.io.InputStream;
+org.junit.Test;%0A
 %0Aimp
@@ -132,29 +132,8 @@
 od;%0A
-import java.net.URL;%0A
 impo
@@ -186,61 +186,8 @@
 et;%0A
-import java.util.Iterator;%0Aimport java.util.TreeSet;%0A
 impo
@@ -388,708 +388,8 @@
 ml%22;
-%0A    private static final String HOME2 = %22http://people.apache.org/~tjungblut/downloads/test2.html%22;%0A%0A    @Test%0A    public void testConsumeStream() throws Exception %7B%0A        String res = %22%3Chtml%3E%3Chead%3E%3Ctitle%3ETutorial:HelloWorld%3C/title%3E%3C/head%3E%3Cbody%3E%22%0A                + %22%3Ch1%3EHelloWorldTutorial%3C/h1%3E%3C/body%3E%3C/html%3E%22;%0A%0A        String consumeStream = OutlinkExtractor.consumeStream(%0A                new URL(HOME).openStream()).replaceAll(%22%5C%5Cs+%22, %22%22);%0A        assertEquals(res, consumeStream);%0A    %7D%0A%0A    @Test%0A    public void testGetConnection() throws IOException %7B%0A        InputStream connection = OutlinkExtractor.getConnection(HOME);%0A        assertNotNull(connection);%0A        connection.close();%0A    %7D
 %0A%0A  
@@ -436,27 +436,8 @@
 rl()
- throws IOException
  %7B%0A 
@@ -1145,33 +1145,16 @@
 sValid()
- throws Exception
  %7B%0A     
@@ -2321,33 +2321,16 @@
 Filter()
- throws Exception
  %7B%0A     
@@ -2553,841 +2553,8 @@
  %7D%0A%0A
-    @Test%0A    public void testExtraction() throws Exception %7B%0A        OutlinkExtractor mock = new OutlinkExtractor();%0A        FetchResult extract = mock.extract(HOME2);%0A        assertEquals(HOME2, extract.getUrl());%0A        assertEquals(4, extract.getOutlinks().size());%0A        TreeSet%3CString%3E sorted = new TreeSet%3C%3E(extract.getOutlinks());%0A        Iterator%3CString%3E it = sorted.iterator();%0A        assertEquals(%22http://people.apache.org/local.html%22, it.next());%0A        assertEquals(%22http://people.apache.org/~tjungblut/downloads/local.html%22,%0A                it.next());%0A        // that is the correct html expansion%0A        assertEquals(%0A                %22http://people.apache.org/~tjungblut/downloads/www.testlol.de/local.html%22,%0A                it.next());%0A        assertEquals(%22http://www.logs.de/local.html%22, it.next());%0A%0A    %7D%0A%0A
     

remove tests depending on some web resource