@@ -0,0 +1,3307 @@
+package de.jeha.kame.crawler.robots;%0A%0Aimport org.junit.Ignore;%0Aimport org.junit.Test;%0A%0Aimport java.io.IOException;%0Aimport java.io.InputStream;%0A%0Aimport static org.junit.Assert.assertFalse;%0Aimport static org.junit.Assert.assertTrue;%0A%0A/**%0A * @author jenshadlich@googlemail.com%0A */%0Apublic class RobotsExclusionParserTest %7B%0A%0A    @Test%0A    public void allRobotsAllowEverything() throws IOException %7B%0A        RobotsExclusion result =%0A                new RobotsExclusionParser().parse(getInputStreamForTest(%22all_robots_allow_everything_robots.txt%22));%0A%0A        assertTrue(result.allowed(UserAgents.ANY, Constants.ROOT_PATH));%0A        assertTrue(result.allowed(%22BadBot%22, %22/%22));%0A    %7D%0A%0A    @Test%0A    public void allRobotsAllowNothing() throws IOException %7B%0A        RobotsExclusion result =%0A                new RobotsExclusionParser().parse(getInputStreamForTest(%22all_robots_allow_nothing_robots.txt%22));%0A%0A        assertFalse(result.allowed(UserAgents.ANY, %22/%22));%0A        assertTrue(result.disallowed(UserAgents.ANY, %22/%22));%0A    %7D%0A%0A    @Test%0A    public void allRobotsDisallowSpecificDirectory() throws IOException %7B%0A        RobotsExclusion result =%0A                new RobotsExclusionParser().parse(getInputStreamForTest(%22all_robots_disallow_specific_directory_robots.txt%22));%0A%0A        // only this file is disallowed for all robots%0A        assertTrue(result.disallowed(UserAgents.ANY, %22/disallowed/%22));%0A        assertTrue(result.disallowed(UserAgents.ANY, %22/disallowed/subdir/%22));%0A        assertFalse(result.allowed(%22BadBot%22, %22/disallowed/%22));%0A        assertFalse(result.allowed(%22BadBot%22, %22/disallowed/file.html%22));%0A%0A        // everything else is allowed%0A        assertTrue(result.allowed(UserAgents.ANY, %22/%22));%0A        assertTrue(result.allowed(UserAgents.ANY, %22/foo%22));%0A        assertTrue(result.allowed(UserAgents.ANY, %22/foo/bar%22));%0A        assertTrue(result.allowed(%22BadBot%22, %22/%22));%0A    %7D%0A%0A    @Test%0A    public void allRobotsDisallowSpecificFile() throws IOException %7B%0A        RobotsExclusion result =%0A                new RobotsExclusionParser().parse(getInputStreamForTest(%22all_robots_disallow_specific_file_robots.txt%22));%0A%0A        // only this file is disallowed for all robots%0A        assertTrue(result.disallowed(UserAgents.ANY, %22/d/file.html%22));%0A        assertFalse(result.allowed(%22BadBot%22, %22/d/file.html%22));%0A%0A        // everything else is allowed%0A        assertTrue(result.allowed(UserAgents.ANY, %22/%22));%0A        assertTrue(result.allowed(%22BadBot%22, %22/%22));%0A    %7D%0A%0A    @Test%0A    public void stackoverflowRobots() throws IOException %7B%0A        RobotsExclusion result =%0A                new RobotsExclusionParser().parse(getInputStreamForTest(%22live-examples/stackoverflow.com_robots.txt%22));%0A%0A        assertTrue(result.disallowed(%22Slurp%22, %22/%22));%0A        assertTrue(result.disallowed(%22Slurp%22, %22/foobar%22));%0A        assertTrue(result.disallowed(%22Slurp%22, %22/foobar/%22));%0A%0A        assertTrue(result.allowed(%22Googlebot%22, %22/%22));%0A        assertTrue(result.allowed(%22Googlebot%22, %22/foobar%22));%0A        assertTrue(result.allowed(%22Googlebot%22, %22/foobar/%22));%0A    %7D%0A%0A    // -----------------------------------------------------------------------------------------------------------------%0A%0A    private InputStream getInputStreamForTest(String resource) %7B%0A        return this.getClass().getResourceAsStream(%22/robots/%22 + resource);%0A    %7D%0A%0A%7D%0A

attach test cases