package de.jeha.kame.crawler.robots;

/**
 * @author jenshadlich@googlemail.com
 */
public final class Constants {

    public static final String ROBOTS_FILE = "/robots.txt";

    public static final String USER_AGENT = "user-agent"; // field "User-agent", lower case
    public static final String DISALLOW = "disallow"; // field "Disallow", lower case

    // currently unsupported nonstandard extensions:
    // Crawl-delay
    // Allow
    // Sitemap

    public static final String COMMENT_DELIM = "#";
    public static final String FIELD_DELIM = ":";

    public static final String ROOT_PATH = "/";
}

@@ -249,28 +249,16 @@
 r-agent%22
-, lower case
 %0A    pub
@@ -327,20 +327,8 @@
 low%22
-, lower case
 %0A%0A  
@@ -386,50 +386,222 @@
     
-// Crawl-delay%0A    // Allow%0A    // 
+public static final String ALLOW = %22allow%22; // field %22Allow%22%0A    public static final String CRAWL_DELAY = %22crawl-delay%22; // field %22Crawl-delay%22%0A    public static final String SITEMAP = %22sitemap%22; // field %22
 Sitemap
+%22
 %0A%0A  

add constants for nonstandard extensions