package de.jeha.kame.crawler.robots;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Parser for robots.txt files. Currently limited to the original standard which means it does only support User-agent
 * and Disallow instructions (no Allow, Crawl-Delay or Sitemap).
 *
 * See http://www.robotstxt.org/
 *
 * @author jenshadlich@googlemail.com
 */
public class RobotsExclusionParser {

    private static final Logger LOG = LoggerFactory.getLogger(RobotsExclusionParser.class);

    /**
     * Parse a given robots.txt file given as InputStream.
     *
     * @param robotsFileInputStream robots.txt
     * @return robots.txt object model to query
     * @throws IOException
     */
    public RobotsExclusion parse(InputStream robotsFileInputStream) throws IOException {
        List<String> lines = IOUtils.readLines(robotsFileInputStream);

        Map<String, List<String>> disallowMap = new HashMap<>();
        int lineCounter = 0;
        String currentUserAgent = null;

        for (String line : lines) {
            ++lineCounter;
            final String trimmedLine = line.trim();
            if (trimmedLine.startsWith(Constants.COMMENT_DELIM)) {
                continue;
            }
            final String trimmedLineWithoutComments = StringUtils.substringBefore(trimmedLine, Constants.COMMENT_DELIM);

            if (trimmedLineWithoutComments.isEmpty()) {
                continue;
            }

            String parts[] = trimmedLineWithoutComments.split(Constants.FIELD_DELIM);
            if (parts.length < 1 || parts.length > 2) {
                LOG.warn("skip line {}, illegal format: '{}'", lineCounter, line);
                continue;
            }
            final String field = parts[0].trim().toLowerCase();
            final String value = (parts.length > 1) ? parts[1].trim() : "";

            switch (field) {
                case Constants.USER_AGENT:
                    currentUserAgent = value.toLowerCase();
                    break;
                case Constants.DISALLOW:
                    if (currentUserAgent == null) {
                        LOG.warn("skip line {}, " + Constants.DISALLOW + " without prior " + Constants.USER_AGENT, lineCounter);
                    } else {
                        if (!disallowMap.containsKey(currentUserAgent)) {
                            disallowMap.put(currentUserAgent, new ArrayList<>());
                        }
                        disallowMap.get(currentUserAgent).add(value);
                    }
                    break;
                default:
                    LOG.info("skip line {}, unsupported instruction: '{}'", lineCounter, line);
                    break;
            }
        }
        return new RobotsExclusion(disallowMap);
    }
}

@@ -2806,72 +2806,313 @@
     
-default:%0A                    LOG.info(%22skip line %7B%7D, unsupported
+case Constants.ALLOW:%0A                case Constants.CRAWL_DELAY:%0A                case Constants.SITEMAP:%0A                    LOG.info(%22skip line %7B%7D, unsupported instruction: '%7B%7D'%22, lineCounter, line);%0A                    break;%0A                default:%0A                    LOG.warn(%22skip line %7B%7D, unknown
  ins

improve logging