/*
 * Zed Attack Proxy (ZAP) and its related class files.
 * 
 * ZAP is an HTTP/HTTPS proxy for assessing web application security.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License. 
 * You may obtain a copy of the License at 
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0 
 *   
 * Unless required by applicable law or agreed to in writing, software 
 * distributed under the License is distributed on an "AS IS" BASIS, 
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 * See the License for the specific language governing permissions and 
 * limitations under the License. 
 */
package org.zaproxy.zap.spider.filters;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.httpclient.URIException;
import org.parosproxy.paros.network.HttpMessage;
import org.parosproxy.paros.network.HttpStatusCode;

/**
 * The DefaultParseFilter is an implementation of a {@link ParseFilter} that is default for
 * spidering process. Its filter rules are the following:<br/>
 * <ul>
 * <li>the resource body should be under MAX_RESPONSE_BODY_SIZE bytes, otherwise it's probably a
 * binary resource.</li>
 * <li>the resource must be of parsable type (text, html, xml, javascript). Actually, the content
 * type should be text/...</li>
 * </ul>
 */
public class DefaultParseFilter extends ParseFilter {

	/**
	 * The Constant MAX_RESPONSE_BODY_SIZE defining the size of response body that is considered too
	 * big for a parsable file.
	 */
	public static final int MAX_RESPONSE_BODY_SIZE = 512000;

	/**
	 * a pattern to match the SQLite based ".svn/wc.db" file name.
	 */
	private static final Pattern svnSQLiteFilenamePattern = Pattern.compile (".*/\\.svn/wc.db$");

	/**
	 * a pattern to match the XML based ".svn/entries" file name.
	 */
	private static final Pattern svnXMLFilenamePattern = Pattern.compile (".*/\\.svn/entries$");

	@Override
	public boolean isFiltered(HttpMessage responseMessage) {

		// Check response body size
		if (responseMessage.getResponseBody().length() > MAX_RESPONSE_BODY_SIZE) {
			if (log.isDebugEnabled()) {
				log.debug("Resource too large: " + responseMessage.getRequestHeader().getURI());
			}
			return true;
		}

		// If it's a redirection, accept it, as the SpiderRedirectParser will process it
		if (HttpStatusCode.isRedirection(responseMessage.getResponseHeader().getStatusCode()))
			return false;

		//if it's a file ending in "/.svn/entries", or "/.svn/wc.db", the SVN Entries parser will process it (regardless of type)
		Matcher svnXMLFilenameMatcher, svnSQLiteFilenameMatcher;
		try {
			String fullfilename = responseMessage.getRequestHeader().getURI().getPath();
			svnSQLiteFilenameMatcher = svnSQLiteFilenamePattern.matcher(fullfilename);
			svnXMLFilenameMatcher = svnXMLFilenamePattern.matcher(fullfilename);
		} catch (URIException e) {
			log.error(e);
			return true;
		}				
		if ( svnSQLiteFilenameMatcher.find() || svnXMLFilenameMatcher.find() )
			return false;
		
		// Check response type.
		if (!responseMessage.getResponseHeader().isText()) {
			if (log.isDebugEnabled()) {
				log.debug("Resource is not text: " + responseMessage.getRequestHeader().getURI());
			}
			return true;
		}

		return false;
	}

}

@@ -1979,16 +1979,159 @@
 es$%22);%0A%0A
+%09/**%0A%09 * a pattern to match the Git index file.%0A%09 */%0A%09private static final Pattern gitFilenamePattern = Pattern.compile (%22.*/%5C%5C.git/index$%22);%0A%0A
 %09@Overri
@@ -2715,14 +2715,22 @@
 ies 
+or Git 
 parser
+s
  wil
@@ -2820,16 +2820,37 @@
 eMatcher
+, gitFilenameMatcher;
 ;%0A%09%09try 
@@ -2931,16 +2931,87 @@
 Path();%0A
+%09%09%09//handle null paths%0A%09%09%09if (fullfilename == null) fullfilename = %22%22;%0A
 %09%09%09svnSQ
@@ -3158,76 +3158,77 @@
 ;%0A%09%09
-%7D catch (URIException e) %7B%0A%09%09%09log.error(e);%0A%09%09%09return true;%0A%09%09%7D%09
+%09gitFilenameMatcher = gitFilenamePattern.matcher(fullfilename);%0A
 %09%09%09%0A
+%09
 %09%09if
@@ -3294,18 +3294,47 @@
 .find() 
+%7C%7C gitFilenameMatcher.find()
 )%0A
+%09
 %09%09%09retur
@@ -3342,16 +3342,115 @@
  false;%0A
+%09%09%7D catch (URIException e) %7B%0A%09%09%09//give other parsers a chance to parse it.%0A%09%09%09log.error(e);%09%09%09%0A%09%09%7D%0A
 %09%09%0A%09%09// 

Add support for spidering using Git metadata (Index file, versions 2,3,4)