Fixed check for robots.txt
Fixed check for robots.txt

git-svn-id: 0ccecee31fc8c854a64f1f5a24824e2d34be7a6b@496 869247f0-a1f3-4d5c-9449-4b73fc252fa6

justcarakas/forkcms,jessedobbelaere/forkcms,riadvice/forkcms,tommyvdv/forkcms,vytenizs/forkcms,sumocoders/forkcms,jessedobbelaere/forkcms,vytsci/forkcms,sumocoders/forkcms,Katrienvh/forkcms,vytenizs/forkcms,mathiashelin/forkcms,gertjanmeire/davy_portfolio,Thijzer/forkcms,jessedobbelaere/forkcms,Katrienvh/forkcms,riadvice/forkcms,bartdc/forkcms,forkcms/forkcms,lowiebenoot/forkcms,bartdc/forkcms,ocpyosep78/forkcms,wengyulin/forkcms,carakas/forkcms,jonasgoderis/jonaz,dmoerman/forkcms,riadvice/forkcms,sumocoders/forkcms,nosnickid/forkcms,WouterSioen/forkcms,jonasdekeukelaere/forkcms,justcarakas/forkcms,carakas/forkcms,matthiasmullie/forkcms,vytsci/forkcms,ikoene/forkcms,DegradationOfMankind/forkcms,DegradationOfMankind/forkcms,wengyulin/forkcms,ikoene/forkcms,forkcms/forkcms,dmoerman/forkcms,jeroendesloovere/forkcms,mathiashelin/forkcms,jacob-v-dam/forkcms,lowiebenoot/forkcms,Katrienvh/forkcms,nosnickid/forkcms,justcarakas/forkcms,WouterSioen/forkcms,jessedobbelaere/forkcms,jeroendesloovere/forkcms,jonasdekeukelaere/forkcms,matthiasmullie/forkcms,WouterSioen/daytwo,jonasdekeukelaere/forkcms,mathiashelin/forkcms,Thijzer/forkcms,dmoerman/forkcms,carakas/forkcms,DegradationOfMankind/forkcms,Thijzer/forkcms,mathiashelin/forkcms,wengyulin/forkcms,Katrienvh/forkcms,ocpyosep78/forkcms,forkcms/forkcms,jacob-v-dam/forkcms,DegradationOfMankind/forkcms,ikoene/forkcms,tommyvdv/forkcms,jonasgoderis/jonaz,vytenizs/forkcms,gertjanmeire/davy_portfolio,jacob-v-dam/forkcms,sumocoders/forkcms,tommyvdv/forkcms,tommyvdv/forkcms,jonasgoderis/jonaz,jacob-v-dam/forkcms,forkcms/forkcms,jeroendesloovere/forkcms,nosnickid/forkcms,Saxithon/gh-pages,jonasdekeukelaere/forkcms,carakas/forkcms,jonasdekeukelaere/forkcms,riadvice/forkcms,matthiasmullie/forkcms,bartdc/forkcms,jeroendesloovere/forkcms,Saxithon/gh-pages,jonasgoderis/jonaz,jonasgoderis/jonaz,sumocoders/forkcms,vytsci/forkcms,carakas/forkcms