#!/usr/bin/python
from pony.orm import *
from datetime import *
from tor_db import *
import sys 
import urllib2
from urllib2 import URLError
import random
import string
import socket

@db_session
def get_domains():
	domains = select(d for d in Domain if d.is_up == True and d.useful_404_scanned_at == NEVER)
	return domains


@db_session 
def scan_404():
	domains = get_domains()
	print("processing %d domains" % count(domains))
	i = 0
	for d in domains:
		i += 1
		r = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase) for _ in range(12))
		url = d.index_url() + r
		code = 0
		try:
			res = urllib2.urlopen(url, None, 60)
			code = int(res.getcode())		
		except urllib2.HTTPError, e:
			code = int(e.code)		
		except (URLError, socket.timeout) as e:
			print("#%d failed (%s)" % (i, url))
			continue
		
		if code in [502, 503]:
			print("#%d failed (%s)" % (i, url))
			continue
		print("#%d tested %s and got %d" % (i, url, code))
		if code == 404:
			d.useful_404 = True
		else:
			d.useful_404 = False	
		d.useful_404_scanned_at = datetime.now()
		commit()

scan_404()
sys.exit(0)
@@ -175,16 +175,27 @@
 t socket
+%0Aimport ssl
 %0A%0A@db_se
@@ -355,24 +355,121 @@
 scan_404():%0A
+%09ctx = ssl.create_default_context()%0A%09ctx.check_hostname = False%0A%09ctx.verify_mode = ssl.CERT_NONE%0A
 %09domains = g
@@ -744,16 +744,29 @@
 None, 60
+, context=ctx
 )%0A%09%09%09cod
@@ -841,24 +841,24 @@
 t(e.code)%09%09%0A
-
 %09%09except (UR
@@ -879,16 +879,38 @@
 .timeout
+, ssl.CertificateError
 ) as e:%0A

handle ssl cert errors in 404 detection