@@ -0,0 +1,2076 @@
+from __future__ import absolute_import%0A%0Aimport json%0Aimport logging%0Aimport functools%0Afrom datetime import datetime%0A%0Aimport requests%0Aimport cqlengine%0Afrom cqlengine import columns%0Afrom cqlengine import management%0Afrom cassandra.cluster import NoHostAvailable%0A%0Afrom scrapi import settings%0A%0A%0Alogger = logging.getLogger(__name__)%0A%0A%0Atry:%0A    cqlengine.connection.setup(settings.CASSANDRA_URI, settings.CASSANDRA_KEYSPACE)%0A    management.create_keyspace(settings.CASSANDRA_KEYSPACE, replication_factor=1, strategy_class='SimpleStrategy')%0Aexcept NoHostAvailable:%0A    logger.error('Could not connect to Cassandra, expect errors.')%0A    if settings.RECORD_HTTP_TRANSACTIONS:%0A        raise%0A%0A%0Aclass HarvesterResponse(cqlengine.Model):%0A    __table_name__ = 'responses'%0A    __keyspace__ = settings.CASSANDRA_KEYSPACE%0A%0A    method = columns.Text(primary_key=True)%0A    url = columns.Text(primary_key=True, required=True, index=True)%0A%0A    # Raw request data%0A    content = columns.Bytes()%0A    headers_str = columns.Text()%0A    time_made = columns.DateTime(default=datetime.now)%0A%0A    @property%0A    def json(self):%0A        return json.loads(self.content)%0A%0A    @property%0A    def headers(self):%0A        # TODO: make case insensitive multidict%0A        return json.loads(self.headers_str)%0A%0A%0Adef record_or_load_response(method, url, **kwargs):%0A    try:%0A        return HarvesterResponse.get(url=url, method=method)%0A    except HarvesterResponse.DoesNotExist:%0A        response = requests.request(method, url, **kwargs)%0A        return HarvesterResponse(%0A            url=url,%0A            method=method,%0A            content=response.content%0A        ).save()%0A%0A%0Adef request(method, url, **kwargs):%0A    if settings.RECORD_HTTP_TRANSACTIONS:%0A        return record_or_load_response(method, url)%0A    return requests.request(method, url, **kwargs)%0A%0A%0A# This has to be done after HarvesterResponse definition%0Amanagement.sync_table(HarvesterResponse)%0A%0Aget = functools.partial(request, 'get')%0Aput = functools.partial(request, 'put')%0Apost = functools.partial(request, 'post')%0Adelete = functools.partial(request, 'delete')%0A

Add scrapi requests recorder