import logging
from datetime import timedelta
from core.errors import ObservableValidationError
from core.feed import Feed
from core.observables import Ip


class DataplaneDNSRecursive(Feed):

    default_values = {
        "frequency": timedelta(hours=2),
        "name": "DataplaneDNSRecursive",
        "source": "https://dataplane.org/dnsrd.txt",
        "description": "Entries below are records of source IP addresses that have been identified as sending recursive DNS queries.",
    }

    def update(self):
        resp = self._make_request(sort=False)
        lines = resp.content.decode("utf-8").split("\n")[64:-5]
        for url in lines:
            self.analyze(url.strip())

    def analyze(self, line):
        val = line.split("|")[2].strip()

        context = {
            "source": self.name,
        }

        try:
            obs = Ip.get_or_create(value=val)
            obs.add_context(context)
            obs.add_source(self.name)
            obs.tag("dataplane")
            obs.tag("dns")
        except ObservableValidationError as e:
            raise logging.error(e)

@@ -8,16 +8,36 @@
 logging%0A
+import pandas as pd%0A
 from dat
@@ -167,16 +167,34 @@
 mport Ip
+, AutonomousSystem
 %0A%0A%0Aclass
@@ -668,62 +668,437 @@
     
-for url in lines:%0A            self.analyze(url.strip()
+columns = %5B%22ASN%22, %22ASname%22, %22ipaddr%22, %22lastseen%22, %22category%22%5D%0A        df = pd.DataFrame(%5Bl.split(%22%7C%22) for l in lines%5D, columns=columns)%0A%0A        for c in columns:%0A            df%5Bc%5D = df%5Bc%5D.str.strip()%0A        df = df.dropna()%0A        df%5B%22lastseen%22%5D = pd.to_datetime(df%5B%22lastseen%22%5D)%0A        if self.last_run:%0A            df = df%5Bdf%5B%22lastseen%22%5D %3E self.last_run%5D%0A        for count, row in df.iterrows():%0A            self.analyze(row
 )%0A%0A 
@@ -1122,23 +1122,23 @@
 lf, 
-line
+row
 ):%0A
+%0A
         
 val 
@@ -1137,61 +1137,179 @@
     
-val = line.split(%22%7C%22)%5B2%5D.strip()%0A%0A        context = %7B
+context_ip = %7B%22source%22: self.name, %22lastseen%22: row%5B%22lastseen%22%5D%7D%0A%0A        try:%0A            ip = Ip.get_or_create(value=row%5B%22ipaddr%22%5D)%0A            ip.add_context(context_ip)
 %0A   
@@ -1317,26 +1317,30 @@
         
-%22
+ip.add_
 source
-%22: 
+(
 self.nam
@@ -1340,17 +1340,17 @@
 elf.name
-,
+)
 %0A       
@@ -1354,69 +1354,232 @@
     
-%7D%0A%0A        try:%0A            obs = Ip.get_or_create(value=val)
+    ip.tag(%22dataplane%22)%0A            ip.tag(%22dns%22)%0A            ip.tag(row%5B%22category%22%5D)%0A%0A            asn = AutonomousSystem.get_or_create(value=row%5B%22ASN%22%5D)%0A            context_ans = %7B%22source%22: self.name, %22name%22: row%5B%22ASname%22%5D%7D
 %0A   
@@ -1583,27 +1583,27 @@
             
-obs
+asn
 .add_context
@@ -1610,16 +1610,20 @@
 (context
+_ans
 )%0A      
@@ -1628,19 +1628,19 @@
         
-obs
+asn
 .add_sou
@@ -1662,27 +1662,27 @@
             
-obs
+asn
 .tag(%22datapl
@@ -1703,21 +1703,46 @@
     
-obs.tag(%22dns%22
+asn.active_link_to(ip, %22AS%22, self.name
 )%0A  

Update dataplane_dnsrd.py