""" try to categorize the onion """
from caepainvestigatio.logging_conf import initLogging
from caepainvestigatio.ORM.categories import Category
from langdetect import detect

log = initLogging()

def language(onion_info):
    """ search to categorize the onion """

    if onion_info.snapshot == "" or onion_info.snapshot is None:
        return None

    lang = detect(onion_info.snapshot)
    log.info("lang detected : %s", lang)
    return lang

def search_category(onion_info):
    """ try to categorize with dictionnary a website """

    category_find = []
    for category in Category.objects():
        for expression in category.expressions:
            # find expression in onion website sources
            if expression in onion_info.snapshot:
                log.debug("detect category %s for onion %s", category.category, onion_info.hiddenService)
                if category not in category_find:
                    category_find.append(category)

    return category_find

@@ -29,16 +29,27 @@
 ion %22%22%22%0A
+import re%0A%0A
 from cae
@@ -608,16 +608,36 @@
 ects():%0A
+        count_w = 0%0A
         
@@ -743,19 +743,73 @@
         
-if 
+count = sum(1 for _ in re.finditer(r'%5Cb%25s%5Cb' %25 re.escape(
 expressi
@@ -806,27 +806,26 @@
 e(expression
- in
+),
  onion_info.
@@ -832,18 +832,148 @@
 snapshot
-:%0A
+))%0A            if count %3E 0:%0A                count_w += count%0A                if count_w %3E 2 and category not in category_find:%0A    
         
@@ -1086,18 +1086,19 @@
         
-if
+   
  categor
@@ -1098,32 +1098,37 @@
 category
- not in 
+_find.append(
 category
 _find:%0A 
@@ -1111,38 +1111,33 @@
 .append(category
-_find:
+)
 %0A               
@@ -1145,38 +1145,13 @@
     
-category_find.append(category)
+break
 %0A%0A  

improve categorization