<jupyter_start><jupyter_text>because of long running tasks and Tokenization of urls (it didnt work locally on Jupyter) so please launch it in google collab (it has to work there)<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
%matplotlib inline

from tqdm import tqdm
from scipy.sparse import csr_matrix

import pandas as pd
import numpy as np
import scipy
import matplotlib.pyplot as plt

import sklearn.model_selection as skl
import sklearn.metrics as sklm
from sklearn.metrics import f1_score

! pip install pymorphy2
! pip install youtokentome
import random
import youtokentome as yttm
import pymorphy2 # Морфологический анализатор

import string # for work with strings
import nltk   # Natural Language Toolkit
from nltk.stem.snowball import SnowballStemmer 

# classifiers
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
train_df = pd.read_csv("/content/drive/My Drive/Colab Notebooks/data/train.csv")

train_df.head()
test_df = pd.read_csv("/content/drive/My Drive/Colab Notebooks/data/test.csv")

test_df.head()
train_df["target"].value_counts()
X_train_title = train_df["title"].values
X_test_title = test_df["title"].values

X_train_url = train_df["url"].values
X_test_url = test_df["url"].values

y_train = train_df["target"].astype(int).values<jupyter_output><empty_output><jupyter_text>Logistic Regression Classifier  Url processing<jupyter_code>train_data_path = "/content/drive/My Drive/Colab Notebooks/data/train_data.txt"
model_path = "/content/drive/My Drive/Colab Notebooks/data/example.model"

with open(train_data_path, "w") as fout:
    for url in X_train_url:
        print(url, file=fout)

# Training model
yttm.BPE.train(data=train_data_path, vocab_size=4000, model=model_path)

# Loading model
bpe = yttm.BPE(model=model_path)

# Two types of tokenization
print(bpe.encode(["www.pornhub.com"], output_type=yttm.OutputType.SUBWORD))<jupyter_output>[['▁www.porn', 'h', 'ub.com']]<jupyter_text>data cleaning<jupyter_code>word_tokenizer = nltk.WordPunctTokenizer()

# get russian and english stop words
nltk.download('stopwords')
stop_words = nltk.corpus.stopwords.words('russian')
stop_words.append(nltk.corpus.stopwords.words('english'))
def process_data(texts):
    updated_texts = []
    
    # string processers
    stemmer = SnowballStemmer('russian')
    morph = pymorphy2.MorphAnalyzer()
    
    for text in tqdm(texts):    
        text_lower = text.lower() # convert words in a text to lower case
        tokens     = word_tokenizer.tokenize(text_lower) # splits the text into tokens (words)
        
        # remove punct and stop words from tokens
        tokens = [word for word in tokens if (word not in string.punctuation and word not in stop_words)]
        
        tmp_str = ''
        for token in tokens:
            # not taking in account 0 or 1 letter tokens
            if (len(token) >= 2):
                # lemmatization
                token = (morph.parse(token))[0].normal_form
                
                # stemming
                # token = stemmer.stem(token)
                
                tmp_str += token + ' '
        
        updated_texts.append(tmp_str.strip())
    
    return updated_texts
print(X_train_title)
X_train_title_processed = process_data(X_train_title)
X_test_title_processed = process_data(X_test_title)
print(X_train_title_processed[0:10])<jupyter_output>0%|          | 70/135309 [00:00<03:15, 692.23it/s]<jupyter_text>Splitting and fitting<jupyter_code># url_vectorizer = TfidfVectorizer(max_features = 3500, tokenizer = lambda x: bpe.encode(x, output_type=yttm.OutputType.SUBWORD))
# title_vectorizer = TfidfVectorizer(max_features = 7500)

url_vectorizer = CountVectorizer(max_features = 3500, tokenizer = lambda x: bpe.encode(x, output_type=yttm.OutputType.SUBWORD))
title_vectorizer = CountVectorizer(max_features = 7500)

clf = LogisticRegression(solver='liblinear')
# splitting using function
X_data = []
for i in range(len(X_train_url)):
    X_data.append((X_train_url[i], X_train_title_processed[i]))

df_X_train, df_X_test, df_y_train, df_y_test = skl.train_test_split(X_data, y_train, test_size=0.2, random_state=42)
print('train X:', df_X_train[0:5])
print('test X:', df_X_test[0:5])
print('y_train:', df_y_train[0:20])
print('y_test:', df_y_test[0:20])
porno_words = ['порно', 'гей', 'лесби', 'хуй', 'член', 'презерватив' ,'секс', 'xxx', 'эротика', 'sex', 'porn', 'dick','cock' ,'gay', 'latina', 'boobs', 'tits']
def get_porn_matrix(data_url, data_title):
    porn_list = []

    for i in range(len(data_url)):
        value = 0
        for pron_word in porno_words:
            if (pron_word in data_url[i] or pron_word in data_title[i]):
            value = 1
            break
        
        porn_list.append(value)

    return csr_matrix(porn_list).transpose()
def get_vectorizers(X_url_train, X_url_test, X_title_train, X_title_test):
    #train
    url_vectorizer.fit(X_url_train)
    url_vectorizer_train = url_vectorizer.transform(X_url_train)

    title_vectorizer.fit(X_title_train)
    title_vectorizer_train = title_vectorizer.transform(X_title_train)

    #test
    url_vectorizer_test = url_vectorizer.transform(X_url_test)
    title_vectorizer_test = title_vectorizer.transform(X_title_test)

    shared_vectorizer_train = scipy.sparse.hstack((url_vectorizer_train, title_vectorizer_train, get_porn_matrix(X_url_train, X_title_train)))
    shared_vectorizer_test = scipy.sparse.hstack((url_vectorizer_test, title_vectorizer_test, get_porn_matrix(X_url_test, X_title_test)))

    return shared_vectorizer_train, shared_vectorizer_test
# fitting vectorizers
df_X_train_urls = [x[0] for x in df_X_train]
df_X_train_titles = [x[1] for x in df_X_train]

df_X_test_urls = [x[0] for x in df_X_test]
df_X_test_titles = [x[1] for x in df_X_test]

shared_vectorizer_train, shared_vectorizer_test = get_vectorizers(df_X_train_urls, df_X_test_urls, df_X_train_titles, df_X_test_titles)
%%time
clf.fit(shared_vectorizer_train, df_y_train)

print(clf.n_iter_)
y_pred = clf.predict(shared_vectorizer_test)
f1_score(df_y_test, y_pred)
#0.9751107039242632<jupyter_output><empty_output><jupyter_text>Submit Logistic<jupyter_code>shared_vectorizer_train, shared_vectorizer_test = get_vectorizers(X_train_url, X_test_url, X_train_title_processed, X_test_title_processed)
%%time
clf.fit(shared_vectorizer_train, y_train)

print(clf.n_iter_)
y_pred = clf.predict(shared_vectorizer_test)
test_df["target"] = y_pred.astype(bool)

test_df[["id", "target"]].to_csv("/content/drive/My Drive/Colab Notebooks/data/ml_baseline_logistic.csv", index=False)

!cat ml_baseline.csv | head<jupyter_output>cat: ml_baseline.csv: No such file or directory
StarcoderdataJupyterStructuredCleanDedup