#!/usr/bin/python3
from tqdm import tqdm
import mmap


#http://stackoverflow.com/a/27518377/2230446
def get_num_lines(filename):
	f = open(filename, "rb")
	num_lines = 0
	buf_size = 1024 * 1024
	read_f = f.raw.read

	buf = read_f(buf_size)
	while buf:
		num_lines += buf.count(b"\n")
		buf = read_f(buf_size)

	return num_lines


def sample_every_nth_line(source, dest, n):
	num_samples = 0
	num_lines = get_num_lines(source)
	with open(dest, "w+") as sample:
		with open(source) as population:
			for i, line in tqdm(enumerate(population), total=num_lines):  
				if i % n == 0:
					# print(line)
					num_samples += 1
					sample.write(line.strip())
					sample.write("\n")
	print("{} samples taken from a population of {} ({:.2f}%)".format(num_samples, num_lines, 100.0*num_samples/num_lines))


def main():
	# source = "../../res/head_safebooru.xml"
	source = "../../res/safebooru.xml"
	sample_every_nth_line(source, dest="../../res/sample_safebooru.xml", n=100)

if __name__ == "__main__":
	main()


@@ -49,16 +49,72 @@
 t mmap%0D%0A
+from bs4 import BeautifulSoup%0D%0Afrom random import random
 %0D%0A%0D%0A#htt
@@ -636,10 +636,8 @@
 es):
-  
 %0D%0A%09%09
@@ -877,24 +877,859 @@
 ines))%0D%0A%0D%0A%0D%0A
+def sample_all_non_zero_scores(source, dest):%0D%0A%09num_samples = 0%0D%0A%09num_lines = get_num_lines(source)%0D%0A%09with open(dest, %22w+%22) as sample:%0D%0A%09%09with open(source) as population:%0D%0A%09%09%09for i, line in tqdm(enumerate(population), total=num_lines):%0D%0A%09%09%09%09soup = BeautifulSoup(line, %22lxml%22)%0D%0A%09%09%09%09post = soup.find(%22post%22)%0D%0A%09%09%09%09if post is not None:%0D%0A%09%09%09%09%09score = int(post%5B%22score%22%5D)%0D%0A%0D%0A%09%09%09%09%09# sample ~20,000 with score 0 and ~20,000 with score 1, and sample all of the images with scores %3E1 or %3C0%0D%0A%09%09%09%09%09if (score %3C 0 or score %3E 1) or (score == 0 and random() %3C 20000/1374164) or (score == 1 and random() %3C 20000/369619):%0D%0A%09%09%09%09%09%09# print(line)%0D%0A%09%09%09%09%09%09num_samples += 1%0D%0A%09%09%09%09%09%09sample.write(line.strip())%0D%0A%09%09%09%09%09%09sample.write(%22%5Cn%22)%0D%0A%09print(%22%7B%7D samples taken from a population of %7B%7D (%7B:.2f%7D%25)%22.format(num_samples, num_lines, 100.0*num_samples/num_lines))%0D%0A%0D%0A%0D%0A
 def main():%0D
@@ -1811,16 +1811,18 @@
 .xml%22%0D%0A%09
+# 
 sample_e
@@ -1889,16 +1889,92 @@
  n=100)%0D
+%0A%09sample_all_non_zero_scores(source, dest=%22../../res/sample_safebooru.xml%22)%0D
 %0A%0D%0Aif __

fix bug in sampler