
/**
 * Test dependecies.
 */

var assert = require('assert');
var tokenizer = require('..');


describe('basic', function() {

	it('split words', function() {
		assert.deepEqual(tokenizer('it is beauty'), ['it', 'is', 'beauty']);
	});
	
});
@@ -228,14 +228,185 @@
 %5D);%0A%09%7D);
+%0A%0A%7D);%0A%0A%0Adescribe('special characters', function() %7B%0A%0A%09it('split words with dots', function() %7B%0A%09%09assert.deepEqual(tokenizer('it is beauty.'), %5B'it', 'is', 'beauty'%5D);%0A%09%7D);
 %0A%09%0A%7D);

add special characters test