var sys = require('sys'),
    fs = require('fs'),
    Buffer = require('buffer').Buffer;

// This size only affects performance, it's not a constraint on data sizes
var CHUNK_SIZE = 1024;
// This is the max size of a single serialized document
var MAX_SIZE = 1024 * 1024;


function Collection(filename) {
    var writer, reader; // FDs for reading and writing to the file
    var index = {};
    var writeQueue = [];

    // Open a single writer and reader
    writer = fs.openSync(filename, "a");
    reader = fs.openSync(filename, "r");

    function loadDatabase() {
        sys.debug("Loading...");

        // Create a buffer for reading chunks from the disk
        var chunk = new Buffer(CHUNK_SIZE);

        // Create an empty stream buffer
        var input = new Buffer(MAX_SIZE);
        input.length = 0;

        // These are positions in the database file
        var offset = 0;
        var base = 0;

        // This is a position within the input stream
        var pos = 0;
        var mid = 0;

        // Read a chunk from the file into `chunk`
        while ((chunk.length = fs.readSync(reader, chunk, 0, CHUNK_SIZE, offset)) > 0) {

            // Move the offset so the outer loop stays in sync
            offset += chunk.length;

            // Copy the chunk onto the input stream
            chunk.copy(input, input.length, 0, chunk.length);
            input.length += chunk.length;

            // See if there is input to consume
            for (var i = pos, l = input.length; i < l; i++) {
                if (input[i] === 9) {
                    mid = i + 1;
                }
                if (mid && input[i] === 10) {
                    var meta = JSON.parse(input.slice(pos, mid - 1).toString());
                    var doc = input.slice(mid, i).toString();
                    meta.position = base + mid;
                    meta.length = i - mid;
                    index[meta.key] = meta;
                    // sys.puts(sys.inspect(meta));
                    // sys.puts(doc);
                    mid = 0;
                    pos = i + 1;
                    // TODO: use
                }
            }

            // Shift the input back down
            if (pos > 0) {
                input.copy(input, 0, pos, input.length);
                input.length -= pos;
                base += pos;
                pos = 0;
            }
        }
        sys.debug(sys.inspect(index));

    }

    loadDatabase();
    Object.keys(index).forEach(function (key) {
        var meta = index[key];
        var buffer = new Buffer(meta.length);

        var offset = 0;
        function getNext() {
            fs.read(reader, buffer, offset, meta.length - offset, meta.position + offset, function (err, bytes) {
                if (bytes < meta.length - offset) {
                    offset += bytes;
                    getNext();
                    return;
                }
                var data = JSON.parse(buffer.toString());
                sys.debug(sys.inspect({meta:meta,data:data}));
            });
        }
        getNext();
    });

    var lock = 0;
    function checkQueue() {
        lock++;
        if (lock > 1 || writeQueue === 0) { return; }
        var next = writeQueue.shift();
        var entry = new Buffer(JSON.stringify(next.meta) + "\t" + JSON.stringify(next.doc) + "\n");
        fs.write(writer, entry, null, entry.length, null, function (err, written) {
            if (err) throw err;
            sys.debug(written);
        });

    }

    return {
        createDocument: function (doc, key) {
            if (!key) {
                key = getNextKey();
            }
            writeQueue.push({meta: {key: key}, doc: doc});
            checkQueue();
        }
    };
}

var users = new Collection("../data/users.db");

// var doc = users.createDocument({name: "Tim", age: 28}, 1);

@@ -542,66 +542,88 @@
     
-function loadDatabase() %7B%0A        sys.debug(%22Loading...%22);
+// Loads the database from disk using blocking I/O%0A    function loadDatabase() %7B
 %0A%0A  
@@ -2414,120 +2414,66 @@
   %7D%0A
+%0A
     
+%7D%0A%0A
     
-sys.debug(sys.inspect(index));%0A%0A    %7D%0A%0A    loadDatabase();%0A    Object.keys(index).forEach(function (key)
+function getByKey(key, callback) %7B%0A        try
  %7B%0A 
@@ -2471,32 +2471,36 @@
   try %7B%0A        
+    
 var meta = index
@@ -2506,16 +2506,20 @@
 x%5Bkey%5D;%0A
+    
         
@@ -2553,24 +2553,28 @@
 a.length);%0A%0A
+    
         var 
@@ -2589,24 +2589,28 @@
  0;%0A        
+    
 function get
@@ -2614,24 +2614,28 @@
 getNext() %7B%0A
+    
             
@@ -2732,24 +2732,182 @@
 r, bytes) %7B%0A
+                    if (err) %7B%0A                        callback(err);%0A                        return;%0A                    %7D%0A                    try %7B%0A        
             
@@ -2958,32 +2958,40 @@
                 
+       
+ 
 offset += bytes;
@@ -2991,16 +2991,24 @@
  bytes;%0A
+        
         
@@ -3046,32 +3046,48 @@
             
-return;%0A
+        return;%0A        
             
@@ -3104,24 +3104,32 @@
             
+        
 var data = J
@@ -3178,55 +3178,184 @@
     
-sys.debug(sys.inspect(%7Bmeta:meta,data:data%7D));%0A
+        data.key = meta.key;%0A                        callback(null, data);%0A                    %7D catch (err) %7B%0A                        callback(err);%0A                    %7D%0A    
     
@@ -3374,26 +3374,32 @@
         
-%7D%0A
+    %7D%0A  
         
 getNext(
@@ -3382,32 +3382,34 @@
     %7D%0A          
+  
 getNext();%0A    %7D
@@ -3399,24 +3399,292 @@
  getNext();%0A
+        %7D catch (err) %7B%0A            callback(err);%0A        %7D%0A    %7D%0A    loadDatabase();%0A    Object.keys(index).forEach(function (key) %7B%0A        getByKey(key, function (err, doc) %7B%0A            if (err) %7B throw err; %7D%0A            sys.debug(sys.inspect(doc));%0A        %7D);%0A
     %7D);%0A%0A   

Refactor the loader into a function.