rs.secondaryOk();
const dryRun = true;
const maxPerSecond = 600;
const flushEvery = 100;
const parseLineRegex = /^([\w-~]+)([ !&\?])(.*)$/;

const isChatGarbage = chat =>
  chat.l.every(l => {
    try {
      const [_, author, flag, text] = l.match(parseLineRegex);
      return author == 'lichess' || isFlagGarbage(flag) || isTextGarbage(text.trim().toLowerCase());
    } catch (e) {
      print(e);
      print(l);
    }
  });

const isFlagGarbage = flag => flag == '!' || flag == '?';

const presets = new Set([
  'hello',
  'good luck',
  'have fun!',
  'you too!',
  'good game',
  'well played',
  'thank you',
  "i've got to go",
  'bye!',
]);

const isTextGarbage = text => text.indexOf(' ') < 0 || presets.has(text);

const numberFormat = n => `${Math.round(n / 1000)}k`;

let read = 0,
  deleted = 0,
  idBuffer = [];

const deleteChat = chat => {
  // if (chat.l.some(l => !l.startsWith('lichess'))) printjson(chat.l);
  // if (chat.l.some(l => l.includes('Good game'))) printjson(chat.l);
  idBuffer.push(chat.id);
  if (idBuffer.length >= flushEvery) flushBuffer();
};

const flushBuffer = () => {
  if (!dryRun) {
    db.chat.remove({ _id: { $in: idBuffer } }, { writeConcern: { w: 0 } });
    sleep(Math.round((1000 * flushEvery) / maxPerSecond));
  }
  idBuffer = [];
  deleted += flushEvery;
  if (deleted % 10000 == 0)
    print(`${numberFormat(deleted)} / ${numberFormat(read)} - ${Math.round((deleted * 100) / read)}%`);
};

db.chat.find().forEach(chat => {
  read++;
  if (isChatGarbage(chat)) deleteChat(chat);
});

flushBuffer();

@@ -276,27 +276,31 @@
 urn 
-a
+isA
 uthor
- == 'lichess'
+Garbage(author)
  %7C%7C 
@@ -398,40 +398,237 @@
 int(
-e);%0A      print(l);%0A    %7D%0A  %7D);%0A
+%60$%7Bchat._id%7D $%7Be%7D $%7Bl%7D%60);%0A      return false;%0A    %7D%0A  %7D);%0A%0A// system, anon, bots%0Aconst isAuthorGarbage = author =%3E author == 'lichess' %7C%7C author == 'w' %7C%7C author == 'b' %7C%7C author.indexOf('BOT~') == 0;%0A%0A// shadowbanned and deleted
 %0Acon
@@ -713,19 +713,8 @@
 t(%5B%0A
-  'hello',%0A
   'g
@@ -824,18 +824,8 @@
 o%22,%0A
-  'bye!',%0A
 %5D);%0A
@@ -1629,15 +1629,51 @@
 chat
+%0A  
 .find()
+%0A  // .skip(240 * 1000 * 1000)%0A  
 .for
@@ -1687,16 +1687,18 @@
 at =%3E %7B%0A
+  
   read++
@@ -1697,24 +1697,26 @@
   read++;%0A  
+ 
+ 
 if (isChatGa
@@ -1727,16 +1727,24 @@
 e(chat))
+ %7B%0A     
  deleteC
@@ -1754,16 +1754,71 @@
 (chat);%0A
+      if (read %25 100000 == 0) printjson(chat);%0A    %7D%0A  
 %7D);%0A%0Aflu

improve chat cleanup