Upload processor

processor.py

@@ -13,7 +13,7 @@ class GIAProcessor(GitProcessor):
         }
         for i in range(len(examples["input_ids"])):
             _input_size = len(examples["input_ids"][i])
-            for j in range(_input_size // self._block_size):
+            for j in range(max(1, _input_size // self._block_size)):
                 results["input_ids"].append(examples["input_ids"][i][j*self._block_size:(j+1)*self._block_size])
                 results["attention_mask"].append(examples["attention_mask"][i][j * self._block_size:(j + 1) * self._block_size])