Training in progress, step 50000, checkpoint

checkpoint-50000/added_tokens.json

@@ -0,0 +1,127 @@
+{
+  "<extra_id_0>": 259,
+  "<extra_id_100>": 359,
+  "<extra_id_101>": 360,
+  "<extra_id_102>": 361,
+  "<extra_id_103>": 362,
+  "<extra_id_104>": 363,
+  "<extra_id_105>": 364,
+  "<extra_id_106>": 365,
+  "<extra_id_107>": 366,
+  "<extra_id_108>": 367,
+  "<extra_id_109>": 368,
+  "<extra_id_10>": 269,
+  "<extra_id_110>": 369,
+  "<extra_id_111>": 370,
+  "<extra_id_112>": 371,
+  "<extra_id_113>": 372,
+  "<extra_id_114>": 373,
+  "<extra_id_115>": 374,
+  "<extra_id_116>": 375,
+  "<extra_id_117>": 376,
+  "<extra_id_118>": 377,
+  "<extra_id_119>": 378,
+  "<extra_id_11>": 270,
+  "<extra_id_120>": 379,
+  "<extra_id_121>": 380,
+  "<extra_id_122>": 381,
+  "<extra_id_123>": 382,
+  "<extra_id_124>": 383,
+  "<extra_id_12>": 271,
+  "<extra_id_13>": 272,
+  "<extra_id_14>": 273,
+  "<extra_id_15>": 274,
+  "<extra_id_16>": 275,
+  "<extra_id_17>": 276,
+  "<extra_id_18>": 277,
+  "<extra_id_19>": 278,
+  "<extra_id_1>": 260,
+  "<extra_id_20>": 279,
+  "<extra_id_21>": 280,
+  "<extra_id_22>": 281,
+  "<extra_id_23>": 282,
+  "<extra_id_24>": 283,
+  "<extra_id_25>": 284,
+  "<extra_id_26>": 285,
+  "<extra_id_27>": 286,
+  "<extra_id_28>": 287,
+  "<extra_id_29>": 288,
+  "<extra_id_2>": 261,
+  "<extra_id_30>": 289,
+  "<extra_id_31>": 290,
+  "<extra_id_32>": 291,
+  "<extra_id_33>": 292,
+  "<extra_id_34>": 293,
+  "<extra_id_35>": 294,
+  "<extra_id_36>": 295,
+  "<extra_id_37>": 296,
+  "<extra_id_38>": 297,
+  "<extra_id_39>": 298,
+  "<extra_id_3>": 262,
+  "<extra_id_40>": 299,
+  "<extra_id_41>": 300,
+  "<extra_id_42>": 301,
+  "<extra_id_43>": 302,
+  "<extra_id_44>": 303,
+  "<extra_id_45>": 304,
+  "<extra_id_46>": 305,
+  "<extra_id_47>": 306,
+  "<extra_id_48>": 307,
+  "<extra_id_49>": 308,
+  "<extra_id_4>": 263,
+  "<extra_id_50>": 309,
+  "<extra_id_51>": 310,
+  "<extra_id_52>": 311,
+  "<extra_id_53>": 312,
+  "<extra_id_54>": 313,
+  "<extra_id_55>": 314,
+  "<extra_id_56>": 315,
+  "<extra_id_57>": 316,
+  "<extra_id_58>": 317,
+  "<extra_id_59>": 318,
+  "<extra_id_5>": 264,
+  "<extra_id_60>": 319,
+  "<extra_id_61>": 320,
+  "<extra_id_62>": 321,
+  "<extra_id_63>": 322,
+  "<extra_id_64>": 323,
+  "<extra_id_65>": 324,
+  "<extra_id_66>": 325,
+  "<extra_id_67>": 326,
+  "<extra_id_68>": 327,
+  "<extra_id_69>": 328,
+  "<extra_id_6>": 265,
+  "<extra_id_70>": 329,
+  "<extra_id_71>": 330,
+  "<extra_id_72>": 331,
+  "<extra_id_73>": 332,
+  "<extra_id_74>": 333,
+  "<extra_id_75>": 334,
+  "<extra_id_76>": 335,
+  "<extra_id_77>": 336,
+  "<extra_id_78>": 337,
+  "<extra_id_79>": 338,
+  "<extra_id_7>": 266,
+  "<extra_id_80>": 339,
+  "<extra_id_81>": 340,
+  "<extra_id_82>": 341,
+  "<extra_id_83>": 342,
+  "<extra_id_84>": 343,
+  "<extra_id_85>": 344,
+  "<extra_id_86>": 345,
+  "<extra_id_87>": 346,
+  "<extra_id_88>": 347,
+  "<extra_id_89>": 348,
+  "<extra_id_8>": 267,
+  "<extra_id_90>": 349,
+  "<extra_id_91>": 350,
+  "<extra_id_92>": 351,
+  "<extra_id_93>": 352,
+  "<extra_id_94>": 353,
+  "<extra_id_95>": 354,
+  "<extra_id_96>": 355,
+  "<extra_id_97>": 356,
+  "<extra_id_98>": 357,
+  "<extra_id_99>": 358,
+  "<extra_id_9>": 268
+}

checkpoint-50000/configuration_sanigec.py

@@ -0,0 +1,58 @@
+from typing import Any
+
+from transformers import PreTrainedConfig
+
+
+class SaniGecConfig(PreTrainedConfig):
+    """
+    Configuration for SaniGec, the non-autoregressive byte-encoder GEC tagger.
+
+    A pretrained ByT5 encoder over raw UTF-8 bytes is pooled back to codepoint resolution and feeds
+    two heads: a binary error-detection head and an edit-correction head over the tag vocabulary
+    (KEEP/DELETE/APPEND_c/REPLACE_c). `backbone_name` is the HF checkpoint the encoder is first
+    transferred from (via `SaniGecModel.from_backbone`); `encoder_config` is that encoder's
+    serialized architecture, persisted so a reload rebuilds the encoder to the right shape offline
+    and the saved state-dict fills it — the model never re-fetches the backbone on reload.
+    `hidden_size` mirrors the encoder's `d_model` so the heads and the pooling anchor are sized to
+    it. The loss hyperparameters live here so a checkpoint records the objective it trained under.
+    """
+
+    model_type = "sani_gec"
+
+    def __init__(
+        self,
+        backbone_name: str = "google/byt5-base",
+        encoder_config: dict[str, Any] | None = None,
+        hidden_size: int = 1536,
+        num_correction_tags: int = 512,
+        correction_class_weights: list[float] | None = None,
+        dropout: float = 0.1,
+        detection_loss_weight: float = 1.0,
+        correction_loss_weight: float = 1.0,
+        focal_gamma: float = 2.0,
+        label_smoothing: float = 0.1,
+        keep_confidence_margin: float = 0.0,
+        max_correction_passes: int = 12,
+        pad_token_id: int = 0,
+        **kwargs: Any,
+    ) -> None:
+        self.backbone_name = backbone_name
+        self.encoder_config = encoder_config
+        # The encoder's d_model is authoritative when its architecture is known; the default is only
+        # a placeholder for a bare config built before a backbone is attached.
+        self.hidden_size = encoder_config["d_model"] if encoder_config else hidden_size
+        self.num_correction_tags = num_correction_tags
+        self.correction_class_weights = correction_class_weights
+        self.dropout = dropout
+        self.detection_loss_weight = detection_loss_weight
+        self.correction_loss_weight = correction_loss_weight
+        self.focal_gamma = focal_gamma
+        self.label_smoothing = label_smoothing
+        # Inference-time decode rule (the iterative corrector reads these), persisted so a reloaded
+        # checkpoint corrects with no external arguments.
+        self.keep_confidence_margin = keep_confidence_margin
+        self.max_correction_passes = max_correction_passes
+        # Forward via a dict so the special tokens ride in **kwargs: the transformers v5 type stub
+        # does not name pad_token_id on PreTrainedConfig.__init__, though the runtime accepts it.
+        init_kwargs: dict[str, Any] = {"pad_token_id": pad_token_id, **kwargs}
+        super().__init__(**init_kwargs)

checkpoint-50000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4e8aa83c3bdf31f37cf2575cdbc16e4162f85051ae1ed26ab229f563c3d20d2
+size 1674814416

checkpoint-50000/modeling_sanigec.py

@@ -0,0 +1,302 @@
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import torch
+from torch import nn
+from transformers import (
+    AutoConfig,
+    AutoModel,
+    PreTrainedModel,
+    PreTrainedTokenizerBase,
+    T5Config,
+    T5EncoderModel,
+)
+from transformers.modeling_outputs import ModelOutput  # type: ignore[attr-defined]
+from transformers.pipelines import PIPELINE_REGISTRY
+from transformers.utils import cached_file, logging  # type: ignore[attr-defined]
+
+from bambara_gec.model.byte_bridge import ByteCodepointBridge
+from bambara_gec.model.configuration_sanigec import SaniGecConfig
+from bambara_gec.model.losses import (
+    IGNORE_INDEX,
+    ClassWeightedLabelSmoothedCrossEntropy,
+    FocalLoss,
+    combine_head_losses,
+)
+from bambara_gec.model.pipeline import TASK_NAME, SaniGecCorrectionPipeline
+from bambara_gec.model.tags import TagScheme
+from bambara_gec.model.vocabulary import CharVocabulary
+
+VOCAB_FILE = "vocab.json"
+
+# Shipped in config.json so `pipeline("sani-gec", model=repo, trust_remote_code=True)` resolves the
+# correction pipeline from the repo's own pipeline.py — the same self-containment contract as
+# auto_map for the config/model.
+CUSTOM_PIPELINES = {
+    TASK_NAME: {
+        "impl": "pipeline.SaniGecCorrectionPipeline",
+        "pt": ["AutoModel"],
+        "tf": [],
+    }
+}
+
+logger = logging.get_logger(__name__)
+
+
+@dataclass
+class SaniGecOutput(ModelOutput):
+    """
+    Forward output: the combined training loss (None without labels) and the two heads' per-position
+    logits — detection over {keep, edit} and correction over the tag vocabulary — at codepoint
+    resolution (the pooled `[anchor, cp₁…cpₙ, eos]` layout).
+    """
+
+    loss: torch.Tensor | None = None
+    detection_logits: torch.Tensor | None = None
+    correction_logits: torch.Tensor | None = None
+
+
+class SaniGecModel(PreTrainedModel):
+    """
+    A pretrained ByT5 encoder over raw UTF-8 bytes, pooled back to codepoint resolution by the
+    byte→codepoint bridge, feeding the two GECToR-style heads. Non-autoregressive: every codepoint
+    is tagged in parallel in one forward; multi-pass correction is the inference loop's job.
+
+    The encoder is built empty from the persisted `encoder_config` here and the pretrained weights
+    are transferred once by `from_backbone` (not inside `__init__`): so reloading a fine-tuned
+    checkpoint rebuilds the encoder to shape and lets the saved state-dict fill it, instead of
+    re-fetching the backbone and overwriting the trained weights. Only the two heads are
+    `_init_weights`-initialised (the framework hook); the encoder owns its own init/loaded weights.
+    The detection/correction labels are codepoint-indexed and align 1:1 with the pooled layout, so
+    the loss reshapes over codepoint positions exactly as before.
+    """
+
+    config_class = SaniGecConfig
+    base_model_prefix = "sani_gec"
+
+    def __init__(self, config: SaniGecConfig) -> None:
+        super().__init__(config)
+        encoder_config = (
+            T5Config.from_dict(config.encoder_config)
+            if config.encoder_config is not None
+            else T5Config.from_pretrained(config.backbone_name)
+        )
+        self.encoder = T5EncoderModel(encoder_config)
+        hidden = encoder_config.d_model
+        self.bridge = ByteCodepointBridge(hidden)
+        self.dropout = nn.Dropout(config.dropout)
+        self.detection_head = nn.Linear(hidden, 2)
+        self.correction_head = nn.Linear(hidden, config.num_correction_tags)
+        self.detection_loss = FocalLoss(gamma=config.focal_gamma)
+        correction_class_weights = (
+            torch.tensor(config.correction_class_weights, dtype=torch.float32)
+            if config.correction_class_weights is not None
+            else None
+        )
+        self.correction_loss = ClassWeightedLabelSmoothedCrossEntropy(
+            class_weight=correction_class_weights, smoothing=config.label_smoothing
+        )
+        # The decode tag scheme (id -> applicable edit). None until attached from a CharVocabulary
+        # (at construction) or rebuilt from the shipped vocab.json (on reload).
+        self.tag_scheme: TagScheme | None = None
+        # The byte tokenizer (stock ByT5). Carried so save_pretrained ships it beside the weights,
+        # making the repo loadable end-to-end by `pipeline(...)` / AutoTokenizer with no external
+        # reference.
+        self.tokenizer: PreTrainedTokenizerBase | None = None
+        self.post_init()
+
+    @classmethod
+    def from_backbone(
+        cls,
+        config: SaniGecConfig,
+        tag_scheme: TagScheme | None = None,
+        tokenizer: PreTrainedTokenizerBase | None = None,
+    ) -> "SaniGecModel":
+        """
+        First construction: load the pretrained ByT5 encoder from `config.backbone_name` exactly
+        once, record its architecture into the config (so later reloads rebuild it offline), build
+        the model with a matching empty encoder, and transfer the pretrained weights into it. A tag
+        scheme and/or tokenizer given here are attached so the saved model is self-contained.
+        """
+        # Force safetensors: transformers refuses torch.load of a .bin checkpoint under torch < 2.6
+        # (CVE-2025-32434), and the byt5 checkpoints ship safetensors — which also skips the
+        # redundant multi-GB .bin download.
+        pretrained = T5EncoderModel.from_pretrained(config.backbone_name, use_safetensors=True)
+        config.encoder_config = pretrained.config.to_dict()
+        config.hidden_size = pretrained.config.d_model
+        config.custom_pipelines = CUSTOM_PIPELINES
+        model = cls(config)
+        model.encoder.load_state_dict(pretrained.state_dict())
+        if tag_scheme is not None:
+            model.set_tag_scheme(tag_scheme)
+        if tokenizer is not None:
+            model.tokenizer = tokenizer
+        return model
+
+    def set_tag_scheme(self, tag_scheme: TagScheme) -> None:
+        """
+        Attach the decode scheme and derive the config's label maps from it. The correction head is
+        already sized to `num_correction_tags`, so the scheme's tag count must match it — a mismatch
+        would mean the head and the decode vocabulary disagree, which is asserted rather than
+        silently shipped.
+        """
+        if tag_scheme.num_tags != self.config.num_correction_tags:
+            raise ValueError(
+                f"tag scheme has {tag_scheme.num_tags} tags but the correction head is sized for "
+                f"{self.config.num_correction_tags}"
+            )
+        self.tag_scheme = tag_scheme
+        self.config.id2label = tag_scheme.id2label
+        self.config.label2id = tag_scheme.label2id
+
+    def save_pretrained(  # type: ignore[override]
+        self, save_directory: str | os.PathLike[str], **kwargs: Any
+    ) -> None:
+        """
+        Ship the tag vocabulary as vocab.json beside the weights and config, so a reload rebuilds
+        the identical decode scheme from the repo alone (the self-containment contract). It is
+        written before the weights/config so it is on disk for any subsequent folder upload; the
+        Trainer's `push_to_hub` (US-09-5) uploads the whole folder, so the pushed repo carries it.
+        """
+        if self.tag_scheme is not None:
+            Path(save_directory).mkdir(parents=True, exist_ok=True)
+            self.tag_scheme.vocabulary.save(Path(save_directory) / VOCAB_FILE)
+        if self.tokenizer is not None:
+            self.tokenizer.save_pretrained(str(save_directory))
+        super().save_pretrained(save_directory, **kwargs)
+
+    @classmethod
+    def from_pretrained(  # type: ignore[override]
+        cls, pretrained_model_name_or_path: str, *model_args: Any, **kwargs: Any
+    ) -> "SaniGecModel":
+        """
+        Load the model and, when the repo ships a vocab.json, rebuild and attach the decode scheme
+        from it (re-deriving the config label maps), so inference needs no external vocabulary. A
+        missing vocab.json is warned about rather than failed: the model still does forward passes
+        but cannot decode until a scheme is attached. Does not support `output_loading_info=True`.
+        """
+        model = super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        vocab_path = cached_file(
+            pretrained_model_name_or_path,
+            VOCAB_FILE,
+            _raise_exceptions_for_missing_entries=False,
+            _raise_exceptions_for_connection_errors=False,
+        )
+        if vocab_path is not None:
+            model.set_tag_scheme(TagScheme(CharVocabulary.load(Path(vocab_path))))
+        else:
+            logger.warning(
+                "no %s alongside %s; model loaded without a decode tag scheme (cannot correct "
+                "until one is attached)",
+                VOCAB_FILE,
+                pretrained_model_name_or_path,
+            )
+        return model
+
+    def _init_weights(self, module: nn.Module) -> None:
+        """
+        The PreTrainedModel init hook, narrowed to exactly the two task heads (identity, not type):
+        post_init() runs this over every submodule, but the encoder owns its own T5 init (then
+        loaded weights), so only the fresh heads are (re)initialised — the encoder's structure-aware
+        init is never overwritten by a generic normal().
+        """
+        if module is self.detection_head or module is self.correction_head:
+            assert isinstance(module, nn.Linear)
+            module.weight.data.normal_(mean=0.0, std=0.02)
+            if module.bias is not None:
+                module.bias.data.zero_()
+
+    def gradient_checkpointing_enable(
+        self, gradient_checkpointing_kwargs: dict[str, object] | None = None
+    ) -> None:
+        """
+        Trade encoder activation memory for recomputation. Only the deep ByT5 encoder benefits (its
+        long byte sequences dominate activation memory); the two small heads are not checkpointed,
+        so this delegates straight to the encoder.
+        """
+        self.encoder.gradient_checkpointing_enable(
+            gradient_checkpointing_kwargs=gradient_checkpointing_kwargs
+        )
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        codepoint_segments: torch.Tensor | None = None,
+        detection_labels: torch.Tensor | None = None,
+        correction_labels: torch.Tensor | None = None,
+        instance_weight: torch.Tensor | None = None,
+        max_codepoints: int | None = None,
+    ) -> SaniGecOutput:
+        if codepoint_segments is None:
+            raise ValueError("codepoint_segments is required to pool byte hidden-states")
+        byte_hidden = self.encoder(
+            input_ids=input_ids, attention_mask=attention_mask
+        ).last_hidden_state
+        if max_codepoints is None:
+            # Prefer the label width (host-side, no device sync); fall back to the segment max only
+            # when there are no labels and the caller did not pass the width explicitly.
+            max_codepoints = (
+                detection_labels.shape[1]
+                if detection_labels is not None
+                else int(codepoint_segments.max().item()) + 1
+            )
+        pooled = self.dropout(
+            self.bridge.pool_batch(byte_hidden, codepoint_segments, max_codepoints)
+        )
+        detection_logits = self.detection_head(pooled)
+        correction_logits = self.correction_head(pooled)
+
+        loss = None
+        if detection_labels is not None and correction_labels is not None:
+            batch, length, _ = pooled.shape
+            weights = None
+            if instance_weight is not None:
+                weights = instance_weight.unsqueeze(1).expand(batch, length).reshape(-1)
+            detection_loss = self.detection_loss(
+                detection_logits.reshape(-1, 2), detection_labels.reshape(-1), weights
+            )
+            correction_loss = self.correction_loss(
+                correction_logits.reshape(-1, self.config.num_correction_tags),
+                correction_labels.reshape(-1),
+                weights,
+            )
+            loss = combine_head_losses(
+                detection_loss,
+                correction_loss,
+                self.config.detection_loss_weight,
+                self.config.correction_loss_weight,
+            )
+
+        return SaniGecOutput(
+            loss=loss,
+            detection_logits=detection_logits,
+            correction_logits=correction_logits,
+        )
+
+
+SaniGecConfig.register_for_auto_class()
+SaniGecModel.register_for_auto_class("AutoModel")
+try:
+    AutoConfig.register("sani_gec", SaniGecConfig)
+    AutoModel.register(SaniGecConfig, SaniGecModel)
+except ValueError:
+    # Already registered (the module was imported more than once in this process).
+    pass
+
+# Register the correction pipeline here, not in pipeline.py, so it can bind to SaniGecModel without
+# pipeline.py importing this module (which would be a cycle). Importing the model is enough to make
+# `pipeline("sani-gec", model=…)` resolvable in-process. This in-process registration and the
+# shipped CUSTOM_PIPELINES config block above are two faces of the same wiring (task "sani-gec",
+# AutoModel-backed text pipeline) and must be kept in sync by hand.
+PIPELINE_REGISTRY.register_pipeline(
+    TASK_NAME,
+    pipeline_class=SaniGecCorrectionPipeline,
+    pt_model=SaniGecModel,
+    type="text",
+)
+
+
+__all__ = ["IGNORE_INDEX", "SaniGecConfig", "SaniGecModel", "SaniGecOutput"]

checkpoint-50000/tokenizer_config.json

@@ -0,0 +1,1162 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "1": {
+      "content": "</s>",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "2": {
+      "content": "<unk>",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "259": {
+      "content": "<extra_id_0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "260": {
+      "content": "<extra_id_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "261": {
+      "content": "<extra_id_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "262": {
+      "content": "<extra_id_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "263": {
+      "content": "<extra_id_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "264": {
+      "content": "<extra_id_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "265": {
+      "content": "<extra_id_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "266": {
+      "content": "<extra_id_7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "267": {
+      "content": "<extra_id_8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "268": {
+      "content": "<extra_id_9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "269": {
+      "content": "<extra_id_10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "270": {
+      "content": "<extra_id_11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "271": {
+      "content": "<extra_id_12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "272": {
+      "content": "<extra_id_13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "273": {
+      "content": "<extra_id_14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "274": {
+      "content": "<extra_id_15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "275": {
+      "content": "<extra_id_16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "276": {
+      "content": "<extra_id_17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "277": {
+      "content": "<extra_id_18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "278": {
+      "content": "<extra_id_19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "279": {
+      "content": "<extra_id_20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "280": {
+      "content": "<extra_id_21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "281": {
+      "content": "<extra_id_22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "282": {
+      "content": "<extra_id_23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "283": {
+      "content": "<extra_id_24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "284": {
+      "content": "<extra_id_25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "285": {
+      "content": "<extra_id_26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "286": {
+      "content": "<extra_id_27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "287": {
+      "content": "<extra_id_28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "288": {
+      "content": "<extra_id_29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "289": {
+      "content": "<extra_id_30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "290": {
+      "content": "<extra_id_31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "291": {
+      "content": "<extra_id_32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "292": {
+      "content": "<extra_id_33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "293": {
+      "content": "<extra_id_34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "294": {
+      "content": "<extra_id_35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "295": {
+      "content": "<extra_id_36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "296": {
+      "content": "<extra_id_37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "297": {
+      "content": "<extra_id_38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "298": {
+      "content": "<extra_id_39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "299": {
+      "content": "<extra_id_40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "300": {
+      "content": "<extra_id_41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "301": {
+      "content": "<extra_id_42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "302": {
+      "content": "<extra_id_43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "303": {
+      "content": "<extra_id_44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "304": {
+      "content": "<extra_id_45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "305": {
+      "content": "<extra_id_46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "306": {
+      "content": "<extra_id_47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "307": {
+      "content": "<extra_id_48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "308": {
+      "content": "<extra_id_49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "309": {
+      "content": "<extra_id_50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "310": {
+      "content": "<extra_id_51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "311": {
+      "content": "<extra_id_52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "312": {
+      "content": "<extra_id_53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "313": {
+      "content": "<extra_id_54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "314": {
+      "content": "<extra_id_55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "315": {
+      "content": "<extra_id_56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "316": {
+      "content": "<extra_id_57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "317": {
+      "content": "<extra_id_58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "318": {
+      "content": "<extra_id_59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "319": {
+      "content": "<extra_id_60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "320": {
+      "content": "<extra_id_61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "321": {
+      "content": "<extra_id_62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "322": {
+      "content": "<extra_id_63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "323": {
+      "content": "<extra_id_64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "324": {
+      "content": "<extra_id_65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "325": {
+      "content": "<extra_id_66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "326": {
+      "content": "<extra_id_67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "327": {
+      "content": "<extra_id_68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "328": {
+      "content": "<extra_id_69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "329": {
+      "content": "<extra_id_70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "330": {
+      "content": "<extra_id_71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "331": {
+      "content": "<extra_id_72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "332": {
+      "content": "<extra_id_73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "333": {
+      "content": "<extra_id_74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "334": {
+      "content": "<extra_id_75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "335": {
+      "content": "<extra_id_76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "336": {
+      "content": "<extra_id_77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "337": {
+      "content": "<extra_id_78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "338": {
+      "content": "<extra_id_79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "339": {
+      "content": "<extra_id_80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "340": {
+      "content": "<extra_id_81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "341": {
+      "content": "<extra_id_82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "342": {
+      "content": "<extra_id_83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "343": {
+      "content": "<extra_id_84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "344": {
+      "content": "<extra_id_85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "345": {
+      "content": "<extra_id_86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "346": {
+      "content": "<extra_id_87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "347": {
+      "content": "<extra_id_88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "348": {
+      "content": "<extra_id_89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "349": {
+      "content": "<extra_id_90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "350": {
+      "content": "<extra_id_91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "351": {
+      "content": "<extra_id_92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "352": {
+      "content": "<extra_id_93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "353": {
+      "content": "<extra_id_94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "354": {
+      "content": "<extra_id_95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "355": {
+      "content": "<extra_id_96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "356": {
+      "content": "<extra_id_97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "357": {
+      "content": "<extra_id_98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "358": {
+      "content": "<extra_id_99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "359": {
+      "content": "<extra_id_100>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "360": {
+      "content": "<extra_id_101>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "361": {
+      "content": "<extra_id_102>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "362": {
+      "content": "<extra_id_103>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "363": {
+      "content": "<extra_id_104>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "364": {
+      "content": "<extra_id_105>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "365": {
+      "content": "<extra_id_106>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "366": {
+      "content": "<extra_id_107>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "367": {
+      "content": "<extra_id_108>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "368": {
+      "content": "<extra_id_109>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "369": {
+      "content": "<extra_id_110>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "370": {
+      "content": "<extra_id_111>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "371": {
+      "content": "<extra_id_112>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "372": {
+      "content": "<extra_id_113>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "373": {
+      "content": "<extra_id_114>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "374": {
+      "content": "<extra_id_115>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "375": {
+      "content": "<extra_id_116>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "376": {
+      "content": "<extra_id_117>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "377": {
+      "content": "<extra_id_118>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "378": {
+      "content": "<extra_id_119>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "379": {
+      "content": "<extra_id_120>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "380": {
+      "content": "<extra_id_121>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "381": {
+      "content": "<extra_id_122>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "382": {
+      "content": "<extra_id_123>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "383": {
+      "content": "<extra_id_124>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "backend": "custom",
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "extra_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>",
+    "<extra_id_100>",
+    "<extra_id_101>",
+    "<extra_id_102>",
+    "<extra_id_103>",
+    "<extra_id_104>",
+    "<extra_id_105>",
+    "<extra_id_106>",
+    "<extra_id_107>",
+    "<extra_id_108>",
+    "<extra_id_109>",
+    "<extra_id_110>",
+    "<extra_id_111>",
+    "<extra_id_112>",
+    "<extra_id_113>",
+    "<extra_id_114>",
+    "<extra_id_115>",
+    "<extra_id_116>",
+    "<extra_id_117>",
+    "<extra_id_118>",
+    "<extra_id_119>",
+    "<extra_id_120>",
+    "<extra_id_121>",
+    "<extra_id_122>",
+    "<extra_id_123>",
+    "<extra_id_124>"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "ByT5Tokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-50000/trainer_state.json

@@ -0,0 +1,1294 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.8981985915225603,
+  "eval_steps": 2500,
+  "global_step": 50000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01898217573698297,
+      "grad_norm": 2.8858866691589355,
+      "learning_rate": 4.735843087630496e-05,
+      "loss": 6.00573583984375,
+      "step": 500
+    },
+    {
+      "epoch": 0.03796435147396594,
+      "grad_norm": 0.22312745451927185,
+      "learning_rate": 9.481176842771275e-05,
+      "loss": 1.7886568603515625,
+      "step": 1000
+    },
+    {
+      "epoch": 0.05694652721094892,
+      "grad_norm": 0.3883834183216095,
+      "learning_rate": 0.0001422651059791205,
+      "loss": 1.4103663330078124,
+      "step": 1500
+    },
+    {
+      "epoch": 0.07592870294793189,
+      "grad_norm": 0.28461548686027527,
+      "learning_rate": 0.0001897184435305283,
+      "loss": 1.2999722900390625,
+      "step": 2000
+    },
+    {
+      "epoch": 0.09491087868491487,
+      "grad_norm": 0.3026827871799469,
+      "learning_rate": 0.00023717178108193608,
+      "loss": 1.27368310546875,
+      "step": 2500
+    },
+    {
+      "epoch": 0.09491087868491487,
+      "eval_correction_accuracy": 0.9691768119898284,
+      "eval_detection_f1": 0.5029481803464545,
+      "eval_loss": 1.2979440689086914,
+      "eval_runtime": 60.3464,
+      "eval_samples_per_second": 371.886,
+      "eval_steps_per_second": 2.916,
+      "step": 2500
+    },
+    {
+      "epoch": 0.09491087868491487,
+      "eval_edit_f0_5": 0.8420576085111199,
+      "eval_edit_f0_5_m0": 0.8057950832842375,
+      "eval_edit_f0_5_m0.1": 0.8175929454030809,
+      "eval_edit_f0_5_m0.2": 0.8316255884796455,
+      "eval_edit_f0_5_m0.3": 0.8420576085111199,
+      "step": 2500
+    },
+    {
+      "epoch": 0.09491087868491487,
+      "eval_real_f0_5": 0.42910447761194037,
+      "eval_real_f0_5_m0": 0.3888400702987698,
+      "eval_real_f0_5_m0.1": 0.4028041825095057,
+      "eval_real_f0_5_m0.2": 0.4175050301810866,
+      "eval_real_f0_5_m0.3": 0.42910447761194037,
+      "step": 2500
+    },
+    {
+      "epoch": 0.11389305442189784,
+      "grad_norm": 0.27056726813316345,
+      "learning_rate": 0.00028462511863334386,
+      "loss": 1.2630726318359375,
+      "step": 3000
+    },
+    {
+      "epoch": 0.1328752301588808,
+      "grad_norm": 0.2008272260427475,
+      "learning_rate": 0.00029996551748314967,
+      "loss": 1.2586767578125,
+      "step": 3500
+    },
+    {
+      "epoch": 0.15185740589586377,
+      "grad_norm": 0.23608435690402985,
+      "learning_rate": 0.00029978808195120705,
+      "loss": 1.2485474853515626,
+      "step": 4000
+    },
+    {
+      "epoch": 0.17083958163284677,
+      "grad_norm": 0.17198926210403442,
+      "learning_rate": 0.00029945995039741313,
+      "loss": 1.2438245849609375,
+      "step": 4500
+    },
+    {
+      "epoch": 0.18982175736982973,
+      "grad_norm": 0.20551930367946625,
+      "learning_rate": 0.00029898145294228977,
+      "loss": 1.2398516845703125,
+      "step": 5000
+    },
+    {
+      "epoch": 0.18982175736982973,
+      "eval_correction_accuracy": 0.9716923312269569,
+      "eval_detection_f1": 0.5269904148108618,
+      "eval_loss": 1.2654461860656738,
+      "eval_runtime": 59.8792,
+      "eval_samples_per_second": 374.788,
+      "eval_steps_per_second": 2.939,
+      "step": 5000
+    },
+    {
+      "epoch": 0.18982175736982973,
+      "eval_edit_f0_5": 0.872664843158936,
+      "eval_edit_f0_5_m0": 0.8547840106743758,
+      "eval_edit_f0_5_m0.1": 0.8604271144418775,
+      "eval_edit_f0_5_m0.2": 0.8685822219678324,
+      "eval_edit_f0_5_m0.3": 0.872664843158936,
+      "step": 5000
+    },
+    {
+      "epoch": 0.18982175736982973,
+      "eval_real_f0_5": 0.46151439299123903,
+      "eval_real_f0_5_m0": 0.4191919191919192,
+      "eval_real_f0_5_m0.1": 0.4297297297297297,
+      "eval_real_f0_5_m0.2": 0.445906432748538,
+      "eval_real_f0_5_m0.3": 0.46151439299123903,
+      "step": 5000
+    },
+    {
+      "epoch": 0.2088039331068127,
+      "grad_norm": 0.20124033093452454,
+      "learning_rate": 0.00029835307098370265,
+      "loss": 1.2357362060546875,
+      "step": 5500
+    },
+    {
+      "epoch": 0.2277861088437957,
+      "grad_norm": 0.23115237057209015,
+      "learning_rate": 0.0002975754367125453,
+      "loss": 1.2375308837890624,
+      "step": 6000
+    },
+    {
+      "epoch": 0.24676828458077865,
+      "grad_norm": 0.1954943835735321,
+      "learning_rate": 0.00029664933247671615,
+      "loss": 1.2349964599609375,
+      "step": 6500
+    },
+    {
+      "epoch": 0.2657504603177616,
+      "grad_norm": 0.17400752007961273,
+      "learning_rate": 0.0002955756899940283,
+      "loss": 1.2303714599609374,
+      "step": 7000
+    },
+    {
+      "epoch": 0.2847326360547446,
+      "grad_norm": 0.17888489365577698,
+      "learning_rate": 0.00029435558941484435,
+      "loss": 1.2309805908203124,
+      "step": 7500
+    },
+    {
+      "epoch": 0.2847326360547446,
+      "eval_correction_accuracy": 0.9723985228660039,
+      "eval_detection_f1": 0.5397412768051171,
+      "eval_loss": 1.2546594142913818,
+      "eval_runtime": 60.3599,
+      "eval_samples_per_second": 371.803,
+      "eval_steps_per_second": 2.916,
+      "step": 7500
+    },
+    {
+      "epoch": 0.2847326360547446,
+      "eval_edit_f0_5": 0.8861485517636936,
+      "eval_edit_f0_5_m0": 0.8749653451621846,
+      "eval_edit_f0_5_m0.1": 0.8788065728226123,
+      "eval_edit_f0_5_m0.2": 0.8814529381764608,
+      "eval_edit_f0_5_m0.3": 0.8861485517636936,
+      "step": 7500
+    },
+    {
+      "epoch": 0.2847326360547446,
+      "eval_real_f0_5": 0.47371638141809297,
+      "eval_real_f0_5_m0": 0.4501055966209081,
+      "eval_real_f0_5_m0.1": 0.4613259668508288,
+      "eval_real_f0_5_m0.2": 0.47164351851851855,
+      "eval_real_f0_5_m0.3": 0.47371638141809297,
+      "step": 7500
+    },
+    {
+      "epoch": 0.30371481179172755,
+      "grad_norm": 0.1810256987810135,
+      "learning_rate": 0.0002929902582353787,
+      "loss": 1.227394775390625,
+      "step": 8000
+    },
+    {
+      "epoch": 0.32269698752871057,
+      "grad_norm": 0.18248461186885834,
+      "learning_rate": 0.00029148107006276056,
+      "loss": 1.22420361328125,
+      "step": 8500
+    },
+    {
+      "epoch": 0.34167916326569353,
+      "grad_norm": 0.1456211358308792,
+      "learning_rate": 0.0002898295432331011,
+      "loss": 1.22528125,
+      "step": 9000
+    },
+    {
+      "epoch": 0.3606613390026765,
+      "grad_norm": 0.1682305634021759,
+      "learning_rate": 0.0002880373392839537,
+      "loss": 1.2209486083984376,
+      "step": 9500
+    },
+    {
+      "epoch": 0.37964351473965946,
+      "grad_norm": 0.1455032080411911,
+      "learning_rate": 0.00028610626128270495,
+      "loss": 1.222905029296875,
+      "step": 10000
+    },
+    {
+      "epoch": 0.37964351473965946,
+      "eval_correction_accuracy": 0.9729365736386112,
+      "eval_detection_f1": 0.5878967328292231,
+      "eval_loss": 1.2507761716842651,
+      "eval_runtime": 60.7194,
+      "eval_samples_per_second": 369.602,
+      "eval_steps_per_second": 2.899,
+      "step": 10000
+    },
+    {
+      "epoch": 0.37964351473965946,
+      "eval_edit_f0_5": 0.8773855767606823,
+      "eval_edit_f0_5_m0": 0.8642143753752115,
+      "eval_edit_f0_5_m0.1": 0.871961638097338,
+      "eval_edit_f0_5_m0.2": 0.8739938939772413,
+      "eval_edit_f0_5_m0.3": 0.8773855767606823,
+      "step": 10000
+    },
+    {
+      "epoch": 0.37964351473965946,
+      "eval_real_f0_5": 0.43992027334851935,
+      "eval_real_f0_5_m0": 0.4214697406340058,
+      "eval_real_f0_5_m0.1": 0.42792792792792794,
+      "eval_real_f0_5_m0.2": 0.42728237791932056,
+      "eval_real_f0_5_m0.3": 0.43992027334851935,
+      "step": 10000
+    },
+    {
+      "epoch": 0.3986256904766424,
+      "grad_norm": 0.20864084362983704,
+      "learning_rate": 0.0002840382520125783,
+      "loss": 1.22280712890625,
+      "step": 10500
+    },
+    {
+      "epoch": 0.4176078662136254,
+      "grad_norm": 0.19003528356552124,
+      "learning_rate": 0.0002818353920180744,
+      "loss": 1.221659423828125,
+      "step": 11000
+    },
+    {
+      "epoch": 0.43659004195060835,
+      "grad_norm": 0.1750059872865677,
+      "learning_rate": 0.0002794998975118153,
+      "loss": 1.2191646728515626,
+      "step": 11500
+    },
+    {
+      "epoch": 0.4555722176875914,
+      "grad_norm": 0.13223567605018616,
+      "learning_rate": 0.00027703411814489855,
+      "loss": 1.216736328125,
+      "step": 12000
+    },
+    {
+      "epoch": 0.47455439342457434,
+      "grad_norm": 0.10733171552419662,
+      "learning_rate": 0.00027444053464300346,
+      "loss": 1.2142664794921876,
+      "step": 12500
+    },
+    {
+      "epoch": 0.47455439342457434,
+      "eval_correction_accuracy": 0.9733371445263059,
+      "eval_detection_f1": 0.5704304192239259,
+      "eval_loss": 1.2477121353149414,
+      "eval_runtime": 60.6105,
+      "eval_samples_per_second": 370.266,
+      "eval_steps_per_second": 2.904,
+      "step": 12500
+    },
+    {
+      "epoch": 0.47455439342457434,
+      "eval_edit_f0_5": 0.8926609218668673,
+      "eval_edit_f0_5_m0": 0.8845489496851842,
+      "eval_edit_f0_5_m0.1": 0.8883563574851988,
+      "eval_edit_f0_5_m0.2": 0.89086859688196,
+      "eval_edit_f0_5_m0.3": 0.8926609218668673,
+      "step": 12500
+    },
+    {
+      "epoch": 0.47455439342457434,
+      "eval_real_f0_5": 0.4787581699346405,
+      "eval_real_f0_5_m0": 0.43419633225458465,
+      "eval_real_f0_5_m0.1": 0.44776119402985065,
+      "eval_real_f0_5_m0.2": 0.46760391198044005,
+      "eval_real_f0_5_m0.3": 0.4787581699346405,
+      "step": 12500
+    },
+    {
+      "epoch": 0.4935365691615573,
+      "grad_norm": 0.1579447090625763,
+      "learning_rate": 0.0002717217563106287,
+      "loss": 1.21451904296875,
+      "step": 13000
+    },
+    {
+      "epoch": 0.5125187448985403,
+      "grad_norm": 0.13867036998271942,
+      "learning_rate": 0.00026888051840597135,
+      "loss": 1.2117508544921876,
+      "step": 13500
+    },
+    {
+      "epoch": 0.5315009206355232,
+      "grad_norm": 0.11431417614221573,
+      "learning_rate": 0.00026591967938908897,
+      "loss": 1.2146973876953124,
+      "step": 14000
+    },
+    {
+      "epoch": 0.5504830963725063,
+      "grad_norm": 0.14605163037776947,
+      "learning_rate": 0.00026284221804611323,
+      "loss": 1.211010498046875,
+      "step": 14500
+    },
+    {
+      "epoch": 0.5694652721094892,
+      "grad_norm": 0.13300460577011108,
+      "learning_rate": 0.0002596512304924075,
+      "loss": 1.2157716064453126,
+      "step": 15000
+    },
+    {
+      "epoch": 0.5694652721094892,
+      "eval_correction_accuracy": 0.9736186980555992,
+      "eval_detection_f1": 0.6009406304526895,
+      "eval_loss": 1.2431864738464355,
+      "eval_runtime": 60.2876,
+      "eval_samples_per_second": 372.249,
+      "eval_steps_per_second": 2.919,
+      "step": 15000
+    },
+    {
+      "epoch": 0.5694652721094892,
+      "eval_edit_f0_5": 0.8888329698596432,
+      "eval_edit_f0_5_m0": 0.8778231292517006,
+      "eval_edit_f0_5_m0.1": 0.8819654131210541,
+      "eval_edit_f0_5_m0.2": 0.8855925167432335,
+      "eval_edit_f0_5_m0.3": 0.8888329698596432,
+      "step": 15000
+    },
+    {
+      "epoch": 0.5694652721094892,
+      "eval_real_f0_5": 0.4537767756482525,
+      "eval_real_f0_5_m0": 0.4154693486590038,
+      "eval_real_f0_5_m0.1": 0.42251755265797397,
+      "eval_real_f0_5_m0.2": 0.4458512931034483,
+      "eval_real_f0_5_m0.3": 0.4537767756482525,
+      "step": 15000
+    },
+    {
+      "epoch": 0.5884474478464722,
+      "grad_norm": 0.16023032367229462,
+      "learning_rate": 0.0002563499270576846,
+      "loss": 1.21418017578125,
+      "step": 15500
+    },
+    {
+      "epoch": 0.6074296235834551,
+      "grad_norm": 0.14038224518299103,
+      "learning_rate": 0.0002529416290562178,
+      "loss": 1.2102242431640624,
+      "step": 16000
+    },
+    {
+      "epoch": 0.6264117993204381,
+      "grad_norm": 0.19150033593177795,
+      "learning_rate": 0.00024942976544539404,
+      "loss": 1.21311474609375,
+      "step": 16500
+    },
+    {
+      "epoch": 0.6453939750574211,
+      "grad_norm": 0.13594546914100647,
+      "learning_rate": 0.0002458178693759725,
+      "loss": 1.2101019287109376,
+      "step": 17000
+    },
+    {
+      "epoch": 0.664376150794404,
+      "grad_norm": 0.0971856564283371,
+      "learning_rate": 0.00024210957463751702,
+      "loss": 1.2085975341796875,
+      "step": 17500
+    },
+    {
+      "epoch": 0.664376150794404,
+      "eval_correction_accuracy": 0.9739441000853563,
+      "eval_detection_f1": 0.6041727242827385,
+      "eval_loss": 1.2378469705581665,
+      "eval_runtime": 60.7237,
+      "eval_samples_per_second": 369.575,
+      "eval_steps_per_second": 2.898,
+      "step": 17500
+    },
+    {
+      "epoch": 0.664376150794404,
+      "eval_edit_f0_5": 0.8950703830407717,
+      "eval_edit_f0_5_m0": 0.8823850308221047,
+      "eval_edit_f0_5_m0.1": 0.8891276795062545,
+      "eval_edit_f0_5_m0.2": 0.891679139641887,
+      "eval_edit_f0_5_m0.3": 0.8950703830407717,
+      "step": 17500
+    },
+    {
+      "epoch": 0.664376150794404,
+      "eval_real_f0_5": 0.45221843003412976,
+      "eval_real_f0_5_m0": 0.4207080504364695,
+      "eval_real_f0_5_m0.1": 0.4268916155419223,
+      "eval_real_f0_5_m0.2": 0.43951612903225806,
+      "eval_real_f0_5_m0.3": 0.45221843003412976,
+      "step": 17500
+    },
+    {
+      "epoch": 0.6833583265313871,
+      "grad_norm": 0.23436909914016724,
+      "learning_rate": 0.00023830861200258084,
+      "loss": 1.2103984375,
+      "step": 18000
+    },
+    {
+      "epoch": 0.70234050226837,
+      "grad_norm": 0.10302391648292542,
+      "learning_rate": 0.00023441880547331984,
+      "loss": 1.209429443359375,
+      "step": 18500
+    },
+    {
+      "epoch": 0.721322678005353,
+      "grad_norm": 0.16614890098571777,
+      "learning_rate": 0.00023044406843431085,
+      "loss": 1.210209228515625,
+      "step": 19000
+    },
+    {
+      "epoch": 0.7403048537423359,
+      "grad_norm": 0.12630845606327057,
+      "learning_rate": 0.00022638839971544675,
+      "loss": 1.2063096923828125,
+      "step": 19500
+    },
+    {
+      "epoch": 0.7592870294793189,
+      "grad_norm": 0.16276787221431732,
+      "learning_rate": 0.00022225587956886718,
+      "loss": 1.20748486328125,
+      "step": 20000
+    },
+    {
+      "epoch": 0.7592870294793189,
+      "eval_correction_accuracy": 0.974335110021822,
+      "eval_detection_f1": 0.6143125770798066,
+      "eval_loss": 1.234763264656067,
+      "eval_runtime": 60.5996,
+      "eval_samples_per_second": 370.332,
+      "eval_steps_per_second": 2.904,
+      "step": 20000
+    },
+    {
+      "epoch": 0.7592870294793189,
+      "eval_edit_f0_5": 0.9024403864410565,
+      "eval_edit_f0_5_m0": 0.8842469830618539,
+      "eval_edit_f0_5_m0.1": 0.8886223627541842,
+      "eval_edit_f0_5_m0.2": 0.8945219837810999,
+      "eval_edit_f0_5_m0.3": 0.9024403864410565,
+      "step": 20000
+    },
+    {
+      "epoch": 0.7592870294793189,
+      "eval_real_f0_5": 0.4630195599022005,
+      "eval_real_f0_5_m0": 0.44520547945205474,
+      "eval_real_f0_5_m0.1": 0.4467213114754099,
+      "eval_real_f0_5_m0.2": 0.4604503464203233,
+      "eval_real_f0_5_m0.3": 0.4630195599022005,
+      "step": 20000
+    },
+    {
+      "epoch": 0.7782692052163019,
+      "grad_norm": 0.16368603706359863,
+      "learning_rate": 0.0002180506655639742,
+      "loss": 1.20710107421875,
+      "step": 20500
+    },
+    {
+      "epoch": 0.7972513809532848,
+      "grad_norm": 0.1419440507888794,
+      "learning_rate": 0.00021377698840466171,
+      "loss": 1.207209716796875,
+      "step": 21000
+    },
+    {
+      "epoch": 0.8162335566902679,
+      "grad_norm": 0.2440534383058548,
+      "learning_rate": 0.00020943914767296716,
+      "loss": 1.203267333984375,
+      "step": 21500
+    },
+    {
+      "epoch": 0.8352157324272508,
+      "grad_norm": 0.11360731720924377,
+      "learning_rate": 0.00020504150750342767,
+      "loss": 1.2064322509765626,
+      "step": 22000
+    },
+    {
+      "epoch": 0.8541979081642338,
+      "grad_norm": 0.19054335355758667,
+      "learning_rate": 0.00020058849219249225,
+      "loss": 1.2044508056640626,
+      "step": 22500
+    },
+    {
+      "epoch": 0.8541979081642338,
+      "eval_correction_accuracy": 0.9746169932390887,
+      "eval_detection_f1": 0.6164803461954562,
+      "eval_loss": 1.2325071096420288,
+      "eval_runtime": 60.5986,
+      "eval_samples_per_second": 370.339,
+      "eval_steps_per_second": 2.904,
+      "step": 22500
+    },
+    {
+      "epoch": 0.8541979081642338,
+      "eval_edit_f0_5": 0.9041808921475346,
+      "eval_edit_f0_5_m0": 0.8936088786835056,
+      "eval_edit_f0_5_m0.1": 0.8979367262723521,
+      "eval_edit_f0_5_m0.2": 0.9006915629322267,
+      "eval_edit_f0_5_m0.3": 0.9041808921475346,
+      "step": 22500
+    },
+    {
+      "epoch": 0.8541979081642338,
+      "eval_real_f0_5": 0.43545659526493796,
+      "eval_real_f0_5_m0": 0.4005456349206349,
+      "eval_real_f0_5_m0.1": 0.4166666666666667,
+      "eval_real_f0_5_m0.2": 0.4315960912052117,
+      "eval_real_f0_5_m0.3": 0.43545659526493796,
+      "step": 22500
+    },
+    {
+      "epoch": 0.8731800839012167,
+      "grad_norm": 0.15163150429725647,
+      "learning_rate": 0.00019608458174740764,
+      "loss": 1.205394287109375,
+      "step": 23000
+    },
+    {
+      "epoch": 0.8921622596381997,
+      "grad_norm": 0.1204524040222168,
+      "learning_rate": 0.00019153430737905516,
+      "loss": 1.20386767578125,
+      "step": 23500
+    },
+    {
+      "epoch": 0.9111444353751827,
+      "grad_norm": 0.14975769817829132,
+      "learning_rate": 0.00018694224694327434,
+      "loss": 1.2036343994140626,
+      "step": 24000
+    },
+    {
+      "epoch": 0.9301266111121657,
+      "grad_norm": 0.09973002225160599,
+      "learning_rate": 0.0001823130203352581,
+      "loss": 1.202430419921875,
+      "step": 24500
+    },
+    {
+      "epoch": 0.9491087868491487,
+      "grad_norm": 0.12494324147701263,
+      "learning_rate": 0.00017765128484165446,
+      "loss": 1.203585205078125,
+      "step": 25000
+    },
+    {
+      "epoch": 0.9491087868491487,
+      "eval_correction_accuracy": 0.9748879967532328,
+      "eval_detection_f1": 0.6170252572497661,
+      "eval_loss": 1.2309163808822632,
+      "eval_runtime": 59.979,
+      "eval_samples_per_second": 374.164,
+      "eval_steps_per_second": 2.934,
+      "step": 25000
+    },
+    {
+      "epoch": 0.9491087868491487,
+      "eval_edit_f0_5": 0.9054563883101525,
+      "eval_edit_f0_5_m0": 0.8931397696883698,
+      "eval_edit_f0_5_m0.1": 0.8968065743753791,
+      "eval_edit_f0_5_m0.2": 0.9016347709646821,
+      "eval_edit_f0_5_m0.3": 0.9054563883101525,
+      "step": 25000
+    },
+    {
+      "epoch": 0.9491087868491487,
+      "eval_real_f0_5": 0.46703296703296693,
+      "eval_real_f0_5_m0": 0.43244626407369496,
+      "eval_real_f0_5_m0.1": 0.44786096256684493,
+      "eval_real_f0_5_m0.2": 0.45962199312714774,
+      "eval_real_f0_5_m0.3": 0.46703296703296693,
+      "step": 25000
+    },
+    {
+      "epoch": 0.9680909625861316,
+      "grad_norm": 0.14670301973819733,
+      "learning_rate": 0.00017296173045504914,
+      "loss": 1.2033277587890625,
+      "step": 25500
+    },
+    {
+      "epoch": 0.9870731383231146,
+      "grad_norm": 0.10602114349603653,
+      "learning_rate": 0.00016824907515554526,
+      "loss": 1.201916748046875,
+      "step": 26000
+    },
+    {
+      "epoch": 1.0060363318843606,
+      "grad_norm": 0.06282790750265121,
+      "learning_rate": 0.00016351806016418486,
+      "loss": 1.199654541015625,
+      "step": 26500
+    },
+    {
+      "epoch": 1.0250185076213436,
+      "grad_norm": 0.06489919126033783,
+      "learning_rate": 0.00015877344517298893,
+      "loss": 1.19592431640625,
+      "step": 27000
+    },
+    {
+      "epoch": 1.0440006833583266,
+      "grad_norm": 0.06663327664136887,
+      "learning_rate": 0.0001540200035564149,
+      "loss": 1.19781689453125,
+      "step": 27500
+    },
+    {
+      "epoch": 1.0440006833583266,
+      "eval_correction_accuracy": 0.9751896612489042,
+      "eval_detection_f1": 0.6337458056595326,
+      "eval_loss": 1.2278345823287964,
+      "eval_runtime": 60.4482,
+      "eval_samples_per_second": 371.26,
+      "eval_steps_per_second": 2.912,
+      "step": 27500
+    },
+    {
+      "epoch": 1.0440006833583266,
+      "eval_edit_f0_5": 0.9052714133661981,
+      "eval_edit_f0_5_m0": 0.8921670023406456,
+      "eval_edit_f0_5_m0.1": 0.896949711459192,
+      "eval_edit_f0_5_m0.2": 0.9007485444968119,
+      "eval_edit_f0_5_m0.3": 0.9052714133661981,
+      "step": 27500
+    },
+    {
+      "epoch": 1.0440006833583266,
+      "eval_real_f0_5": 0.47462941847206386,
+      "eval_real_f0_5_m0": 0.4401750972762646,
+      "eval_real_f0_5_m0.1": 0.46072895277207393,
+      "eval_real_f0_5_m0.2": 0.4667207792207792,
+      "eval_real_f0_5_m0.3": 0.47462941847206386,
+      "step": 27500
+    },
+    {
+      "epoch": 1.0629828590953094,
+      "grad_norm": 0.06707123667001724,
+      "learning_rate": 0.0001492625175690478,
+      "loss": 1.196161865234375,
+      "step": 28000
+    },
+    {
+      "epoch": 1.0819650348322924,
+      "grad_norm": 0.04345293343067169,
+      "learning_rate": 0.00014450577353435873,
+      "loss": 1.1952852783203125,
+      "step": 28500
+    },
+    {
+      "epoch": 1.1009472105692755,
+      "grad_norm": 0.03876705467700958,
+      "learning_rate": 0.00013975455702936845,
+      "loss": 1.19596826171875,
+      "step": 29000
+    },
+    {
+      "epoch": 1.1199293863062585,
+      "grad_norm": 0.054508332163095474,
+      "learning_rate": 0.00013501364807006326,
+      "loss": 1.1936068115234375,
+      "step": 29500
+    },
+    {
+      "epoch": 1.1389115620432415,
+      "grad_norm": 0.05647437274456024,
+      "learning_rate": 0.0001302878163024043,
+      "loss": 1.19433984375,
+      "step": 30000
+    },
+    {
+      "epoch": 1.1389115620432415,
+      "eval_correction_accuracy": 0.9752921942086351,
+      "eval_detection_f1": 0.6477482980155775,
+      "eval_loss": 1.2278577089309692,
+      "eval_runtime": 59.9225,
+      "eval_samples_per_second": 374.517,
+      "eval_steps_per_second": 2.937,
+      "step": 30000
+    },
+    {
+      "epoch": 1.1389115620432415,
+      "eval_edit_f0_5": 0.9029944709038156,
+      "eval_edit_f0_5_m0": 0.8870581982460802,
+      "eval_edit_f0_5_m0.1": 0.8917932478127851,
+      "eval_edit_f0_5_m0.2": 0.8975887293416418,
+      "eval_edit_f0_5_m0.3": 0.9029944709038156,
+      "step": 30000
+    },
+    {
+      "epoch": 1.1389115620432415,
+      "eval_real_f0_5": 0.46045785639958375,
+      "eval_real_f0_5_m0": 0.411169744942832,
+      "eval_real_f0_5_m0.1": 0.4227605118829982,
+      "eval_real_f0_5_m0.2": 0.4366602687140115,
+      "eval_real_f0_5_m0.3": 0.46045785639958375,
+      "step": 30000
+    },
+    {
+      "epoch": 1.1578937377802243,
+      "grad_norm": 0.08290459215641022,
+      "learning_rate": 0.00012558181620377085,
+      "loss": 1.196980224609375,
+      "step": 30500
+    },
+    {
+      "epoch": 1.1768759135172073,
+      "grad_norm": 0.03389839455485344,
+      "learning_rate": 0.00012090038229966375,
+      "loss": 1.19385400390625,
+      "step": 31000
+    },
+    {
+      "epoch": 1.1958580892541903,
+      "grad_norm": 0.03219689056277275,
+      "learning_rate": 0.0001162482244004811,
+      "loss": 1.1932216796875,
+      "step": 31500
+    },
+    {
+      "epoch": 1.2148402649911734,
+      "grad_norm": 0.03629528358578682,
+      "learning_rate": 0.00011163002286315993,
+      "loss": 1.1932420654296876,
+      "step": 32000
+    },
+    {
+      "epoch": 1.2338224407281562,
+      "grad_norm": 0.05208583176136017,
+      "learning_rate": 0.00010705042388244893,
+      "loss": 1.1948599853515625,
+      "step": 32500
+    },
+    {
+      "epoch": 1.2338224407281562,
+      "eval_correction_accuracy": 0.9756340806370627,
+      "eval_detection_f1": 0.6500135636133466,
+      "eval_loss": 1.2252331972122192,
+      "eval_runtime": 60.3519,
+      "eval_samples_per_second": 371.852,
+      "eval_steps_per_second": 2.916,
+      "step": 32500
+    },
+    {
+      "epoch": 1.2338224407281562,
+      "eval_edit_f0_5": 0.9102684230832827,
+      "eval_edit_f0_5_m0": 0.8981327019318732,
+      "eval_edit_f0_5_m0.1": 0.9021497806898794,
+      "eval_edit_f0_5_m0.2": 0.9072063717200373,
+      "eval_edit_f0_5_m0.3": 0.9102684230832827,
+      "step": 32500
+    },
+    {
+      "epoch": 1.2338224407281562,
+      "eval_real_f0_5": 0.4674586776859504,
+      "eval_real_f0_5_m0": 0.4159212880143113,
+      "eval_real_f0_5_m0.1": 0.4346804511278195,
+      "eval_real_f0_5_m0.2": 0.4519704433497537,
+      "eval_real_f0_5_m0.3": 0.4674586776859504,
+      "step": 32500
+    },
+    {
+      "epoch": 1.2528046164651392,
+      "grad_norm": 0.06928142160177231,
+      "learning_rate": 0.00010251403481655096,
+      "loss": 1.1938966064453125,
+      "step": 33000
+    },
+    {
+      "epoch": 1.2717867922021222,
+      "grad_norm": 0.05749162659049034,
+      "learning_rate": 9.80254195518373e-05,
+      "loss": 1.1921290283203125,
+      "step": 33500
+    },
+    {
+      "epoch": 1.2907689679391052,
+      "grad_norm": 0.04986289143562317,
+      "learning_rate": 9.358909391129737e-05,
+      "loss": 1.19255517578125,
+      "step": 34000
+    },
+    {
+      "epoch": 1.3097511436760882,
+      "grad_norm": 0.1535189300775528,
+      "learning_rate": 8.920952111134303e-05,
+      "loss": 1.19104052734375,
+      "step": 34500
+    },
+    {
+      "epoch": 1.328733319413071,
+      "grad_norm": 0.042191170156002045,
+      "learning_rate": 8.489110727153841e-05,
+      "loss": 1.191498779296875,
+      "step": 35000
+    },
+    {
+      "epoch": 1.328733319413071,
+      "eval_correction_accuracy": 0.9758862919367224,
+      "eval_detection_f1": 0.6438216536684808,
+      "eval_loss": 1.2241498231887817,
+      "eval_runtime": 59.6334,
+      "eval_samples_per_second": 376.333,
+      "eval_steps_per_second": 2.951,
+      "step": 35000
+    },
+    {
+      "epoch": 1.328733319413071,
+      "eval_edit_f0_5": 0.9100482181455414,
+      "eval_edit_f0_5_m0": 0.9021638917511796,
+      "eval_edit_f0_5_m0.1": 0.9037821317469846,
+      "eval_edit_f0_5_m0.2": 0.9073168050134681,
+      "eval_edit_f0_5_m0.3": 0.9100482181455414,
+      "step": 35000
+    },
+    {
+      "epoch": 1.328733319413071,
+      "eval_real_f0_5": 0.4544167610419026,
+      "eval_real_f0_5_m0": 0.4211689587426326,
+      "eval_real_f0_5_m0.1": 0.42387983706720966,
+      "eval_real_f0_5_m0.2": 0.4281914893617021,
+      "eval_real_f0_5_m0.3": 0.4544167610419026,
+      "step": 35000
+    },
+    {
+      "epoch": 1.347715495150054,
+      "grad_norm": 0.029605668038129807,
+      "learning_rate": 8.063819698177278e-05,
+      "loss": 1.192787841796875,
+      "step": 35500
+    },
+    {
+      "epoch": 1.366697670887037,
+      "grad_norm": 0.048531949520111084,
+      "learning_rate": 7.645506893133574e-05,
+      "loss": 1.190103759765625,
+      "step": 36000
+    },
+    {
+      "epoch": 1.38567984662402,
+      "grad_norm": 0.03906732425093651,
+      "learning_rate": 7.234593160429277e-05,
+      "loss": 1.18997265625,
+      "step": 36500
+    },
+    {
+      "epoch": 1.404662022361003,
+      "grad_norm": 0.03661827743053436,
+      "learning_rate": 6.831491904549145e-05,
+      "loss": 1.191602294921875,
+      "step": 37000
+    },
+    {
+      "epoch": 1.423644198097986,
+      "grad_norm": 0.043553389608860016,
+      "learning_rate": 6.436608670145828e-05,
+      "loss": 1.1902423095703125,
+      "step": 37500
+    },
+    {
+      "epoch": 1.423644198097986,
+      "eval_correction_accuracy": 0.9760330030848904,
+      "eval_detection_f1": 0.6487232574189096,
+      "eval_loss": 1.222518801689148,
+      "eval_runtime": 60.4683,
+      "eval_samples_per_second": 371.137,
+      "eval_steps_per_second": 2.911,
+      "step": 37500
+    },
+    {
+      "epoch": 1.423644198097986,
+      "eval_edit_f0_5": 0.9084162497938542,
+      "eval_edit_f0_5_m0": 0.8946242310778283,
+      "eval_edit_f0_5_m0.1": 0.9003717872730211,
+      "eval_edit_f0_5_m0.2": 0.9063674492074731,
+      "eval_edit_f0_5_m0.3": 0.9084162497938542,
+      "step": 37500
+    },
+    {
+      "epoch": 1.423644198097986,
+      "eval_real_f0_5": 0.44614147909967844,
+      "eval_real_f0_5_m0": 0.4033728350045579,
+      "eval_real_f0_5_m0.1": 0.4206963249516441,
+      "eval_real_f0_5_m0.2": 0.4333333333333333,
+      "eval_real_f0_5_m0.3": 0.44614147909967844,
+      "step": 37500
+    },
+    {
+      "epoch": 1.442626373834969,
+      "grad_norm": 0.045576661825180054,
+      "learning_rate": 6.050340734036973e-05,
+      "loss": 1.1904107666015624,
+      "step": 38000
+    },
+    {
+      "epoch": 1.461608549571952,
+      "grad_norm": 0.04765897989273071,
+      "learning_rate": 5.673076705520346e-05,
+      "loss": 1.1911429443359376,
+      "step": 38500
+    },
+    {
+      "epoch": 1.480590725308935,
+      "grad_norm": 0.10767372697591782,
+      "learning_rate": 5.3051961354089736e-05,
+      "loss": 1.1882059326171874,
+      "step": 39000
+    },
+    {
+      "epoch": 1.499572901045918,
+      "grad_norm": 0.036085955798625946,
+      "learning_rate": 4.94706913417966e-05,
+      "loss": 1.190911376953125,
+      "step": 39500
+    },
+    {
+      "epoch": 1.5185550767829008,
+      "grad_norm": 0.08071824908256531,
+      "learning_rate": 4.599055999619126e-05,
+      "loss": 1.1896280517578126,
+      "step": 40000
+    },
+    {
+      "epoch": 1.5185550767829008,
+      "eval_correction_accuracy": 0.976151361067345,
+      "eval_detection_f1": 0.6554347875581136,
+      "eval_loss": 1.2218546867370605,
+      "eval_runtime": 60.355,
+      "eval_samples_per_second": 371.833,
+      "eval_steps_per_second": 2.916,
+      "step": 40000
+    },
+    {
+      "epoch": 1.5185550767829008,
+      "eval_edit_f0_5": 0.910992458853966,
+      "eval_edit_f0_5_m0": 0.9010734128054373,
+      "eval_edit_f0_5_m0.1": 0.9040790831568084,
+      "eval_edit_f0_5_m0.2": 0.9071533963604569,
+      "eval_edit_f0_5_m0.3": 0.910992458853966,
+      "step": 40000
+    },
+    {
+      "epoch": 1.5185550767829008,
+      "eval_real_f0_5": 0.44711014176663033,
+      "eval_real_f0_5_m0": 0.4186210131332083,
+      "eval_real_f0_5_m0.1": 0.4342492639842983,
+      "eval_real_f0_5_m0.2": 0.4353493222106361,
+      "eval_real_f0_5_m0.3": 0.44711014176663033,
+      "step": 40000
+    },
+    {
+      "epoch": 1.5375372525198838,
+      "grad_norm": 0.030372846871614456,
+      "learning_rate": 4.2615068543422434e-05,
+      "loss": 1.19057763671875,
+      "step": 40500
+    },
+    {
+      "epoch": 1.5565194282568668,
+      "grad_norm": 0.03362102061510086,
+      "learning_rate": 3.934761293547178e-05,
+      "loss": 1.1867962646484376,
+      "step": 41000
+    },
+    {
+      "epoch": 1.5755016039938496,
+      "grad_norm": 0.057195816189050674,
+      "learning_rate": 3.619148043361715e-05,
+      "loss": 1.1873057861328125,
+      "step": 41500
+    },
+    {
+      "epoch": 1.5944837797308327,
+      "grad_norm": 0.03772829845547676,
+      "learning_rate": 3.314984630124587e-05,
+      "loss": 1.1879420166015624,
+      "step": 42000
+    },
+    {
+      "epoch": 1.6134659554678157,
+      "grad_norm": 0.1718292534351349,
+      "learning_rate": 3.022577060934427e-05,
+      "loss": 1.1901536865234374,
+      "step": 42500
+    },
+    {
+      "epoch": 1.6134659554678157,
+      "eval_correction_accuracy": 0.9762463112036875,
+      "eval_detection_f1": 0.6603370175756478,
+      "eval_loss": 1.2204469442367554,
+      "eval_runtime": 59.5936,
+      "eval_samples_per_second": 376.584,
+      "eval_steps_per_second": 2.953,
+      "step": 42500
+    },
+    {
+      "epoch": 1.6134659554678157,
+      "eval_edit_f0_5": 0.9098653476229733,
+      "eval_edit_f0_5_m0": 0.8979657559980677,
+      "eval_edit_f0_5_m0.1": 0.9045865598093898,
+      "eval_edit_f0_5_m0.2": 0.9075277034772642,
+      "eval_edit_f0_5_m0.3": 0.9098653476229733,
+      "step": 42500
+    },
+    {
+      "epoch": 1.6134659554678157,
+      "eval_real_f0_5": 0.44262720664589816,
+      "eval_real_f0_5_m0": 0.42281105990783413,
+      "eval_real_f0_5_m0.1": 0.4292809839167455,
+      "eval_real_f0_5_m0.2": 0.43297455968688847,
+      "eval_real_f0_5_m0.3": 0.44262720664589816,
+      "step": 42500
+    },
+    {
+      "epoch": 1.6324481312047987,
+      "grad_norm": 0.027738308534026146,
+      "learning_rate": 2.742219515787808e-05,
+      "loss": 1.186643798828125,
+      "step": 43000
+    },
+    {
+      "epoch": 1.6514303069417817,
+      "grad_norm": 0.03820585086941719,
+      "learning_rate": 2.4741940516161014e-05,
+      "loss": 1.1873551025390625,
+      "step": 43500
+    },
+    {
+      "epoch": 1.6704124826787647,
+      "grad_norm": 0.05109170451760292,
+      "learning_rate": 2.2187703185188212e-05,
+      "loss": 1.1882010498046875,
+      "step": 44000
+    },
+    {
+      "epoch": 1.6893946584157478,
+      "grad_norm": 0.04304545372724533,
+      "learning_rate": 1.9762052884790763e-05,
+      "loss": 1.18957080078125,
+      "step": 44500
+    },
+    {
+      "epoch": 1.7083768341527306,
+      "grad_norm": 0.05135943740606308,
+      "learning_rate": 1.7467429968339254e-05,
+      "loss": 1.1895230712890625,
+      "step": 45000
+    },
+    {
+      "epoch": 1.7083768341527306,
+      "eval_correction_accuracy": 0.9763739004493976,
+      "eval_detection_f1": 0.6614078220399721,
+      "eval_loss": 1.220071792602539,
+      "eval_runtime": 60.3321,
+      "eval_samples_per_second": 371.975,
+      "eval_steps_per_second": 2.917,
+      "step": 45000
+    },
+    {
+      "epoch": 1.7083768341527306,
+      "eval_edit_f0_5": 0.912360661138872,
+      "eval_edit_f0_5_m0": 0.9017536332922186,
+      "eval_edit_f0_5_m0.1": 0.9063933099541408,
+      "eval_edit_f0_5_m0.2": 0.9094366273700224,
+      "eval_edit_f0_5_m0.3": 0.912360661138872,
+      "step": 45000
+    },
+    {
+      "epoch": 1.7083768341527306,
+      "eval_real_f0_5": 0.44325313807531386,
+      "eval_real_f0_5_m0": 0.4201485608170845,
+      "eval_real_f0_5_m0.1": 0.42870905587668595,
+      "eval_real_f0_5_m0.2": 0.4367469879518072,
+      "eval_real_f0_5_m0.3": 0.44325313807531386,
+      "step": 45000
+    },
+    {
+      "epoch": 1.7273590098897136,
+      "grad_norm": 0.03698679432272911,
+      "learning_rate": 1.5306142967598462e-05,
+      "loss": 1.1872025146484375,
+      "step": 45500
+    },
+    {
+      "epoch": 1.7463411856266964,
+      "grad_norm": 0.043697986751794815,
+      "learning_rate": 1.3280366270202187e-05,
+      "loss": 1.1902381591796876,
+      "step": 46000
+    },
+    {
+      "epoch": 1.7653233613636794,
+      "grad_norm": 0.0478336326777935,
+      "learning_rate": 1.1392137932085692e-05,
+      "loss": 1.188957763671875,
+      "step": 46500
+    },
+    {
+      "epoch": 1.7843055371006624,
+      "grad_norm": 0.04866790026426315,
+      "learning_rate": 9.643357627076176e-06,
+      "loss": 1.188841064453125,
+      "step": 47000
+    },
+    {
+      "epoch": 1.8032877128376454,
+      "grad_norm": 0.08654382824897766,
+      "learning_rate": 8.035784735703915e-06,
+      "loss": 1.1869093017578125,
+      "step": 47500
+    },
+    {
+      "epoch": 1.8032877128376454,
+      "eval_correction_accuracy": 0.9764408271080002,
+      "eval_detection_f1": 0.6623505252062661,
+      "eval_loss": 1.2196118831634521,
+      "eval_runtime": 60.2937,
+      "eval_samples_per_second": 372.211,
+      "eval_steps_per_second": 2.919,
+      "step": 47500
+    },
+    {
+      "epoch": 1.8032877128376454,
+      "eval_edit_f0_5": 0.9131843391356872,
+      "eval_edit_f0_5_m0": 0.901242536711312,
+      "eval_edit_f0_5_m0.1": 0.9049602423324499,
+      "eval_edit_f0_5_m0.2": 0.9078295881168726,
+      "eval_edit_f0_5_m0.3": 0.9131843391356872,
+      "step": 47500
+    },
+    {
+      "epoch": 1.8032877128376454,
+      "eval_real_f0_5": 0.4527455121436114,
+      "eval_real_f0_5_m0": 0.4291393826005612,
+      "eval_real_f0_5_m0.1": 0.4339440694310511,
+      "eval_real_f0_5_m0.2": 0.4473161033797216,
+      "eval_real_f0_5_m0.3": 0.4527455121436114,
+      "step": 47500
+    },
+    {
+      "epoch": 1.8222698885746285,
+      "grad_norm": 0.07231193780899048,
+      "learning_rate": 6.571036575157384e-06,
+      "loss": 1.187695556640625,
+      "step": 48000
+    },
+    {
+      "epoch": 1.8412520643116115,
+      "grad_norm": 0.04208879545331001,
+      "learning_rate": 5.250586772162613e-06,
+      "loss": 1.1868779296875,
+      "step": 48500
+    },
+    {
+      "epoch": 1.8602342400485945,
+      "grad_norm": 0.08146563172340393,
+      "learning_rate": 4.07576378042419e-06,
+      "loss": 1.1862469482421876,
+      "step": 49000
+    },
+    {
+      "epoch": 1.8792164157855773,
+      "grad_norm": 0.035731397569179535,
+      "learning_rate": 3.047749544119016e-06,
+      "loss": 1.1868427734375,
+      "step": 49500
+    },
+    {
+      "epoch": 1.8981985915225603,
+      "grad_norm": 0.049494218081235886,
+      "learning_rate": 2.167578308787865e-06,
+      "loss": 1.18650390625,
+      "step": 50000
+    },
+    {
+      "epoch": 1.8981985915225603,
+      "eval_correction_accuracy": 0.9764698396496604,
+      "eval_detection_f1": 0.662213053912394,
+      "eval_loss": 1.219179391860962,
+      "eval_runtime": 60.3298,
+      "eval_samples_per_second": 371.989,
+      "eval_steps_per_second": 2.917,
+      "step": 50000
+    },
+    {
+      "epoch": 1.8981985915225603,
+      "eval_edit_f0_5": 0.9142088207832153,
+      "eval_edit_f0_5_m0": 0.904126540054877,
+      "eval_edit_f0_5_m0.1": 0.9078098748035337,
+      "eval_edit_f0_5_m0.2": 0.9097596861206475,
+      "eval_edit_f0_5_m0.3": 0.9142088207832153,
+      "step": 50000
+    },
+    {
+      "epoch": 1.8981985915225603,
+      "eval_real_f0_5": 0.44823906083244397,
+      "eval_real_f0_5_m0": 0.4280018761726078,
+      "eval_real_f0_5_m0.1": 0.43232177263969174,
+      "eval_real_f0_5_m0.2": 0.4433867735470942,
+      "eval_real_f0_5_m0.3": 0.44823906083244397,
+      "step": 50000
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 52682,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 2500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1632343079571684e+18,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-50000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae772d11cb059004c86012d10f5afc5062b15c0c8ba45184d07ee30411bd6a0a
+size 5368

checkpoint-50000/vocab.json

@@ -0,0 +1 @@
+{"tokens": ["<pad>", "<bos>", "<eos>", "<unk>", "\u0003", "\t", "\n", "\f", "\r", "\u0012", "\u0015", " ", "!", "\"", "#", "$", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", ":", ";", "<", "=", ">", "?", "@", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "[", "\\", "]", "^", "_", "`", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "{", "|", "}", "~", "€", "ƒ", "", "‘", "’", "“", "”", "•", "–", "—", "˜", "™", "œ", "", " ", "¡", "¢", "£", "¤", "¥", "¦", "§", "¨", "©", "«", "¬", "­", "®", "¯", "°", "±", "²", "³", "´", "µ", "¶", "·", "¹", "º", "»", "¼", "½", "¾", "¿", "À", "Á", "Â", "Ã", "Ä", "Å", "Æ", "Ç", "È", "É", "Ê", "Í", "Î", "Ð", "Ó", "Õ", "Ö", "×", "Ø", "Ù", "Ú", "Û", "Ü", "Ý", "Þ", "ß", "à", "á", "â", "ã", "ä", "å", "æ", "ç", "è", "é", "ê", "ë", "ì", "í", "î", "ï", "ð", "ñ", "ò", "ó", "ô", "õ", "ö", "ø", "ù", "ú", "û", "ü", "ý", "þ", "ÿ", "Ā", "ā", "ă", "ą", "Ć", "ć", "ċ", "Č", "č", "ď", "Đ", "đ", "ē", "ę", "ě", "Ğ", "ğ", "Ģ", "ģ", "ĩ", "ī", "ĭ", "İ", "ı", "Ķ", "ķ", "ļ", "Ľ", "Ł", "ł", "ń", "ņ", "ň", "Ŋ", "ŋ", "Ō", "ō", "Ŏ", "ŏ", "ő", "Œ", "œ", "Ř", "ř", "Ś", "ś", "Ş", "ş", "Š", "š", "ţ", "Ũ", "Ū", "ū", "ŭ", "ů", "ű", "ż", "Ž", "ž", "ƒ", "Ɲ", "ư", "ƴ", "ǎ", "ǒ", "ș", "ȝ", "ɑ", "ɓ", "ɔ", "ɕ", "ɖ", "ɘ", "ə", "ɛ", "ɡ", "ɣ", "ɧ", "ɪ", "ɬ", "ɯ", "ɲ", "ʀ", "ʁ", "ʃ", "ʉ", "ʊ", "ʋ", "ʌ", "ʏ", "ʔ", "ʕ", "ʖ", "ʰ", "ʻ", "ʿ", "ˁ", "ˆ", "ˈ", "ˌ", "ˎ", "ː", "˚", "˜", "˟", "̀", "́", "̂", "̃", "̅", "̈", "̌", "̥", "̯", "͜", "͡", "ͼ", "Ό", "Α", "Γ", "Δ", "Ε", "Λ", "Ν", "Ξ", "Π", "Τ", "Φ", "Ψ", "Ω", "ά", "έ", "ή", "ί", "α", "β", "γ", "δ", "ε", "ζ", "η", "θ", "ι", "κ", "λ", "μ", "ν", "ο", "π", "ρ", "ς", "σ", "τ", "υ", "φ", "χ", "ψ", "ω", "ό", "ύ", "ώ", "ϒ", "Ђ", "І", "А", "Б", "В", "Г", "Д", "И", "К", "М", "Н", "П", "Р", "С", "Т", "Ф", "Ц", "Ч", "Ш", "Ю", "Я", "а", "б", "в", "г", "д", "е", "ж", "з", "и", "й", "к", "л", "м", "н", "о", "п", "р", "с", "т", "у", "ф", "х", "ц", "ч", "ш", "щ", "ы", "ь", "ю", "я", "ђ", "ѕ", "і", "ј", "ў", "һ", "ԁ", "ԡ", "Ե", "Զ", "ա", "գ", "կ", "ն", "ս", "ր", "և", "֊", "ִ", "ָ", "ּ", "ׁ", "א", "ג", "ד", "ה", "ו", "ח", "י", "מ", "נ", "ס", "פ", "ץ", "צ", "ק", "ר", "ש", "ת", "،", "آ", "إ", "ئ", "ا", "ب", "ة", "ت", "ث", "ج", "ح", "خ", "د", "ذ", "ر", "ز", "ش", "ط", "ع", "ف", "ق", "ل", "م", "ن", "ه", "و", "ى", "ي", "ٹ", "پ", "چ", "ک", "گ", "ہ", "ی", "ރ", "ކ", "ं", "आ", "क", "ग", "च", "ज", "त", "न", "प", "फ", "म", "य", "र", "ल", "व", "श", "स", "ा", "ि", "ी", "ु", "ू", "े", "्", "চ", "জ", "ত", "ন", "প", "ব", "ম", "র", "্", "୨", "அ", "உ", "க", "த", "ய", "ர", "ல", "ள", "வ", "ா", "ி", "ு", "்", "ิ", "฿", "ไ", "་", "།", "ག", "ཆ", "ན", "པ", "ར", "ི", "ུ", "ེ", "ོ", "ដ", "ឋ", "ភ", "រ", "ស", "ា", "្", "᠇", "ᦁ", "ᦆ", "ᦇ", "ᦈ", "ᦗ", "ᦚ", "ḍ", "Ḥ", "ḥ", "Ḫ", "ḫ", "ḻ", "ṅ", "ṇ", "ṛ", "Ṣ", "ṣ", "Ṭ", "ṭ", "ẓ", "ạ", "ả", "ầ", "ẽ", "ệ", "ị", "Ỗ", "ỗ", "ớ", "ἔ", "Ἰ", "ῶ", " ", " ", " ", " ", " ", "​", "‌", "‍", "‎", "‏", "‐", "‑", "–", "—", "―", "‖", "‘", "’", "‚", "“", "”", "„", "†", "‡", "•", "…", "‬", " ", "′", "″", "‷", "‹", "›", "※", "‼", "⁄", "⁉", "⁠", "⁨", "⁴", "⁵", "⁸", "₂", "₤", "₩", "€", "₹", "℃", "№", "℗", "℠", "™", "⅄", "⅓", "⅗", "⅝", "Ⅰ", "Ⅱ", "ⅼ", "ⅽ", "←", "↑", "→", "↓", "↔", "↵", "⇚", "⇧", "∀", "∃", "∅", "∆", "∑", "−", "√", "≈", "≠", "≤", "≥", "⊞", "⌐", "⌥", "⏎", "①", "②", "③", "⒇", "Ⓟ", "─", "│", "├", "□", "►", "▽", "◄", "◆", "○", "◌", "●", "◟", "★", "☆", "☕", "☝", "☠", "☰", "☹", "☺", "♀", "♂", "♌", "♒", "♙", "♠", "♡", "♥", "♦", "♪", "♫", "⚜", "⚠", "⚡", "✅", "✈", "✊", "✋", "✌", "✓", "✔", "✨", "❌", "❤", "❶", "⟨", "⟩", "⭐", "ⲟ", "ⲡ", "ⲣ", "⽝", "、", "。", "《", "》", "「", "」", "『", "』", "【", "】", "〓", "〜", "〝", "〞", "〟", "〢", "あ", "い", "う", "お", "か", "が", "き", "げ", "こ", "ご", "さ", "ざ", "し", "じ", "す", "せ", "そ", "た", "ち", "っ", "て", "で", "と", "ど", "な", "に", "ね", "の", "は", "ま", "み", "め", "も", "や", "よ", "ら", "り", "わ", "を", "ん", "ゞ", "゠", "ァ", "ア", "イ", "オ", "ク", "コ", "シ", "セ", "ゼ", "ダ", "ツ", "ョ", "��", "ル", "レ", "ン", "・", "ー", "ㄟ", "㻳", "㻷", "䇂", "一", "丁", "七", "三", "不", "世", "中", "九", "五", "京", "伝", "传", "你", "例", "傲", "儿", "先", "典", "列", "動", "包", "千", "卣", "参", "叉", "反", "咖", "喱", "四", "国", "土", "地", "塗", "外", "大", "天", "太", "好", "妃", "学", "寫", "少", "尾", "山", "島", "帅", "年", "店", "庵", "庸", "强", "影", "往", "怖", "总", "恐", "息", "扁", "手", "投", "挂", "擾", "敏", "斯", "方", "施", "日", "昌", "显", "月", "期", "本", "東", "某", "树", "桂", "桐", "次", "民", "水", "洛", "津", "点", "煎", "燒", "爱", "猫", "理", "生", "电", "白", "百", "直", "真", "眼", "硫", "硼", "示", "祝", "神", "稿", "穆", "穴", "米", "紙", "經", "经", "胆", "膽", "苯", "英", "蒋", "衆", "西", "要", "覧", "览", "言", "記", "語", "诚", "谋", "貴", "起", "足", "辞", "运", "鄉", "酒", "醉", "释", "重", "金", "钠", "长", "閲", "阳", "院", "陳", "陽", "预", "騒", "黄", "Ꙑ", "ꙑ", "ꜣ", "교", "그", "기", "길", "낞", "니", "다", "당", "동", "로", "론", "릿", "면", "방", "백", "사", "삼", "선", "수", "습", "시", "식", "신", "실", "암", "용", "을", "인", "있", "준", "크", "토", "하", "확", "", "", "", "ff", "fi", "fl", "️", "﹩", "﹫", "ﻛ", "ﻜ", "ﻩ", "ﻴ", "", "！", "＃", "＄", "＆", "＇", "（", "＋", "，", "－", "／", "３", "：", "＞", "？", "Ａ", "Ｃ", "Ｄ", "Ｉ", "Ｒ", "Ｓ", "Ｔ", "Ｕ", "Ｙ", "＿", "ａ", "ｂ", "ｃ", "ｄ", "ｅ", "ｆ", "ｈ", "ｉ", "ｊ", "ｋ", "ｌ", "ｍ", "ｑ", "ｒ", "ｓ", "ｔ", "ｕ", "ｖ", "ｘ", "ｙ", "｛", "｜", "～", "｢", "｣", "､", "･", "ﾢ", "ￒ", "￥", "", "�", "𐀏", "𐅇", "𐬀", "𐬌", "𐬎", "𐬑", "𐬙", "𐬞", "𐬭", "𐬱", "𔅴", "𔱰", "𖯗", "𖾺", "𗽋", "𝖆", "𝖊", "𝖍", "𝖑", "𝖓", "𝖘", "𝖙", "𝖜", "𞂾", "𞃳", "🇦", "🇨", "🇩", "🇪", "🇬", "🇭", "🇮", "🇱", "🇵", "🇷", "🇸", "🇺", "🇻", "🇾", "🇿", "🌌", "🌞", "🌟", "🌹", "🌺", "🍀", "🍁", "🍞", "🍾", "🎀", "🎈", "🎉", "🎊", "🎖", "🎶", "🎻", "🏅", "🏈", "🏚", "🏤", "🏥", "🏦", "🏻", "🏼", "🏽", "🏾", "🐌", "🐙", "🐚", "🐯", "🐱", "🐲", "👀", "👈", "👉", "👊", "👋", "👌", "👍", "👏", "👑", "👹", "👺", "💀", "💅", "💌", "💐", "💓", "💔", "💕", "💖", "💗", "💙", "💚", "💛", "💜", "💞", "💣", "💥", "💨", "💩", "💪", "💫", "💯", "💰", "📉", "📋", "📡", "📸", "🔕", "🔵", "🕶", "🖤", "🗞", "😀", "😁", "😂", "😃", "😄", "😅", "😆", "😈", "😉", "😊", "😋", "😌", "😍", "😎", "😏", "😐", "😑", "😒", "😓", "😔", "😕", "😖", "😘", "😛", "😜", "😝", "😞", "😟", "😡", "😢", "😤", "😥", "😧", "😨", "😩", "😪", "😫", "😬", "😭", "😮", "😯", "😰", "😱", "😲", "😳", "😵", "😶", "😹", "🙀", "🙂", "🙃", "🙄", "🙈", "🙌", "🙏", "🚨", "🚮", "🚽", "🛑", "🛒", "🛫", "🟩", "🤌", "🤍", "🤑", "🤓", "🤔", "🤗", "🤘", "🤛", "🤜", "🤝", "🤞", "🤡", "🤢", "🤣", "🤤", "🤦", "🤧", "🤨", "🤩", "🤫", "🤬", "🤭", "🤮", "🤯", "🤷", "🥋", "🥚", "🥰", "🥱", "🥲", "🥳", "🥵", "🥹", "🥺", "🦃", "🦜", "🦾", "🧊", "🧐", "🧠", "🩵", "🩷", "🩹", "🪙", "🪪", "🪲", "🪽", "🫀", "🫂", "🫠", "🫡", "🫢", "🫣", "🫶", "򟝥", "򟧷", "󉜹"]}