add CV 11

README.template.md

@@ -131,7 +131,7 @@ Additional fields include `accent`, `age`, `client_id`, `up_votes`, `down_votes`
 
 The speech material has been subdivided into portions for dev, train, test, validated, invalidated, reported and other.
 
-The validated data is data that has been validated with reviewers and recieved upvotes that the data is of high quality.
+The validated data is data that has been validated with reviewers and received upvotes that the data is of high quality.
 
 The invalidated data is data has been invalidated by reviewers
 and received downvotes indicating that the data is of low quality.
@@ -153,7 +153,7 @@ In addition, the majority of training sentences end in punctuation ( . or ? or !
 ```python
 from datasets import load_dataset
 
-ds = load_dataset("mozilla-foundation/{{NAME}}", "en", use_auth_token=True)
+ds = load_dataset("mozilla-foundation/{{DATASET_PATH}}", "en", use_auth_token=True)
 
 def prepare_dataset(batch):
   """Function to preprocess the dataset with the .map method"""

generate_datasets.py

@@ -54,6 +54,11 @@ VERSIONS = [
         "name": "common_voice_10_0",
         "release": "cv-corpus-10.0-2022-07-04",
     },
+    {
+        "semver": "11.0.0",
+        "name": "common_voice_11_0",
+        "release": "cv-corpus-11.0-2022-09-21",
+    },
 ]
 
 
@@ -102,6 +107,7 @@ def main():
         with open(f"README.template.md", "r") as fin:
             readme = fin.read()
             readme = readme.replace("{{NAME}}", release_stats["name"])
+            readme = readme.replace("{{DATASET_PATH}}", version["name"])
 
             locales = sorted(release_stats["locales"].keys())
             languages = [f"- {loc}" for loc in locales]

languages.ftl

@@ -49,6 +49,7 @@ gom = Goan Konkani
 ha = Hausa
 he = Hebrew
 hi = Hindi
+hil = Hiligaynon
 hr = Croatian
 hsb = Sorbian, Upper
 ht = Haitian
@@ -63,6 +64,7 @@ is = Icelandic
 it = Italian
 izh = Izhorian
 ja = Japanese
+jbo = Lojban
 ka = Georgian
 kaa = Karakalpak
 kab = Kabyle
@@ -71,6 +73,7 @@ ki = Kikuyu
 kk = Kazakh
 km = Khmer
 kmr = Kurmanji Kurdish
+kn = Kannada
 knn = Konkani (Devanagari)
 ko = Korean
 kpv = Komi-Zyrian
@@ -79,6 +82,8 @@ ky = Kyrgyz
 lb = Luxembourgish
 lg = Luganda
 lij = Ligurian
+ln = Lingala
+lo = Lao
 lt = Lithuanian
 lv = Latvian
 mai = Maithili
@@ -125,11 +130,13 @@ sah = Sakha
 sat = Santali (Ol Chiki)
 sc = Sardinian
 scn = Sicilian
+sdh = Southern Kurdish
 shi = Shilha
 si = Sinhala
 sk = Slovak
 skr = Saraiki
 sl = Slovenian
+snk = Soninke
 so = Somali
 sq = Albanian
 sr = Serbian
@@ -167,6 +174,7 @@ xh = Xhosa
 yi = Yiddish
 yo = Yoruba
 yue = Cantonese
+zgh = Tamazight
 zh-CN = Chinese (China)
 zh-HK = Chinese (Hong Kong)
 zh-TW = Chinese (Taiwan)

test.py

@@ -1,5 +1,5 @@
 from datasets import load_dataset
 
-dataset = load_dataset("mozilla-foundation/common_voice_10_0", "et", split="test", use_auth_token=True)
+dataset = load_dataset("./common_voice_11_0", "et", split="test", use_auth_token=True)
 print(dataset)
 print(dataset[100])