Upload model

prepare_dataset.py

@@ -10,6 +10,7 @@ import datasets
 import duckdb
 import numpy as np
 import pandas as pd
+from huggingface_hub import hf_hub_download
 
 from .create_section_files import create_section_files
 
@@ -58,7 +59,7 @@ def lookup_tables(con, tables):
                 
         luts_dict[k]['total'] = start_idx
 
-    with open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lookup_tables.json'), 'w') as file:
+    with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lookup_tables.json'), 'w') as file:
         json.dump(luts_dict, file)
 
 
@@ -148,8 +149,10 @@ def prepare_dataset(physionet_dir, database_dir, num_workers=None):
     )
     
     # Load the table configuration:
-    with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tables.json'), 'r') as file:
-        tables = json.load(file)
+    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tables.json')
+    path = path if os.path.exists(path) else hf_hub_download(repo_id='aehrc/cxrmate-ed', filename='tables.json')
+    with open(path, 'r') as f:
+        tables = json.load(f)
             
     # Create lookup tables:
     lookup_tables(con, tables)