<jupyter_start><jupyter_text>Prediction anonymous___This notebook aims to present and show results from two manually scored hypnograms on the same night's recording. One hypnogram is from our dataset, and the other hypnogram was manually scored by an electrophysiologist.As stated in [this paper](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2635577/__ffn_sectitle), significant differences may lay between these two hypnograms, because both scored didn't followed the same scoring manual. Our dataset being made in the early 90s, scorers followed R&K manual, while scorers now generally follow AASM manual.The recording that was randomly selected is `SC4441EC-Hypnogram.edf`. It corresponds to an 74 year old woman.Source: Moser, Doris, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, et al. « Sleep Classification According to AASM and Rechtschaffen & Kales: Effects on Sleep Scoring Parameters ». Sleep 32, nᵒ 2 (1 février 2009): 139‑49.<jupyter_code>%load_ext autoreload
%autoreload 2
%matplotlib inline

import mne
import json
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime, timedelta
from sklearn.metrics import (confusion_matrix,
                             classification_report,
                             cohen_kappa_score)

from utils import fetch_data
from constants import (SLEEP_STAGES_VALUES,
                       DATASET_SLEEP_STAGES_VALUES,
                       ELECTROPHYSIOLOGIST_SLEEP_STAGES,
                       N_STAGES,
                       EEG_CHANNELS,
                       EPOCH_DURATION,
                       SAMPLING_FREQ)
from models.model_utils import (print_hypnogram)
NB_EPOCHS_AWAKE_MORNING = 60
    
NYQUIST_FREQ = SAMPLING_FREQ/2
MAX_TIME = EPOCH_DURATION - 1. / SAMPLING_FREQ  # tmax in included

SUBJECT = [44]
NIGHT = [1]
FOLDER_NAME = f"./data/subject{SUBJECT[0]}-night{NIGHT[0]}"
df_records = pd.read_csv("./data/recordings-info.csv")
df_records.head(2)<jupyter_output><empty_output><jupyter_text>Extract data___<jupyter_code>def get_hypnogram_from_filenames(signal_filename, hypnogram_filename, is_electrophysiologist=False):

    raw_data = mne.io.read_raw_edf(signal_filename, preload=False, stim_channel=None, verbose=False)
    annot = mne.read_annotations(hypnogram_filename)
    raw_data.set_annotations(annot, emit_warning=False)
    
    info = df_records[(df_records['subject'] == SUBJECT[0]) & (df_records['night'] == NIGHT[0])]
    print(info)

    # Number of seconds since file began
    closed_lights_time = info['LightsOffSecond'].values[0]
    woke_up_time = closed_lights_time + info['NightDuration'].values[0] + NB_EPOCHS_AWAKE_MORNING*EPOCH_DURATION

    raw_data.crop(tmin=closed_lights_time, tmax=min(woke_up_time, raw_data.times[-1]))

    events, annot_event_id = mne.events_from_annotations(
        raw_data,
        event_id=ELECTROPHYSIOLOGIST_SLEEP_STAGES if is_electrophysiologist else DATASET_SLEEP_STAGES_VALUES,
        chunk_duration=EPOCH_DURATION,
        verbose=False)

    # Few files do not have N3 sleep (i.e. SC4202EC-Hypnogram), so we have to filter out key-value pairs that are not in the annotations.
    event_id = { 
        event_key: SLEEP_STAGES_VALUES[event_key] 
        for event_key in SLEEP_STAGES_VALUES
        if SLEEP_STAGES_VALUES[event_key] in annot_event_id.values()
    }

    epochs = mne.Epochs(
        raw=raw_data,
        events=events,
        event_id=event_id,
        tmin=0.,
        tmax=MAX_TIME,
        preload=True,
        baseline=None,
        verbose=False)

    epochs.drop_channels([
     'EOG horizontal',
     'Resp oro-nasal',
     'EMG submental',
     'Temp rectal',
     'Event marker'])

    y = np.array([event[-1] for event in epochs.events])
    
    return y
file_names = fetch_data(subjects=SUBJECT, recording=NIGHT)
print(file_names)

y_physionet = get_hypnogram_from_filenames(file_names[0][0], file_names[0][1])
y_electrophysiologist = get_hypnogram_from_filenames(
    file_names[0][0],
    f"{FOLDER_NAME}/electrophysiologist_Hypnogram.edf",
    is_electrophysiologist=True
)
plt.rcParams["figure.figsize"] = (20,5)
print_hypnogram(
    [y_physionet, y_electrophysiologist],
    labels=['physionet', 'electrophysiologist'],
    subject=44,
    night=1
)
print("Matrice de confusion: \n", confusion_matrix(y_physionet, y_electrophysiologist))

print("\n\n", classification_report(y_physionet, y_electrophysiologist, target_names=SLEEP_STAGES_VALUES.keys()))

print("Agreement score (Cohen Kappa): ", cohen_kappa_score(y_physionet, y_electrophysiologist))
electro_hypno = {
    'timestamps': [
        int((datetime.fromtimestamp(615514680) + timedelta(seconds=index*EPOCH_DURATION)).timestamp())
        for index, stage in enumerate(y_electrophysiologist)
    ],
    'stages': [
        str(list(SLEEP_STAGES_VALUES.keys())[list(SLEEP_STAGES_VALUES.values()).index(stage)])
        for index, stage in enumerate(y_electrophysiologist)
    ]
}

with open('data/electrophysiologist_woman78yo_sleepedf.json', 'w') as fp:
    json.dump(dict(epochs=electro_hypno), fp)
sleepedf_hypno = {
    'timestamps': [
        int((datetime.fromtimestamp(615514680) + timedelta(seconds=index*EPOCH_DURATION)).timestamp())
        for index, stage in enumerate(y_physionet)
    ],
    'stages': [
        str(list(SLEEP_STAGES_VALUES.keys())[list(SLEEP_STAGES_VALUES.values()).index(stage)])
        for index, stage in enumerate(y_physionet)
    ]
}

with open('data/physionet_woman78yo_sleepedf.json', 'w') as fp:
    json.dump(dict(epochs=sleepedf_hypno), fp)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup