<jupyter_start><jupyter_text>Data Processing - ADNI and demographics<jupyter_code>import pandas as pd
import regex as re
import numpy as np
path = 'C:/Users/glh19/OneDrive/Desktop/Winter_Spring 2022/Tosun-Turgut/data/'

adni1 = pd.read_csv(path + 'adni1_muse_results_with_dates.csv')
adni2 = pd.read_csv(path + 'adni2_muse_results_with_dates.csv')
adni3 = pd.read_csv(path + 'adni3_muse_results_with_dates.csv')

DATADIC = pd.read_csv(path + 'DATADIC.csv')
DXSUM = pd.read_csv(path + 'DXSUM_PDXCONV_ADNIALL.csv')
PTDEMOG = pd.read_csv(path + 'PTDEMOG.csv')
PTDEMOG_DXSUM = PTDEMOG.merge(DXSUM,  how='right')
PTDEMOG_DXSUM
def reorder_columns(dataframe, col_name, position):
    """Reorder a dataframe's column.
    Args:
        dataframe (pd.DataFrame): dataframe to use
        col_name (string): column name to move
        position (0-indexed position): where to relocate column to
    Returns:
        pd.DataFrame: re-assigned dataframe
    """
    temp_col = dataframe[col_name]
    dataframe = dataframe.drop(columns=[col_name])
    dataframe.insert(loc=position, column=col_name, value=temp_col)
    return dataframe

df = reorder_columns(dataframe=PTDEMOG_DXSUM, col_name='PTDOBMM', position=2)
df = reorder_columns(dataframe=df, col_name='PTDOBYY', position=3)

# slim_PTDEMOG_DXSUM = PTDEMOG_DXSUM.loc[:,['Phase','ID', 'RID','PTDOBMM','PTDOBYY','DXCHANGE','DXCURREN','DIAGNOSIS']]
keep_cols = ['Phase','ID', 'RID','PTDOBMM','PTDOBYY','DXCHANGE','DXCURREN','DIAGNOSIS']
remove_cols = ['SITEID','VISCODE','VISCODE2','USERDATE','USERDATE2',
 'PTSOURCE','PTGENDER','PTHAND','PTMARRY','PTEDUCAT','PTWORKHS',
 'PTWORK', 'PTWRECNT', 'PTNOTRT', 'PTRTYR', 'PTHOME', 'PTOTHOME',
 'PTTLANG', 'PTPLANG', 'PTPSPEC', 'PTCOGBEG', 'PTMCIBEG', 'PTADBEG',
 'PTADDX', 'PTETHCAT', 'PTRACCAT', 'update_stamp','PTID','DXCONV',
 'DXCONTYP', 'DXREV', 'DXNORM', 'DXNODEP', 'DXMCI', 'DXMDES',
 'DXMPTR1', 'DXMPTR2', 'DXMPTR3', 'DXMPTR4', 'DXMPTR5', 'DXMPTR6',
 'DXMDUE', 'DXMOTHET', 'DXMOTHSP', 'DXDSEV', 'DXDDUE', 'DXAD',
 'DXADES', 'DXAPP', 'DXAPROB', 'DXAMETASP', 'DXAOTHRSP', 'DXAPOSS',
 'DXAATYSP', 'DXAMETSP', 'DXAOTHSP', 'DXPARK', 'DXPARKSP', 'DXPDES',
 'DXPCOG', 'DXPATYP', 'DXPOTHSP', 'DXDEP', 'DXDEPSP', 'DXOTHDEM',
 'DXODES', 'DXOOTHSP', 'DXCONFID']


new_order = ['Phase','ID', 'RID','PTDOBMM','PTDOBYY','DXCHANGE','DXCURREN','DIAGNOSIS',
 'SITEID','VISCODE','VISCODE2','USERDATE','USERDATE2',
 'PTSOURCE','PTGENDER','PTHAND','PTMARRY','PTEDUCAT','PTWORKHS',
 'PTWORK', 'PTWRECNT', 'PTNOTRT', 'PTRTYR', 'PTHOME', 'PTOTHOME',
 'PTTLANG', 'PTPLANG', 'PTPSPEC', 'PTCOGBEG', 'PTMCIBEG', 'PTADBEG',
 'PTADDX', 'PTETHCAT', 'PTRACCAT', 'update_stamp','PTID','DXCONV',
 'DXCONTYP', 'DXREV', 'DXNORM', 'DXNODEP', 'DXMCI', 'DXMDES',
 'DXMPTR1', 'DXMPTR2', 'DXMPTR3', 'DXMPTR4', 'DXMPTR5', 'DXMPTR6',
 'DXMDUE', 'DXMOTHET', 'DXMOTHSP', 'DXDSEV', 'DXDDUE', 'DXAD',
 'DXADES', 'DXAPP', 'DXAPROB', 'DXAMETASP', 'DXAOTHRSP', 'DXAPOSS',
 'DXAATYSP', 'DXAMETSP', 'DXAOTHSP', 'DXPARK', 'DXPARKSP', 'DXPDES',
 'DXPCOG', 'DXPATYP', 'DXPOTHSP', 'DXDEP', 'DXDEPSP', 'DXOTHDEM',
 'DXODES', 'DXOOTHSP', 'DXCONFID']


# PTDEMOG_DXSUM.columns.tolist()

# PTDEMOG_DXSUM = PTDEMOG_DXSUM.reindex(columns=new_order)
# PTDEMOG_DXSUM
# slim_PTDEMOG_DXSUM = PTDEMOG_DXSUM.drop(remove_cols, axis=1, inplace=True)
# slim_PTDEMOG_DXSUM = PTDEMOG_DXSUM.drop(~PTDEMOG_DXSUM.isin(keep_cols), axis=1)
slim_PTDEMOG_DXSUM = PTDEMOG_DXSUM[['Phase','ID', 'RID','PTDOBMM','PTDOBYY','DXCHANGE','DXCURREN','DIAGNOSIS']]
slim_PTDEMOG_DXSUM.head(5)
#drop ADNIGO rows which originated from the DXSUM portion of dataframe
slim_PTDEMOG_DXSUM = slim_PTDEMOG_DXSUM[slim_PTDEMOG_DXSUM['Phase'] !='ADNIGO']
slim_PTDEMOG_DXSUM[slim_PTDEMOG_DXSUM['Phase']=='ADNIGO'].size<jupyter_output><empty_output><jupyter_text>Creating complete diagnosis col<jupyter_code>#Success:
slim_PTDEMOG_DXSUM['CHAN_CURR'] = slim_PTDEMOG_DXSUM['DXCHANGE'].fillna(value=slim_PTDEMOG_DXSUM['DXCURREN'])
# slim_PTDEMOG_DXSUM['CHAN_CURR']
slim_PTDEMOG_DXSUM['complete_diagnosis'] = slim_PTDEMOG_DXSUM['CHAN_CURR'].fillna(value=slim_PTDEMOG_DXSUM['DIAGNOSIS'])

# slim_PTDEMOG_DXSUM[slim_PTDEMOG_DXSUM['complete_diagnosis']=='nan']
slim_PTDEMOG_DXSUM['complete_diagnosis']
adni12 = pd.concat([adni1,adni2], axis=0)
adni123 = pd.concat([adni12, adni3], axis=0)

# adni1.head(5)
# adni2.head(5)
# adni12
# adni12[adni12['project']=='ADNI2']
adni123[adni123['project']=='ADNI3']
# adni123<jupyter_output><empty_output><jupyter_text>cleaning and removing unwanted roi volumes<jupyter_code># to remove: uppercase items
adni123.drop(adni123.index[adni123['roi_name'].str.isupper()], inplace=True)
print('adni123 size post uppercase removal:')
print(adni123.size)

discard = (['Right Cerebellum Exterior', 'Left Cerebellum Exterior',
            'Right Cerebellum White Matter','Left Cerebellum White Matter',
            'frontal lobe WM right','frontal lobe WM left',
            'occipital lobe WM right','occipital lobe WM left',
            'parietal lobe WM right','parietal lobe WM left',
            'temporal lobe WM right','temporal lobe WM left',
            'fornix right','fornix left','corpus callosum'])

# to remove unwanted brain regions
adni123 = adni123[adni123.roi_name.isin(discard) == False]
print('adni123 size post clean:')
print(adni123.size)
#remove unwanted ADNI1 partial string from code entries:
adni123['code'] = adni123['code'].str.strip('ADNI1_')
adni123['split_code'] = adni123['code'].str.split('_')
adni123.tail(5)
# split up code into new columns: PTID, RID, unique image #
# example code: 002_S_0295xia_118671

adni123['SITEID'] = adni123['split_code'].str[0]
adni123['RID_VISCODE'] = adni123['split_code'].str[2]
adni123['IMAGEUID'] = adni123['split_code'].str[3]
# RID=1326 and VISCODE=m12
rid_pattern = r'(\d\d\d\d)'
adni123['RID_VISCODE'] =  adni123['RID_VISCODE'].str.split(pat=rid_pattern)

# adni123['RID_VISCODE'].head(5)
adni123['RID'] = adni123['RID_VISCODE'].str[1]
adni123['VISCODE'] = adni123['RID_VISCODE'].str[2]
adni123[adni123['roi_name'] == 'icv'][['code','roi_name','roi_volume']]<jupyter_output><empty_output><jupyter_text>% icv column build<jupyter_code>icv_values = (adni123[adni123['roi_name'] == 'icv']['roi_volume'])

adni123.groupby(['project','code'])['roi_volume'].size()
icv_values
adni123[(adni123.index.duplicated())&(adni123['project']=='ADNI1')]
adni123[adni123.index.duplicated()]
# reindex adni123
adni123 = adni123.reset_index() 
adni123['icv_vals'] = adni123[adni123['roi_name']=='icv']['roi_volume']
adni123['icv_vals']
adni123['icv_vals'] = adni123['icv_vals'].fillna(0)

def divide_two_cols(adni123):
    adni123['%_icv'] = adni123['roi_volume'] / adni123['icv_vals'].max()
    return adni123['%_icv']

# I might have to groupby 'code' and 'project' b/c the same patient may be in multiple phases with different brain vol measurements 
# checking if function filtered data properly when computing % icv:

adni123['%_icv'] = adni123.groupby(['code','project']).apply(divide_two_cols).tolist()

# adni123[adni123['code']=='016_S_1326m12_i102675'][['code','roi_name','roi_volume','%_icv']]

adni123[((adni123['roi_name']=='icv') | (adni123['roi_name']== 'Left Amygdala'))&(adni123['project']=='ADNI2')][['project','code','roi_name','roi_volume','%_icv']]


# adni123['%_icv'] = adni123.groupby('code').apply(divide_two_cols).tolist()
adni123[adni123['code']=='016_S_1326m12_i102675'][['code','roi_name','roi_volume','%_icv']]
# final merge:
adni_merged = slim_PTDEMOG_DXSUM.merge(adni123, on='RID', how='left')
# adni_merged = pd.concat(adni123,slim_PTDEMOG_DXSUM)

adni_merged<jupyter_output><empty_output><jupyter_text>Calculating age<jupyter_code>from datetime import datetime as dt
from datetime import timedelta

#using fake data until I am able to successfully acces the mm and yy columns
# of the dataframe while taking into account the indexing and formatting

age_practice = pd.DataFrame(data= {
    'PTDOBMM':[4,12,5,1],
    'PTDOBYY':[1931,1944,1924,1938],
    'examDate':['4/18/2006','11/2/2006','7/23/2008','5/22/2009']
    
})

age_practice['defaultDD'] = 15*np.ones(age_practice.shape[0]).astype(int)
#columns to make date: from month, day, year:
cols = ["PTDOBMM","defaultDD","PTDOBYY"]
age_practice["DOB"] = age_practice[cols].apply(lambda x: '/'.join(x.values.astype(str)), axis="columns")
# age_practice["DOB"] = age_practice["DOB"].str.split('/').str[0]

#transform to datetime format:
age_practice['examDate'] = age_practice['examDate'].str.split('/').str[2]+'-'+age_practice['examDate'].str.split('/').str[0]+'-'+age_practice['examDate'].str.split('/').str[1]
age_practice['DOB'] = age_practice['DOB'].str.split('/').str[2]+'-'+age_practice['DOB'].str.split('/').str[0]+'-'+age_practice['DOB'].str.split('/').str[1]

age_practice['examDate'] = pd.to_datetime(age_practice['examDate'],format='%Y-%m-%d')

age_practice['DOB'] = pd.to_datetime(age_practice['DOB'],format='%Y-%m-%d')


# compute age:
age_practice['age_in_days'] = age_practice['examDate'] -age_practice['DOB']

age_practice['age_in_years'] = age_practice["age_in_days"] / timedelta(days=365)


age_practice<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup