<jupyter_start><jupyter_text>Lecture Demo<jupyter_code>pew_data = pd.read_csv("data/pew.csv")
pew_data
pew_data_tidy = pd.melt(pew_data, id_vars=["religion"], var_name = "income", value_name="frequency")

pew_data_tidy
tb_data = pd.read_csv("data/tb.csv")
tb_data
tb_data.columns
# m04 means Male from 0 to 4 years
tb_data_unpivoted = tb_data.melt(id_vars=["iso2","year"], var_name="gender_and_age", value_name="frequency")
tb_data_unpivoted
gender = tb_data_unpivoted.gender_and_age.str.slice(0,1)
age_range = tb_data_unpivoted.gender_and_age.str.slice(1)

age_range.unique()
tb_data_unpivoted["gender"] = gender
tb_data_unpivoted["age_range"] = age_range
tb_data_tidy = tb_data_unpivoted.drop(["gender_and_age"], axis = 1) # axis = 0 by rows, 1 by columns
tb_data_unpivoted
tb_data_tidy = tb_data_tidy[ ["iso2","year", "age_range","gender", "frequency"]  ]
tb_data_tidy
age_ranges = tb_data_tidy.age_range.str.extract("(\d)(\d{2})").sample(20)
tb_data_tidy["age_lower"] = age_ranges.loc[:,0]
tb_data_tidy["age_upper"] = age_ranges.loc[:,1]
tb_data_tidy
tb_data_tidy = tb_data_tidy.drop(["age_range"], axis=1)
tb_data_tidy<jupyter_output><empty_output><jupyter_text>Task 2<jupyter_code>weather_data = pd.read_csv("data/weather.csv")
weather_data
weather_data_unpivoted = pd.melt(weather_data, id_vars=["id", "year", "month", "element"])
weather_data_unpivoted
days = weather_data_unpivoted.variable.str.slice(1).astype(int)
days
weather_data_unpivoted["day"] = days
weather_data_unpivoted = weather_data_unpivoted.drop(["variable"], axis=1)
weather_data_unpivoted.sample(30)
weather_data_unpivoted[weather_data_unpivoted.month == 3]
weather_data_tidy = weather_data_unpivoted.pivot_table(index=["id", "year", "month", "day"], columns=["element"], values=["value"])
weather_data_tidy
weather_data_tidy = weather_data_tidy.reset_index()
weather_data_tidy
pd.to_datetime(weather_data_tidy[["year", "month", "day"]])<jupyter_output><empty_output><jupyter_text>task 3<jupyter_code>songs_data = pd.read_csv("data/billboard.csv")
songs_data
songs_tidy = songs_data.melt(id_vars=["year", "artist", "track", "time", "date.entered"])
songs_tidy
songs_tidy["week"] =  songs_tidy.variable.str.slice(2).astype(int)
songs_tidy = songs_tidy.drop(["variable"], axis=1)
songs_tidy
# To drop NaN
songs_tidy = songs_tidy.dropna()
songs_tidy
songs_tidy.value = songs_tidy.value.astype(int)
songs_tidy.dtypes
songs_tidy[["year", "artist", "track", "time"]]
songs_tidy[["date.entered", "value", "week"]]
songs_tidy.head() # tail() , sample()
songs_tidy.sample(100_000, replace=True)
weather_data_tidy.columns
weather_data_tidy[("value", "tmax")].min() #max(), nlargest(10)
weather_data_tidy.nlargest(10, ("value", "tmax"))
weather_data_tidy.sample(len(weather_data_tidy), replace= False)<jupyter_output><empty_output><jupyter_text>TYPE is different<jupyter_code>type(songs_tidy["date.entered"])
type(songs_tidy[["date.entered"]])
pac_filter = songs_tidy["artist"]== "2 Pac"
songs_tidy[pac_filter].track.unique()
rapers = (songs_tidy["artist"]== "2 Pac") | (songs_tidy["artist"]== "Eminem")
songs_tidy[rapers]["track"].unique()<jupyter_output><empty_output><jupyter_text>Grouping<jupyter_code>for relidion, data in pew_data_tidy.groupby("religion"):
    print(relidion)
    print('______________')
    print(data)
    print('==============')

for relidion, data in pew_data_tidy.groupby("religion"):
    print(relidion, data.frequency.sum())
for income, data in pew_data_tidy.groupby("income"):
    print(income, data.frequency.sum())<jupyter_output>$10-20k 2781
$100-150k 3197
$20-30k 3357
$30-40k 3302
$40-50k 3085
$50-75k 5185
$75-100k 3990
<$10k 1930
>150k 2608
Don't know/refused 6121<jupyter_text>Cleaning data<jupyter_code>weather_year = pd.read_csv("https://raw.githubusercontent.com/synesthesiam/blog/master/posts/data/weather_year.csv")
weather_year
weather_year.columns = ["date", "max_temp", "mean_temp", "min_temp", "max_dew",
"mean_dew", "min_dew", "max_humidity", "mean_humidity",
"min_humidity", "max_pressure", "mean_pressure",
"min_pressure", "max_visibilty", "mean_visibility",
"min_visibility", "max_wind", "mean_wind", "min_wind",
"precipitation", "cloud_cover", "events", "wind_dir"]
weather_year.shape
def string_to_date(date_string):
    return datetime.datetime.strptime(date_string, "%Y-%m-%d")
string_to_date("2012-3-10")
weather_year.date = weather_year.date.apply(string_to_date)
weather_year
weather_year.precipitation.unique()
weather_year.loc[weather_year.precipitation == 'T', "precipitation"] = 1e-5
weather_year.precipitation = weather_year.precipitation.astype(float) 
weather_year
weather_year.dtypes
weather_year.events.unique()
weather_year.describe().T
iris_data = pd.read_csv("https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data", header = None)
iris_data
iris_data.columns = ["sepal_length", "sepal_width", "petal_length", "petal_width", "class"]
iris_data
iris_data.sepal_length[(iris_data.sepal_length > 4.5) & (iris_data.sepal_length <= 10)].mean()
iris_data.dtypes
red_wine = pd.read_csv("http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv", sep=';')
red_wine
white_wine = pd.read_csv("http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv", sep=';')
white_wine
white_wine.quality.mean()
red_wine.quality.mean()
red_wine.corr()
white_wine.corr()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup