Exploratory Data Analysis 3

#uvoz biblioteka

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sb
from pathlib import Path
from scipy.stats import chi2_contingency, f_oneway
from statsmodels.stats.multicomp import pairwise_tukeyhsd

#podesavanje ispisa (na dve decimale)
pd.set_option('display.float_format', lambda x: '%.2f' % x)
DATA_PATH = Path("city_temperature.csv")

##ucitavanje info
df = pd.read_csv(DATA_PATH)

###dopuna podataka
#-menjanje svih u avgtemperature sa -99 i nultog dana na nan
df.loc[df["AvgTemperature"] == -99, "AvgTemperature"] = np.nan
df.loc[df["Day"] == 0, "Day"] = np.nan
#...kao i godina
df["Year"] = df["Year"].replace({200: np.nan, 201: np.nan})

#izbacivanje suvisne KOLONE + forward fill temp
df = df.drop("State", axis=1)
df["AvgTemperature"] = df["AvgTemperature"].ffill()

#izbacivanje nan VRSTA
df = df.dropna(axis=0)

print(df.shape)
print(df.head()) #d

###grupisanje i pivot tabele
grp_city_month = (df.groupby(["Region","Country","City","Month"], as_index = False)\
                 ["AvgTemperature"].mean())
print(grp_city_month.head(10)) #d

#pivotiranje....
pivot_city_month = grp_city_month.pivot_table(index="City", columns="Month", \
                                              values = "AvgTemperature")
print(pivot_city_month.head()) #d

#heatmap vizuelizacija
plt.figure(figsize=(10,6))
sb.heatmap(pivot_city_month.iloc[:10], annot=False, cmap="coolwarm", linewidths=.5)
plt.title("Heatmap: AvgTemperatre City by Month (mean)")
plt.tight_layout()
plt.show()

###LINIJSKI GRAFIKONI
#primer 1. promena temperature po jednom gradu

top_city = df["City"].value_counts().index[0]
#izdvajanje po jednom gradu
sub = df[df["City"] == top_city] 

monthly_city = (sub.groupby("Month", as_index=False)["AvgTemperature"].mean()\
                .sort_values("Month"))

plt.figure(figsize=(6,4))
#plot po X i Y respektivno
plt.plot(monthly_city["Month"], monthly_city["AvgTemperature"], marker="o")
plt.title(f"Temp mesec {top_city}")
plt.xlabel("Month")
plt.ylabel("AvgTemperature")
#ovde od 1 do 13 (1-12 bez 13 tj. jan-dec)
plt.xticks(range(1,13))

#namestanje grida
plt.grid(True)
plt.show()

#primer 2. trend promene temperature kroz godine
city_year = (df.groupby(["City","Year"],as_index=False)\
             ["AvgTemperature"].mean())

#eksplicitno izdvajanje city_yeara po proslom odabranom gradu
cy = city_year[city_year["City"] == "Portland"]

plt.figure(figsize=(6,4))
plt.plot(cy["Year"], cy["AvgTemperature"],marker="o")
plt.title("Annual trend, Portland")
plt.xlabel("Year")
plt.ylabel("AvgTemperature")
plt.grid(True)
plt.show()

#hmm....pojavio se ekstrem
#utvrdjuje se:
df_city = df.loc[df["City"] == 'Portland']
df_city_2020 = df_city.loc[df_city["Year"] == 2020]
print(df_city_2020["Month"].unique())
#ispisace se [1, 2, 3, 4, 5] - nema podataka za celu '20. godinu

#primer 3. prosecna temperatura po region x month
g_rm = df.groupby(["Region","Month"],as_index=False)["AvgTemperature"].mean()

plt.figure(figsize=(10,5))
sb.lineplot(data=g_rm, x="Month", y="AvgTemperature", hue="Region", marker="o")
plt.title("AvgTemperature per month per region")
plt.xlabel("Month")
plt.ylabel("AvgTemperature (°C)")

#bbox to anchor
plt.legend(title="Region", bbox_to_anchor=(1.02, 1), loc="upper left")
plt.tight_layout()
plt.show()

###BAR GRAFIKONI
#primer 1. top 10 gradova po prosecnoj temperaturi

#najpre se izdvaja top 10
city_mean = df.groupby("City", as_index = True)["AvgTemperature"].mean().\
            sort_values(ascending=False).head(10)

plt.figure(figsize=(9,4))
sb.barplot(x=city_mean.index.astype(str), y=city_mean.values)
plt.title("Top 10 cities")
plt.xlabel("City")
plt.ylabel("AvgTemperature (°F)")
plt.xticks(rotation=45, ha="right")
plt.tight_layout()
plt.show()

#primer 2. prosecna temperatura po regionu po mesecima
regions = ["Europe", "Australia/South Pacific"]

#izdvajanje samo onih kolona iz dataframe cija se imena nalaze u regions listi
g2 = df[df["Region"].isin(regions)]

sb.barplot(data=g2, x="Month", y="AvgTemperature", hue="Region")
plt.legend(title="Region", bbox_to_anchor=(1.02, 1), loc="upper left")
plt.show()

###ANALIZA KATEGORICKIH VARIJABLI - VALUE_COUNTS
#najzastupljenijih 10 gradova i zemalja

#izdvajanje i prikaz u listi
vc_city = df["City"].value_counts(normalize=True).head(10) * 100
print(vc_city.round(2).astype(str) + "%") #d


plt.figure(figsize=(9,4))
sb.barplot(x=vc_city.index.astype(str), y=vc_city.values)
plt.title("Top 10 Cities")
plt.xlabel("City")
plt.ylabel("%")
plt.xticks(rotation=45, ha="right")
plt.tight_layout()
plt.show()

#....i top 10 zemalja
vc_country = df["Country"].value_counts(normalize=True).head(10) * 100
print(vc_country.round(2).astype(str) + "%") #d

plt.figure(figsize=(9,4))
sb.barplot(x=vc_country.index.astype(str), y=vc_country.values)
plt.title("Top-10 Country (procenat učešća)")
plt.ylabel("Učešće (%)"); plt.xlabel("Country")
plt.xticks(rotation=45, ha="right")
plt.tight_layout()
plt.show()

###TABELA KONTIGENCIJE
#za to treba da se ubaci "from scipy.stats import chi2_contingency"

#10 zemalja sa najvise uzoraka
ten_countries = df["Country"].value_counts.head(10).index
sub = df[df["Country"].isin(ten_countries)]
ct = pd.crosstab(sub["Region"], sub["Country"])
print(ct) #d

#hi na kvadrat
chi2, p, dof, expected = chi2_contingency(ct)
print(f"chi2={chi2:.2f}, dof={dof}, p-value={p:.3e}")

# trivijalno je test pozitivan, znaci hi2 je visok a p je blizu 0; varijable za regione
# i drzave medjusobno zavisne (zemlja jednoznacno odgovara regionu)

###JEDNOFAKTORSKA ANOVA
#postoji li statisticki znacanja razlika izmedju prosecnih vrednosti varijable za vise grupa (kontinent)
#za to treba da se urade: 
# from scipy.stats import f_oneway
# from statsmodels.stats.multicomp import pairwise_tukeyhsd

groups = [g["AvgTemperature"].values for _, g in df.groupby("Region")]

F, p = f_oneway(*groups)
print(f"ANOVA F={F:.2f}, p-value={p:.3e}")
print(pairwise_tukeyhsd(df["AvgTemperature"], df["Region"]))