Homec4science
Diffusion cop-mining-participants (master) Browse

cop-mining-participants/code/scripts/prepare_intervention_data.py
06a11b90392cmaster

Last Change
Raw File

prepare_intervention_data.py
View Options

""" Prepare the intervention data for predictive modelling
"""
import pandas as pd
import country_converter as coco
import plots.plot_government as plot_government
import plots.plot_experience as plot_experience
import plots.plot_gender_rate as plot_gender_rate
import find_experience as exp
metadata = pd.read_csv("../data/meetings_metadata.csv")
issue_list = pd.read_csv("../data/data_tatiana/list_meetings.csv")
issue_dict = {}
countries_dict = {"european union": "european union"}
def is_processed_meeting(issue_number):
meeting_label = get_meeting_label(issue_number)
return meeting_label in set(metadata["label"])
def get_meeting_label(issue_number):
if issue_number in issue_dict:
return issue_dict[issue_number]
else:
meeting = issue_list.loc[issue_list["issue_number"] == issue_number]
label = (str(meeting.iloc[0, 0])).lower() + str(meeting.iloc[0, 1])
issue_dict[issue_number] = label
return label
def convert_country_names(country):
if country in countries_dict:
return countries_dict[country]
else:
converted = coco.convert(names=[(str(country)).lower()], to="name_short", not_found="None")
if isinstance(converted, list):
converted = converted[0]
countries_dict[country] = converted
return converted
# prepare Tatianas data
def prepare_intervention_data():
intervention_data = pd.read_csv("../data/data_tatiana/interventions.csv")
prepared_data = pd.DataFrame(columns={"meeting", "country", "nb_interventions"})
intervention_data = intervention_data.loc[intervention_data.issue_number.apply(is_processed_meeting)]
intervention_data["entity"] = intervention_data.entity.apply(convert_country_names)
intervention_data = intervention_data.loc[intervention_data["entity"] != "None"]
prepared_data["meeting"] = intervention_data["issue_number"].apply(get_meeting_label)
prepared_data["country"] = intervention_data["entity"]
prepared_data["nb_interventions"] = intervention_data["interventions"]
prepared_data.to_csv("../data/data_regression/interventions_prepared.csv",
encoding="utf-8-sig",
index=False)
# Prepare my data
def prepare_affiliation_data():
complete_data = pd.read_csv("../results/complete_dataset.csv",
encoding="utf-8-sig")
complete_data_with_experience = pd.read_csv("../results/complete_dataset_experience-def.csv",
encoding="utf-8-sig")
parties = complete_data.loc[complete_data["affiliation_category"] == "parties"]
parties_with_experience = complete_data_with_experience.loc[complete_data_with_experience["affiliation_category"] == "parties"]
prepared_data = pd.DataFrame(columns={"meeting", "country", "nb_delegates",
"meeting_type", "government_rate", "diplomacy_rate",
"security_rate", "press_rate", "no_description_rate", "no_role_rate",
"nb_fossil_fuel_industry", "woman_proportion", "experience score"})
for label in metadata["label"]:
this_meeting = parties_with_experience.loc[parties_with_experience["meeting"] == label]
by_country = this_meeting.groupby("affiliation")
for aff, people in by_country:
meeting_type = 0
if label.startswith("sb"):
meeting_type = 1
gov, dipl, sec, press, uni, nodescr, nokeyword = plot_government.get_roles(people)
"""experience_score = plot_experience.get_experience_score(list(people["experience"]))"""
experience_score_cop = exp.get_experience_score(list(people["experience cop"]))
experience_score_sb = exp.get_experience_score(list(people["experience sb"]))
experience_score_party_rate = (0 if experience_score_cop == 0 and experience_score_sb == 0 else
exp.get_experience_score(list(people["experience party"])) / (experience_score_cop + experience_score_sb))
# TODO does groupby work on a people object?
woman_proportion = plot_gender_rate.get_women_proportion(people)
# fossil fuel industry (not in the experience plot)
fossil_fuel_related = people[people.fossil_fuel_industry == 1]
nb_fossil_fuel_delegates = len(fossil_fuel_related)
"""nb_fossil_fuel_delegates = 0"""
prepared_data = prepared_data.append({
"meeting": label,
"country": aff,
"nb_delegates": len(people),
"meeting_type": meeting_type,
"government_rate": gov,
"diplomacy_rate": dipl,
"security_rate": sec,
"press_rate": press,
"university_rate": uni,
"no_description_rate": nodescr,
"no_role_rate": nokeyword,
"nb_fossil_fuel_industry": nb_fossil_fuel_delegates,
"woman_proportion": woman_proportion,
"experience score cop": experience_score_cop,
"experience score sb": experience_score_sb,
"experience score parties rate": experience_score_party_rate
}, ignore_index=True)
# fix order and store it
prepared_data = prepared_data[ ["meeting", "country", "nb_delegates",
"meeting_type", "government_rate", "diplomacy_rate",
"security_rate", "press_rate", "university_rate", "no_description_rate", "no_role_rate",
"nb_fossil_fuel_industry", "woman_proportion", "experience score cop",
"experience score sb", "experience score parties rate"]]
prepared_data.to_csv("../data/data_regression/interventions_prepared_aff.csv",
encoding="utf-8-sig",
index=False)
def fuse_datasets():
intervention_data = pd.read_csv("../data/data_regression/interventions_prepared.csv",
encoding="utf-8-sig")
affiliation_data = pd.read_csv("../data/data_regression/interventions_prepared_aff.csv",
encoding="utf-8-sig")
entire_dataset = pd.DataFrame(columns={"meeting", "country", "nb_delegates",
"meeting_type", "government_rate", "diplomacy_rate",
"security_rate", "press_rate", "university_rate", "no_description_rate", "no_role_rate",
"nb_fossil_fuel_industry", "woman_proportion", "experience score",
"nb_interventions"})
by_meeting = intervention_data.groupby("meeting")
affiliation_data_by_meeting = affiliation_data.groupby("meeting")
# default to 0
affiliation_data["nb_interventions"] = 0
for label, affiliations in by_meeting:
interventions_by_country = affiliations.groupby("country")
affiliation_data_meeting = affiliation_data_by_meeting.get_group(label)
for country, rest in interventions_by_country:
if country in set(affiliation_data["country"]):
sum = rest["nb_interventions"].sum()
affiliation_data.loc[(affiliation_data["meeting"] == label) & (affiliation_data["country"] == country), "nb_interventions"] = sum if sum > 0 else 0
entire_dataset = affiliation_data
entire_dataset.to_csv("../data/data_regression/dataset_interventions.csv",
encoding="utf-8-sig",
index=False)
# prepare_intervention_data()
prepare_affiliation_data()
fuse_datasets()
c4science · Help