find_most_common_word_nokeyword.py
f3a3b0f515a6master

find_most_common_word_nokeyword.py
View Options

			""" Finds the most common word in the description of participants for which we didn't find a role.
			"""

			import pandas as pd

			complete_data = pd.read_csv("../results/complete_dataset.csv",
			encoding="utf-8-sig")

			no_keyword_participants = complete_data.loc[complete_data["role"] == "no keyword found"]
			no_keyword_participants = no_keyword_participants.loc[no_keyword_participants["affiliation_category"] == "parties"]

			words_dict = dict()
			line_dict = dict()

			for description in no_keyword_participants["description"]:
			description = description.replace(";", " ")
			# description
			if description in line_dict:
			line_dict[description] += 1
			else:
			line_dict[description] = 1
			# words
			for word in description.split(" "):
			if word in words_dict:
			words_dict[word] += 1
			else:
			words_dict[word] = 1

			sorted_word_dict = sorted(words_dict.items(), key=lambda x: x[1], reverse=True)
			sorted_line_dict = sorted(line_dict.items(), key=lambda x: x[1], reverse=True)
			print("Most common words:")
			print(sorted_word_dict[:100])
			print("Most common lines:")
			print(sorted_line_dict[:100])