extract_descriptions.py
f3a3b0f515a6master

extract_descriptions.py
View Options

			"""Analyzes the descriptions of all the participants in complete_dataset.csv"""

			import pandas as pd
			import numpy as np
			import re
			from collections import Counter

			from partlistproc.MeetingAnalyzer import MeetingAnalyzer


			descriptions = []

			participants = pd.read_csv("../results/complete_dataset.csv", encoding="utf-8-sig")

			for index, participant in participants.iterrows():
			description = str(participant["description"])
			description_list = re.split(MeetingAnalyzer.description_splitter,
			description)
			description_list = filter(None, description_list)
			#description_list = [str(line) for line in description_list]
			descriptions.extend(description_list)
			if index % 1000 == 0:
			print(index)


			#print(descriptions)
			print("Find the most common lines:")
			counter = Counter(descriptions)
			print("Totally found " + str(sum(counter.values())) + " distinct lines, the 20 most common being")
			print(counter.most_common(20))

			# save most common 200
			most_common_lines = counter.most_common(200)
			output_file = open("most_common_descriptions.txt", "a")
			for line, count in most_common_lines:
			output_file.write(str(count) + " times the line: " + line)
			output_file.write("\n")
			output_file.close()