13hannes11/UU_NCML_Project
1
0
Fork 0
mirror of https://github.com/13hannes11/UU_NCML_Project.git synced 2024-09-03 20:50:59 +02:00
Code Issues Projects Releases Wiki Activity

adapt preprocessing to new title loading

Browse Source
This commit is contained in:
Hannes Kuchelmeister
2021-04-26 18:29:00 +02:00
parent 2998717cb4
commit 5c0fce0384
1 changed files with 35 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
preprocess_de.py
View File

@@ -1,37 +1,50 @@
import pandas as pd
import os
title_file = "filename_to_titles.csv"
# Preprocess data
vote_counter = -1
data = pd.DataFrame()
name_column = 'Bezeichnung'
column_to_filename = {}
# Load filename to
filename_to_title_map = {}
with open(os.path.join("./de/csv/", title_file), 'r') as file:
for line in file.readlines():
filename, title = line.split(';', 2)
filename_to_title_map[filename] = title
vote_column_to_title = {}
voting_features = ['ja', 'nein', 'Enthaltung', 'ungültig']
for dirname, _, filenames in os.walk('./de/csv'):
for filename in filenames:
vote_counter += 1
print(os.path.join(dirname, filename))
df = pd.read_csv(os.path.join(dirname, filename))
# Give each voting behaviour type an identifier from 0 to len(voting_features) - 1
for i, feature in enumerate(voting_features):
df[feature] *= i
vote_column_name = f'vote_{vote_counter}'
# Map column name of vote to filename -> allows retrieving what the vote was about
column_to_filename[vote_column_name] = filename
# add feature for the vote
df[vote_column_name] = df[voting_features].sum(axis=1)
if data.empty:
# if first file that is loaded set data equal to data from first file
data = df[[name_column, vote_column_name]]
else:
# merge data with already loaded data
data = data.merge(df[[name_column, vote_column_name]], on=name_column)
if filename != title_file:
vote_counter += 1
print(os.path.join(dirname, filename))
df = pd.read_csv(os.path.join(dirname, filename))
# Give each voting behaviour type an identifier from 0 to len(voting_features) - 1
for i, feature in enumerate(voting_features):
df[feature] *= i
vote_column_name = f'vote_{vote_counter}'
# Map column name of vote to filename -> allows retrieving what the vote was about
vote_column_to_title[vote_column_name] = filename_to_title_map[filename]
# add feature for the vote
df[vote_column_name] = df[voting_features].sum(axis=1)
if data.empty:
# if first file that is loaded set data equal to data from first file
data = df[[name_column, vote_column_name]]
else:
# merge data with already loaded data
data = data.merge(df[[name_column, vote_column_name]], on=name_column)
print(column_to_filename)
print(vote_column_to_title)
print(data)