make_ds_rand.py
212df03765c4master

make_ds_rand.py
View Options

			#!/usr/bin/env python
			import numpy as np
			import pandas as pd
			import xarray as xr
			import os
			import sys
			import time
			import util
			from features import Training, Testing
			from meteo_data import Meteo_Reader

			data_path = sys.argv[1]
			dataset = sys.argv[2]
			features = sys.argv[3].split(",")
			targets = sys.argv[4].split(",")
			sample_size = float(sys.argv[5])*1000000
			val_ratio = float(sys.argv[6])
			year = 2001

			# path = os.path.join(data_path, 'raw_data')
			# filepath = os.path.join(path, ('%d-%02d_table.csv' %(year, month)))

			tt = util.Timer()
			ds = Meteo_Reader(os.path.join(data_path, 'raw_data'), variables = targets)
			ds.read_yearly(year)
			tbl = ds.make_table(ftrs = features)
			tt.stop()

			sample_size = min(sample_size, len(tbl))
			N_train = int(0.8*sample_size)
			N_test = int(0.2*sample_size)

			new_set = Training(data_path, dataset, features, targets, data_type = 'table');
			new_set.make_dataset(table = tbl.sample(N_train), test_table = tbl.sample(N_test))

			new_set.normalize_all(feature_norm = 'mean', target_norm = 'mean', val_ratio = val_ratio, force_normalization = True)