make_ds_rand_month.py
212df03765c4master

make_ds_rand_month.py
View Options

			#!/usr/bin/env python
			import numpy as np
			import pandas as pd
			import xarray as xr
			import os
			import sys
			import time
			import util
			from features import Training, Testing
			from meteo_data import Meteo_Reader

			data_path = sys.argv[1]
			dataset = sys.argv[2]
			features = sys.argv[3].split(",")
			targets = sys.argv[4].split(",")
			sample_size = float(sys.argv[5])*1000000
			val_ratio = float(sys.argv[6])
			month = int(sys.argv[7])
			year = 2001

			start_date, end_date = util.month_to_dates(year, month)

			tt = util.Timer()
			ds = Meteo_Reader(os.path.join(data_path, 'raw_data'), variables = targets)
			ds.read_data(start_date,end_date)
			tbl = ds.make_table(ftrs = features)
			tt.stop()

			sample_size = min(sample_size, len(tbl))
			N_train = int(0.8*sample_size)
			N_test = int(0.2*sample_size)

			new_set = Training(data_path, dataset, features, targets, data_type = 'table');
			new_set.make_dataset(table = tbl.sample(N_train), test_table = tbl.sample(N_test))

			new_set.normalize_all(feature_norm = 'mean', target_norm = 'mean', val_ratio = val_ratio, force_normalization = True)