Homec4science
Diffusion ann-nims-game (master) Browse

ann-nims-game/montecarlo.py
77a5c85d1c04master

Last Change
Raw File

montecarlo.py
View Options

import random
import pandas as pd
import numpy as np
class MonteCarlo(object):
def __init__(self, ini_q=0.0, epsilon=0.1, alpha=0.2, gamma=0.9):
self.q = self.initQ(ini_q)
self.epsilon = epsilon
self.alpha = alpha
self.gamma = gamma
self.state_actions_this_episode = []
def getQ(self, state, action):
state = tuple(state)
action = tuple(action)
return self.q[action][state]
def setQ(self, state, action, val):
state = tuple(state)
action = tuple(action)
self.q[action][state] = val
def initQ(self, ini_q, max_obj=7):
actions=[]
for i in range(1,4):
for j in range(1,max_obj+1):
actions.append((i,j))
self.actions = actions
q = pd.DataFrame(columns=actions, dtype=np.float64)
for i in range(0,max_obj+1):
for j in range(0,max_obj+1):
for k in range(0,max_obj+1):
q = q.append(pd.Series([ini_q]*len(actions),
index=q.columns,
name=(i,j,k)))
return q
def decide(self, state):
elig_actions = [a for a in self.actions if state[a[0]-1] >= a[1]]
if random.random() < self.epsilon:
action = random.choice(elig_actions)
else:
q = [self.getQ(state, a) for a in self.actions if state[a[0]-1] >= a[1]]
maxQ = max(q)
count = q.count(maxQ)
if count > 1:
best = [i for i in range(len(elig_actions)) if q[i] == maxQ]
i = random.choice(best)
else:
i = q.index(maxQ)
action = elig_actions[i]
return list(action)
def exploit(self, state):
elig_actions = [a for a in self.actions if state[a[0]-1] >= a[1]]
q = [self.getQ(state, a) for a in self.actions if state[a[0]-1] >= a[1]]
maxQ = max(q)
count = q.count(maxQ)
if count > 1:
best = [i for i in range(len(elig_actions)) if q[i] == maxQ]
i = random.choice(best)
else:
i = q.index(maxQ)
action = elig_actions[i]
return list(action)
def append_state_actions(self, state, action):
self.state_actions_this_episode.append((state, action))
def learn(self, reward):
for i, (state, action) in enumerate(reversed(self.state_actions_this_episode)):
newv = reward / (i+1)
self.setQ(state, action, newv)
reward *= self.gamma
self.state_actions_this_episode = []
c4science · Help