Homec4science
Diffusion ann-nims-game (master) Browse

ann-nims-game/sarsa_lambda.py
77a5c85d1c04master

Last Change
Raw File

sarsa_lambda.py
View Options

import random
import pandas as pd
import numpy as np
class SARSA_LAMBDA(object):
def __init__(self, ini_q=0.0, epsilon=0.1, alpha=0.2, gamma=0.9, lambda_=0.9):
self.q = self.initQ(ini_q)
self.e = self.initE(0.0)
self.epsilon = epsilon
self.alpha = alpha
self.gamma = gamma
self.lambda_ = lambda_
def resetE(self):
self.e = 0.0*self.q
def getQ(self, state, action):
state = tuple(state)
action = tuple(action)
return self.q[action][state]
def setQ(self, state, action, val):
state = tuple(state)
action = tuple(action)
self.e[action][state] = val
def getE(self, state, action):
state = tuple(state)
action = tuple(action)
return self.e[action][state]
def setE(self, state, action, val):
state = tuple(state)
action = tuple(action)
self.e[action][state] = val
def initQ(self, ini_q, max_obj=7):
actions=[]
for i in range(1,4):
for j in range(1,max_obj+1):
actions.append((i,j))
self.actions = actions
q = pd.DataFrame(columns=actions, dtype=np.float64)
for i in range(0,max_obj+1):
for j in range(0,max_obj+1):
for k in range(0,max_obj+1):
q = q.append(pd.Series([ini_q]*len(actions),
index=q.columns,
name=(i,j,k)))
return q
def initE(self, ini_q, max_obj=7):
actions=[]
for i in range(1,4):
for j in range(1,max_obj+1):
actions.append((i,j))
self.actions = actions
q = pd.DataFrame(columns=actions, dtype=np.float64)
for i in range(0,max_obj+1):
for j in range(0,max_obj+1):
for k in range(0,max_obj+1):
q = q.append(pd.Series([ini_q]*len(actions),
index=q.columns,
name=(i,j,k)))
return q
def learnQ(self, delta):
self.q = self.q + self.alpha*delta*self.e
self.e = self.gamma*self.lambda_*self.e
def decide(self, state):
elig_actions = [a for a in self.actions if state[a[0]-1] >= a[1]]
if random.random() < self.epsilon:
action = random.choice(elig_actions)
else:
q = [self.getQ(state, a) for a in self.actions if state[a[0]-1] >= a[1]]
maxQ = max(q)
count = q.count(maxQ)
if count > 1:
best = [i for i in range(len(elig_actions)) if q[i] == maxQ]
i = random.choice(best)
else:
i = q.index(maxQ)
action = elig_actions[i]
return list(action)
def exploit(self, state):
elig_actions = [a for a in self.actions if state[a[0]-1] >= a[1]]
q = [self.getQ(state, a) for a in self.actions if state[a[0]-1] >= a[1]]
maxQ = max(q)
count = q.count(maxQ)
if count > 1:
best = [i for i in range(len(elig_actions)) if q[i] == maxQ]
i = random.choice(best)
else:
i = q.index(maxQ)
action = elig_actions[i]
return list(action)
def learn(self, state1, action1, reward, state2=[0,0,0], action2=None):
# print('learning')
if state2 == [0, 0, 0]:
qnext = 0.0 # terminal state
else:
qnext = self.getQ(state2, action2)
delta = reward + self.gamma * qnext - self.getQ(state1, action1)
newE = self.getE(state1, action1) + 1
self.setE(state1, action1, newE)
self.learnQ(delta)
c4science · Help