talksik/pacman_reinforcement
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

attempt at prioritized

Browse Source
This commit is contained in:
Arjun Patel
2019-03-07 16:54:58 -08:00
parent b177b47969
commit 3cbe8b49a2
6 changed files with 853 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
analysis.py
+23 -16
View File
@@ -21,50 +21,57 @@
def question2():
answerDiscount = 0.9
answerNoise = 0.2
answerNoise = 0
return answerDiscount, answerNoise
def question3a():
answerDiscount = None
answerNoise = None
answerLivingReward = None
answerDiscount = .1
answerNoise = 0
answerLivingReward = .8
return answerDiscount, answerNoise, answerLivingReward
# If not possible, return 'NOT POSSIBLE'
def question3b():
answerDiscount = None
answerNoise = None
answerLivingReward = None
answerDiscount = .1
answerNoise = .2
answerLivingReward = .8
return answerDiscount, answerNoise, answerLivingReward
# If not possible, return 'NOT POSSIBLE'
def question3c():
answerDiscount = None
answerNoise = None
answerLivingReward = None
answerDiscount = .5
answerNoise = 0
answerLivingReward = .5
return answerDiscount, answerNoise, answerLivingReward
# If not possible, return 'NOT POSSIBLE'
def question3d():
answerDiscount = None
answerNoise = None
answerLivingReward = None
answerDiscount = .9
answerNoise = .1
answerLivingReward = .2
return answerDiscount, answerNoise, answerLivingReward
# If not possible, return 'NOT POSSIBLE'
def question3e():
answerDiscount = None
answerNoise = None
answerLivingReward = None
answerDiscount = 0
answerNoise = 0
answerLivingReward = 0
return answerDiscount, answerNoise, answerLivingReward
# If not possible, return 'NOT POSSIBLE'
def question8():
answerEpsilon = None
answerLearningRate = None
return answerEpsilon, answerLearningRate
# If not possible, return 'NOT POSSIBLE'
if __name__ == '__main__':
print('Answers to analysis questions:')
import analysis
test_cases/q5/1-tinygrid.test_output
+150
View File
@@ -0,0 +1,150 @@
Values at iteration 0 are correct.
Student/correct solution:
values_k_0: """
0.0000 0.0000 0.0000
"""
Q-Values at iteration 0 for action south are correct.
Student/correct solution:
q_values_k_0_action_south: """
illegal 0.0000 illegal
"""
Q-Values at iteration 0 for action north are correct.
Student/correct solution:
q_values_k_0_action_north: """
illegal 0.0000 illegal
"""
Q-Values at iteration 0 for action exit are correct.
Student/correct solution:
q_values_k_0_action_exit: """
10.0000 illegal -10.0000
"""
Q-Values at iteration 0 for action west are correct.
Student/correct solution:
q_values_k_0_action_west: """
illegal 0.0000 illegal
"""
Q-Values at iteration 0 for action east are correct.
Student/correct solution:
q_values_k_0_action_east: """
illegal 0.0000 illegal
"""
Values at iteration 1 are correct.
Student/correct solution:
values_k_1: """
10.0000 0.0000 0.0000
"""
Q-Values at iteration 1 for action south are correct.
Student/correct solution:
q_values_k_1_action_south: """
illegal 0.0000 illegal
"""
Q-Values at iteration 1 for action north are correct.
Student/correct solution:
q_values_k_1_action_north: """
illegal 0.0000 illegal
"""
Q-Values at iteration 1 for action exit are correct.
Student/correct solution:
q_values_k_1_action_exit: """
10.0000 illegal -10.0000
"""
Q-Values at iteration 1 for action west are correct.
Student/correct solution:
q_values_k_1_action_west: """
illegal 5.0000 illegal
"""
Q-Values at iteration 1 for action east are correct.
Student/correct solution:
q_values_k_1_action_east: """
illegal 0.0000 illegal
"""
Values at iteration 2 are NOT correct.
Student solution:
values_k_2: """
10.0000 5.0000 0.0000
"""
Correct solution:
values_k_2: """
10.0000 0.0000 -10.0000
"""
Q-Values at iteration 2 for action south are NOT correct.
Student solution:
q_values_k_2_action_south: """
illegal 2.5000 illegal
"""
Correct solution:
q_values_k_2_action_south: """
illegal 0.0000 illegal
"""
Q-Values at iteration 2 for action north are NOT correct.
Student solution:
q_values_k_2_action_north: """
illegal 2.5000 illegal
"""
Correct solution:
q_values_k_2_action_north: """
illegal 0.0000 illegal
"""
Q-Values at iteration 2 for action exit are correct.
Student/correct solution:
q_values_k_2_action_exit: """
10.0000 illegal -10.0000
"""
Q-Values at iteration 2 for action west are correct.
Student/correct solution:
q_values_k_2_action_west: """
illegal 5.0000 illegal
"""
Q-Values at iteration 2 for action east are NOT correct.
Student solution:
q_values_k_2_action_east: """
illegal 0.0000 illegal
"""
Correct solution:
q_values_k_2_action_east: """
illegal -5.0000 illegal
"""
test_cases/q5/2-tinygrid-noisy.test_output
+156
View File
@@ -0,0 +1,156 @@
Values at iteration 0 are correct.
Student/correct solution:
values_k_0: """
0.0000 0.0000 0.0000
"""
Q-Values at iteration 0 for action south are correct.
Student/correct solution:
q_values_k_0_action_south: """
illegal 0.0000 illegal
"""
Q-Values at iteration 0 for action north are correct.
Student/correct solution:
q_values_k_0_action_north: """
illegal 0.0000 illegal
"""
Q-Values at iteration 0 for action exit are correct.
Student/correct solution:
q_values_k_0_action_exit: """
10.0000 illegal -10.0000
"""
Q-Values at iteration 0 for action west are correct.
Student/correct solution:
q_values_k_0_action_west: """
illegal 0.0000 illegal
"""
Q-Values at iteration 0 for action east are correct.
Student/correct solution:
q_values_k_0_action_east: """
illegal 0.0000 illegal
"""
Values at iteration 1 are correct.
Student/correct solution:
values_k_1: """
10.0000 0.0000 0.0000
"""
Q-Values at iteration 1 for action south are correct.
Student/correct solution:
q_values_k_1_action_south: """
illegal 0.9375 illegal
"""
Q-Values at iteration 1 for action north are correct.
Student/correct solution:
q_values_k_1_action_north: """
illegal 0.9375 illegal
"""
Q-Values at iteration 1 for action exit are correct.
Student/correct solution:
q_values_k_1_action_exit: """
10.0000 illegal -10.0000
"""
Q-Values at iteration 1 for action west are correct.
Student/correct solution:
q_values_k_1_action_west: """
illegal 5.6250 illegal
"""
Q-Values at iteration 1 for action east are correct.
Student/correct solution:
q_values_k_1_action_east: """
illegal 0.0000 illegal
"""
Values at iteration 2 are NOT correct.
Student solution:
values_k_2: """
10.0000 5.6250 0.0000
"""
Correct solution:
values_k_2: """
10.0000 0.0000 -10.0000
"""
Q-Values at iteration 2 for action south are NOT correct.
Student solution:
q_values_k_2_action_south: """
illegal 4.1016 illegal
"""
Correct solution:
q_values_k_2_action_south: """
illegal 0.0000 illegal
"""
Q-Values at iteration 2 for action north are NOT correct.
Student solution:
q_values_k_2_action_north: """
illegal 4.1016 illegal
"""
Correct solution:
q_values_k_2_action_north: """
illegal 0.0000 illegal
"""
Q-Values at iteration 2 for action exit are correct.
Student/correct solution:
q_values_k_2_action_exit: """
10.0000 illegal -10.0000
"""
Q-Values at iteration 2 for action west are NOT correct.
Student solution:
q_values_k_2_action_west: """
illegal 6.6797 illegal
"""
Correct solution:
q_values_k_2_action_west: """
illegal 5.6250 illegal
"""
Q-Values at iteration 2 for action east are NOT correct.
Student solution:
q_values_k_2_action_east: """
illegal 1.0547 illegal
"""
Correct solution:
q_values_k_2_action_east: """
illegal -5.6250 illegal
"""
test_cases/q5/3-bridge.test_output
+194
View File
@@ -0,0 +1,194 @@
Values at iteration 0 are correct.
Student/correct solution:
values_k_0: """
__________ 0.0000 0.0000 0.0000 0.0000 0.0000 __________
0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000
__________ 0.0000 0.0000 0.0000 0.0000 0.0000 __________
"""
Q-Values at iteration 0 for action south are correct.
Student/correct solution:
q_values_k_0_action_south: """
__________ illegal illegal illegal illegal illegal __________
illegal 0.0000 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Q-Values at iteration 0 for action north are correct.
Student/correct solution:
q_values_k_0_action_north: """
__________ illegal illegal illegal illegal illegal __________
illegal 0.0000 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Q-Values at iteration 0 for action exit are correct.
Student/correct solution:
q_values_k_0_action_exit: """
__________ -100.0000 -100.0000 -100.0000 -100.0000 -100.0000 __________
1.0000 illegal illegal illegal illegal illegal 10.0000
__________ -100.0000 -100.0000 -100.0000 -100.0000 -100.0000 __________
"""
Q-Values at iteration 0 for action west are correct.
Student/correct solution:
q_values_k_0_action_west: """
__________ illegal illegal illegal illegal illegal __________
illegal 0.0000 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Q-Values at iteration 0 for action east are correct.
Student/correct solution:
q_values_k_0_action_east: """
__________ illegal illegal illegal illegal illegal __________
illegal 0.0000 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Values at iteration 1 are correct.
Student/correct solution:
values_k_1: """
__________ 0.0000 0.0000 0.0000 0.0000 0.0000 __________
0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000
__________ -100.0000 0.0000 0.0000 0.0000 0.0000 __________
"""
Q-Values at iteration 1 for action south are correct.
Student/correct solution:
q_values_k_1_action_south: """
__________ illegal illegal illegal illegal illegal __________
illegal -76.5000 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Q-Values at iteration 1 for action north are correct.
Student/correct solution:
q_values_k_1_action_north: """
__________ illegal illegal illegal illegal illegal __________
illegal 0.0000 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Q-Values at iteration 1 for action exit are correct.
Student/correct solution:
q_values_k_1_action_exit: """
__________ -100.0000 -100.0000 -100.0000 -100.0000 -100.0000 __________
1.0000 illegal illegal illegal illegal illegal 10.0000
__________ -100.0000 -100.0000 -100.0000 -100.0000 -100.0000 __________
"""
Q-Values at iteration 1 for action west are correct.
Student/correct solution:
q_values_k_1_action_west: """
__________ illegal illegal illegal illegal illegal __________
illegal -4.2500 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Q-Values at iteration 1 for action east are correct.
Student/correct solution:
q_values_k_1_action_east: """
__________ illegal illegal illegal illegal illegal __________
illegal -4.2500 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Values at iteration 2 are NOT correct.
Student solution:
values_k_2: """
__________ 0.0000 0.0000 0.0000 0.0000 0.0000 __________
0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000
__________ -100.0000 0.0000 0.0000 0.0000 0.0000 __________
"""
Correct solution:
values_k_2: """
__________ -100.0000 0.0000 0.0000 0.0000 0.0000 __________
0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000
__________ -100.0000 0.0000 0.0000 0.0000 0.0000 __________
"""
Q-Values at iteration 2 for action south are correct.
Student/correct solution:
q_values_k_2_action_south: """
__________ illegal illegal illegal illegal illegal __________
illegal -76.5000 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Q-Values at iteration 2 for action north are NOT correct.
Student solution:
q_values_k_2_action_north: """
__________ illegal illegal illegal illegal illegal __________
illegal 0.0000 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Correct solution:
q_values_k_2_action_north: """
__________ illegal illegal illegal illegal illegal __________
illegal -76.5000 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Q-Values at iteration 2 for action exit are correct.
Student/correct solution:
q_values_k_2_action_exit: """
__________ -100.0000 -100.0000 -100.0000 -100.0000 -100.0000 __________
1.0000 illegal illegal illegal illegal illegal 10.0000
__________ -100.0000 -100.0000 -100.0000 -100.0000 -100.0000 __________
"""
Q-Values at iteration 2 for action west are NOT correct.
Student solution:
q_values_k_2_action_west: """
__________ illegal illegal illegal illegal illegal __________
illegal -4.2500 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Correct solution:
q_values_k_2_action_west: """
__________ illegal illegal illegal illegal illegal __________
illegal -8.5000 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Q-Values at iteration 2 for action east are NOT correct.
Student solution:
q_values_k_2_action_east: """
__________ illegal illegal illegal illegal illegal __________
illegal -4.2500 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
Correct solution:
q_values_k_2_action_east: """
__________ illegal illegal illegal illegal illegal __________
illegal -8.5000 0.0000 0.0000 0.0000 0.0000 illegal
__________ illegal illegal illegal illegal illegal __________
"""
test_cases/q5/4-discountgrid.test_output
+238
View File
@@ -0,0 +1,238 @@
Values at iteration 0 are correct.
Student/correct solution:
values_k_0: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 __________ 0.0000
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 0.0000 0.0000 0.0000 0.0000
"""
Q-Values at iteration 0 for action south are correct.
Student/correct solution:
q_values_k_0_action_south: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
0.0000 0.0000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Q-Values at iteration 0 for action north are correct.
Student/correct solution:
q_values_k_0_action_north: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
0.0000 0.0000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Q-Values at iteration 0 for action exit are correct.
Student/correct solution:
q_values_k_0_action_exit: """
illegal illegal illegal illegal illegal
illegal __________ illegal illegal illegal
illegal __________ 1.0000 __________ 10.0000
illegal illegal illegal illegal illegal
-10.0000 -10.0000 -10.0000 -10.0000 -10.0000
"""
Q-Values at iteration 0 for action west are correct.
Student/correct solution:
q_values_k_0_action_west: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
0.0000 0.0000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Q-Values at iteration 0 for action east are correct.
Student/correct solution:
q_values_k_0_action_east: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
0.0000 0.0000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Values at iteration 1 are correct.
Student/correct solution:
values_k_1: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 __________ 0.0000
0.0000 0.0000 0.0000 0.0000 0.0000
-10.0000 0.0000 0.0000 0.0000 0.0000
"""
Q-Values at iteration 1 for action south are correct.
Student/correct solution:
q_values_k_1_action_south: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
-7.2000 0.0000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Q-Values at iteration 1 for action north are correct.
Student/correct solution:
q_values_k_1_action_north: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
0.0000 0.0000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Q-Values at iteration 1 for action exit are correct.
Student/correct solution:
q_values_k_1_action_exit: """
illegal illegal illegal illegal illegal
illegal __________ illegal illegal illegal
illegal __________ 1.0000 __________ 10.0000
illegal illegal illegal illegal illegal
-10.0000 -10.0000 -10.0000 -10.0000 -10.0000
"""
Q-Values at iteration 1 for action west are correct.
Student/correct solution:
q_values_k_1_action_west: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
-0.9000 0.0000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Q-Values at iteration 1 for action east are correct.
Student/correct solution:
q_values_k_1_action_east: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
-0.9000 0.0000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Values at iteration 2 are NOT correct.
Student solution:
values_k_2: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 __________ 0.0000
0.0000 0.0000 0.0000 0.0000 0.0000
-10.0000 0.0000 0.0000 0.0000 0.0000
"""
Correct solution:
values_k_2: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 __________ 0.0000
0.0000 0.0000 0.0000 0.0000 0.0000
-10.0000 -10.0000 0.0000 0.0000 0.0000
"""
Q-Values at iteration 2 for action south are NOT correct.
Student solution:
q_values_k_2_action_south: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
-7.2000 0.0000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_2_action_south: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
-7.2000 -7.2000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Q-Values at iteration 2 for action north are correct.
Student/correct solution:
q_values_k_2_action_north: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
0.0000 0.0000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Q-Values at iteration 2 for action exit are correct.
Student/correct solution:
q_values_k_2_action_exit: """
illegal illegal illegal illegal illegal
illegal __________ illegal illegal illegal
illegal __________ 1.0000 __________ 10.0000
illegal illegal illegal illegal illegal
-10.0000 -10.0000 -10.0000 -10.0000 -10.0000
"""
Q-Values at iteration 2 for action west are NOT correct.
Student solution:
q_values_k_2_action_west: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
-0.9000 0.0000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_2_action_west: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
-0.9000 -0.9000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Q-Values at iteration 2 for action east are NOT correct.
Student solution:
q_values_k_2_action_east: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
-0.9000 0.0000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
Correct solution:
q_values_k_2_action_east: """
0.0000 0.0000 0.0000 0.0000 0.0000
0.0000 __________ 0.0000 0.0000 0.0000
0.0000 __________ illegal __________ illegal
-0.9000 -0.9000 0.0000 0.0000 0.0000
illegal illegal illegal illegal illegal
"""
valueIterationAgents.py
+91
View File
@@ -174,6 +174,24 @@ class AsynchronousValueIterationAgent(ValueIterationAgent):
def runValueIteration(self):
"*** YOUR CODE HERE ***"
INF, NEG_INF = float("inf"), -float("inf")
# MDP states
mdp_states = self.mdp.getStates()
# Run through iterations
for i in range(self.iterations):
# copy function defined?
curr_state = mdp_states[i % len(mdp_states)]
# not iterating all actions this time
if not self.mdp.isTerminal(curr_state):
options_actions = self.mdp.getPossibleActions(curr_state)
optimal = max([self.getQValue(curr_state, x)
for x in options_actions])
# add optimal to the policy
self.values[curr_state] = optimal
class PrioritizedSweepingValueIterationAgent(AsynchronousValueIterationAgent):
@@ -196,3 +214,76 @@ class PrioritizedSweepingValueIterationAgent(AsynchronousValueIterationAgent):
def runValueIteration(self):
"*** YOUR CODE HERE ***"
# initiliaze empty PQ
# Use priority queue from utils for algorithm order
hinge = util.PriorityQueue()
dictPrev = {}
mdp_states = self.mdp.getStates()
# computing the predecssors for all states
# For each non-terminal state, do:
for curr_state in mdp_states:
# exit the iteration
if self.mdp.isTerminal(curr_state):
continue
options_actions = self.mdp.getPossibleActions(curr_state)
for action in options_actions:
all_transitions = self.mdp.getTransitionStatesAndProbs(
curr_state, action)
for new_state, prob in all_transitions:
if new_state in dictPrev:
dictPrev[new_state].add(curr_state)
else:
dictPrev[new_state] = {curr_state}
mdp_states = self.mdp.getStates()
# Find the absolute value of the difference between the current value of s in self.values and the highest Q-value across all possible actions from s (this represents what the value should be); call this number diff. Do NOT update self.values[s] in this step.
# Push s into the priority queue with priority -diff (note that this is negative). We use a negative because the priority queue is a min heap, but we want to prioritize updating states that have a higher error.
for curr_state in mdp_states:
if not self.mdp.isTerminal(curr_state):
options_actions = self.mdp.getPossibleActions(curr_state)
optimal = max([self.getQValue(curr_state, x)
for x in options_actions])
# finding -diff
diff = abs(optimal - self.values[curr_state])
hinge.update(curr_state, - diff)
# For iterations
# For iteration in 0, 1, 2, ..., self.iterations - 1, do:
# If the priority queue is empty, then terminate.
# Pop a state s off the priority queue.
# Update s's value (if it is not a terminal state) in self.values.
# For each predecessor p of s, do:
# Find the absolute value of the difference between the current value of p in self.values and the highest Q-value across all possible actions from p (this represents what the value should be); call this number diff. Do NOT update self.values[p] in this step.
# If diff > theta, push p into the priority queue with priority -diff (note that this is negative), as long as it does not already exist in the priority queue with equal or lower priority. As before, we use a negative because the priority queue is a min heap, but we want to prioritize updating states that have a higher error.
for i in range(self.iterations):
# no processing to do
if hinge.isEmpty():
break
curr_state = hinge.pop()
if not self.mdp.isTerminal(curr_state):
options_actions = self.mdp.getPossibleActions(curr_state)
optimal = max([self.getQValue(curr_state, x)
for x in options_actions])
self.values[curr_state] = optimal
for prev in dictPrev[curr_state]:
if self.mdp.isTerminal(prev):
continue
options_actions = self.mdp.getPossibleActions(curr_state)
optimal = max([self.getQValue(curr_state, x)
for x in options_actions])
diff = abs(optimal - self.values[prev])
# difference large enough?
if diff > self.theta:
hinge.update(prev, -diff)