From 3cbe8b49a23601254d7b6aa0fb9317d356eabe71 Mon Sep 17 00:00:00 2001 From: Arjun Patel Date: Thu, 7 Mar 2019 16:54:58 -0800 Subject: [PATCH] attempt at prioritized --- analysis.py | 41 ++-- test_cases/q5/1-tinygrid.test_output | 150 +++++++++++++ test_cases/q5/2-tinygrid-noisy.test_output | 156 ++++++++++++++ test_cases/q5/3-bridge.test_output | 194 +++++++++++++++++ test_cases/q5/4-discountgrid.test_output | 238 +++++++++++++++++++++ valueIterationAgents.py | 91 ++++++++ 6 files changed, 853 insertions(+), 17 deletions(-) create mode 100644 test_cases/q5/1-tinygrid.test_output create mode 100644 test_cases/q5/2-tinygrid-noisy.test_output create mode 100644 test_cases/q5/3-bridge.test_output create mode 100644 test_cases/q5/4-discountgrid.test_output diff --git a/analysis.py b/analysis.py index 5e8ca5a..53dcdd1 100644 --- a/analysis.py +++ b/analysis.py @@ -4,7 +4,7 @@ # educational purposes provided that (1) you do not distribute or publish # solutions, (2) you retain this notice, and (3) you provide clear # attribution to UC Berkeley, including a link to http://ai.berkeley.edu. -# +# # Attribution Information: The Pacman AI projects were developed at UC Berkeley. # The core projects and autograders were primarily created by John DeNero # (denero@cs.berkeley.edu) and Dan Klein (klein@cs.berkeley.edu). @@ -21,50 +21,57 @@ def question2(): answerDiscount = 0.9 - answerNoise = 0.2 + answerNoise = 0 return answerDiscount, answerNoise + def question3a(): - answerDiscount = None - answerNoise = None - answerLivingReward = None + answerDiscount = .1 + answerNoise = 0 + answerLivingReward = .8 return answerDiscount, answerNoise, answerLivingReward # If not possible, return 'NOT POSSIBLE' + def question3b(): - answerDiscount = None - answerNoise = None - answerLivingReward = None + answerDiscount = .1 + answerNoise = .2 + answerLivingReward = .8 return answerDiscount, answerNoise, answerLivingReward # If not possible, return 'NOT POSSIBLE' + def question3c(): - answerDiscount = None - answerNoise = None - answerLivingReward = None + answerDiscount = .5 + answerNoise = 0 + answerLivingReward = .5 return answerDiscount, answerNoise, answerLivingReward # If not possible, return 'NOT POSSIBLE' + def question3d(): - answerDiscount = None - answerNoise = None - answerLivingReward = None + answerDiscount = .9 + answerNoise = .1 + answerLivingReward = .2 return answerDiscount, answerNoise, answerLivingReward # If not possible, return 'NOT POSSIBLE' + def question3e(): - answerDiscount = None - answerNoise = None - answerLivingReward = None + answerDiscount = 0 + answerNoise = 0 + answerLivingReward = 0 return answerDiscount, answerNoise, answerLivingReward # If not possible, return 'NOT POSSIBLE' + def question8(): answerEpsilon = None answerLearningRate = None return answerEpsilon, answerLearningRate # If not possible, return 'NOT POSSIBLE' + if __name__ == '__main__': print('Answers to analysis questions:') import analysis diff --git a/test_cases/q5/1-tinygrid.test_output b/test_cases/q5/1-tinygrid.test_output new file mode 100644 index 0000000..3698ae0 --- /dev/null +++ b/test_cases/q5/1-tinygrid.test_output @@ -0,0 +1,150 @@ +Values at iteration 0 are correct. + Student/correct solution: + values_k_0: """ + 0.0000 0.0000 0.0000 +""" + + +Q-Values at iteration 0 for action south are correct. + Student/correct solution: + q_values_k_0_action_south: """ + illegal 0.0000 illegal +""" + + +Q-Values at iteration 0 for action north are correct. + Student/correct solution: + q_values_k_0_action_north: """ + illegal 0.0000 illegal +""" + + +Q-Values at iteration 0 for action exit are correct. + Student/correct solution: + q_values_k_0_action_exit: """ + 10.0000 illegal -10.0000 +""" + + +Q-Values at iteration 0 for action west are correct. + Student/correct solution: + q_values_k_0_action_west: """ + illegal 0.0000 illegal +""" + + +Q-Values at iteration 0 for action east are correct. + Student/correct solution: + q_values_k_0_action_east: """ + illegal 0.0000 illegal +""" + + +Values at iteration 1 are correct. + Student/correct solution: + values_k_1: """ + 10.0000 0.0000 0.0000 +""" + + +Q-Values at iteration 1 for action south are correct. + Student/correct solution: + q_values_k_1_action_south: """ + illegal 0.0000 illegal +""" + + +Q-Values at iteration 1 for action north are correct. + Student/correct solution: + q_values_k_1_action_north: """ + illegal 0.0000 illegal +""" + + +Q-Values at iteration 1 for action exit are correct. + Student/correct solution: + q_values_k_1_action_exit: """ + 10.0000 illegal -10.0000 +""" + + +Q-Values at iteration 1 for action west are correct. + Student/correct solution: + q_values_k_1_action_west: """ + illegal 5.0000 illegal +""" + + +Q-Values at iteration 1 for action east are correct. + Student/correct solution: + q_values_k_1_action_east: """ + illegal 0.0000 illegal +""" + + +Values at iteration 2 are NOT correct. + Student solution: + values_k_2: """ + 10.0000 5.0000 0.0000 +""" + + + Correct solution: + values_k_2: """ + 10.0000 0.0000 -10.0000 +""" + + +Q-Values at iteration 2 for action south are NOT correct. + Student solution: + q_values_k_2_action_south: """ + illegal 2.5000 illegal +""" + + + Correct solution: + q_values_k_2_action_south: """ + illegal 0.0000 illegal +""" + + +Q-Values at iteration 2 for action north are NOT correct. + Student solution: + q_values_k_2_action_north: """ + illegal 2.5000 illegal +""" + + + Correct solution: + q_values_k_2_action_north: """ + illegal 0.0000 illegal +""" + + +Q-Values at iteration 2 for action exit are correct. + Student/correct solution: + q_values_k_2_action_exit: """ + 10.0000 illegal -10.0000 +""" + + +Q-Values at iteration 2 for action west are correct. + Student/correct solution: + q_values_k_2_action_west: """ + illegal 5.0000 illegal +""" + + +Q-Values at iteration 2 for action east are NOT correct. + Student solution: + q_values_k_2_action_east: """ + illegal 0.0000 illegal +""" + + + Correct solution: + q_values_k_2_action_east: """ + illegal -5.0000 illegal +""" + + diff --git a/test_cases/q5/2-tinygrid-noisy.test_output b/test_cases/q5/2-tinygrid-noisy.test_output new file mode 100644 index 0000000..6d309b9 --- /dev/null +++ b/test_cases/q5/2-tinygrid-noisy.test_output @@ -0,0 +1,156 @@ +Values at iteration 0 are correct. + Student/correct solution: + values_k_0: """ + 0.0000 0.0000 0.0000 +""" + + +Q-Values at iteration 0 for action south are correct. + Student/correct solution: + q_values_k_0_action_south: """ + illegal 0.0000 illegal +""" + + +Q-Values at iteration 0 for action north are correct. + Student/correct solution: + q_values_k_0_action_north: """ + illegal 0.0000 illegal +""" + + +Q-Values at iteration 0 for action exit are correct. + Student/correct solution: + q_values_k_0_action_exit: """ + 10.0000 illegal -10.0000 +""" + + +Q-Values at iteration 0 for action west are correct. + Student/correct solution: + q_values_k_0_action_west: """ + illegal 0.0000 illegal +""" + + +Q-Values at iteration 0 for action east are correct. + Student/correct solution: + q_values_k_0_action_east: """ + illegal 0.0000 illegal +""" + + +Values at iteration 1 are correct. + Student/correct solution: + values_k_1: """ + 10.0000 0.0000 0.0000 +""" + + +Q-Values at iteration 1 for action south are correct. + Student/correct solution: + q_values_k_1_action_south: """ + illegal 0.9375 illegal +""" + + +Q-Values at iteration 1 for action north are correct. + Student/correct solution: + q_values_k_1_action_north: """ + illegal 0.9375 illegal +""" + + +Q-Values at iteration 1 for action exit are correct. + Student/correct solution: + q_values_k_1_action_exit: """ + 10.0000 illegal -10.0000 +""" + + +Q-Values at iteration 1 for action west are correct. + Student/correct solution: + q_values_k_1_action_west: """ + illegal 5.6250 illegal +""" + + +Q-Values at iteration 1 for action east are correct. + Student/correct solution: + q_values_k_1_action_east: """ + illegal 0.0000 illegal +""" + + +Values at iteration 2 are NOT correct. + Student solution: + values_k_2: """ + 10.0000 5.6250 0.0000 +""" + + + Correct solution: + values_k_2: """ + 10.0000 0.0000 -10.0000 +""" + + +Q-Values at iteration 2 for action south are NOT correct. + Student solution: + q_values_k_2_action_south: """ + illegal 4.1016 illegal +""" + + + Correct solution: + q_values_k_2_action_south: """ + illegal 0.0000 illegal +""" + + +Q-Values at iteration 2 for action north are NOT correct. + Student solution: + q_values_k_2_action_north: """ + illegal 4.1016 illegal +""" + + + Correct solution: + q_values_k_2_action_north: """ + illegal 0.0000 illegal +""" + + +Q-Values at iteration 2 for action exit are correct. + Student/correct solution: + q_values_k_2_action_exit: """ + 10.0000 illegal -10.0000 +""" + + +Q-Values at iteration 2 for action west are NOT correct. + Student solution: + q_values_k_2_action_west: """ + illegal 6.6797 illegal +""" + + + Correct solution: + q_values_k_2_action_west: """ + illegal 5.6250 illegal +""" + + +Q-Values at iteration 2 for action east are NOT correct. + Student solution: + q_values_k_2_action_east: """ + illegal 1.0547 illegal +""" + + + Correct solution: + q_values_k_2_action_east: """ + illegal -5.6250 illegal +""" + + diff --git a/test_cases/q5/3-bridge.test_output b/test_cases/q5/3-bridge.test_output new file mode 100644 index 0000000..737397c --- /dev/null +++ b/test_cases/q5/3-bridge.test_output @@ -0,0 +1,194 @@ +Values at iteration 0 are correct. + Student/correct solution: + values_k_0: """ + __________ 0.0000 0.0000 0.0000 0.0000 0.0000 __________ + 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 + __________ 0.0000 0.0000 0.0000 0.0000 0.0000 __________ +""" + + +Q-Values at iteration 0 for action south are correct. + Student/correct solution: + q_values_k_0_action_south: """ + __________ illegal illegal illegal illegal illegal __________ + illegal 0.0000 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + +Q-Values at iteration 0 for action north are correct. + Student/correct solution: + q_values_k_0_action_north: """ + __________ illegal illegal illegal illegal illegal __________ + illegal 0.0000 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + +Q-Values at iteration 0 for action exit are correct. + Student/correct solution: + q_values_k_0_action_exit: """ + __________ -100.0000 -100.0000 -100.0000 -100.0000 -100.0000 __________ + 1.0000 illegal illegal illegal illegal illegal 10.0000 + __________ -100.0000 -100.0000 -100.0000 -100.0000 -100.0000 __________ +""" + + +Q-Values at iteration 0 for action west are correct. + Student/correct solution: + q_values_k_0_action_west: """ + __________ illegal illegal illegal illegal illegal __________ + illegal 0.0000 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + +Q-Values at iteration 0 for action east are correct. + Student/correct solution: + q_values_k_0_action_east: """ + __________ illegal illegal illegal illegal illegal __________ + illegal 0.0000 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + +Values at iteration 1 are correct. + Student/correct solution: + values_k_1: """ + __________ 0.0000 0.0000 0.0000 0.0000 0.0000 __________ + 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 + __________ -100.0000 0.0000 0.0000 0.0000 0.0000 __________ +""" + + +Q-Values at iteration 1 for action south are correct. + Student/correct solution: + q_values_k_1_action_south: """ + __________ illegal illegal illegal illegal illegal __________ + illegal -76.5000 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + +Q-Values at iteration 1 for action north are correct. + Student/correct solution: + q_values_k_1_action_north: """ + __________ illegal illegal illegal illegal illegal __________ + illegal 0.0000 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + +Q-Values at iteration 1 for action exit are correct. + Student/correct solution: + q_values_k_1_action_exit: """ + __________ -100.0000 -100.0000 -100.0000 -100.0000 -100.0000 __________ + 1.0000 illegal illegal illegal illegal illegal 10.0000 + __________ -100.0000 -100.0000 -100.0000 -100.0000 -100.0000 __________ +""" + + +Q-Values at iteration 1 for action west are correct. + Student/correct solution: + q_values_k_1_action_west: """ + __________ illegal illegal illegal illegal illegal __________ + illegal -4.2500 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + +Q-Values at iteration 1 for action east are correct. + Student/correct solution: + q_values_k_1_action_east: """ + __________ illegal illegal illegal illegal illegal __________ + illegal -4.2500 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + +Values at iteration 2 are NOT correct. + Student solution: + values_k_2: """ + __________ 0.0000 0.0000 0.0000 0.0000 0.0000 __________ + 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 + __________ -100.0000 0.0000 0.0000 0.0000 0.0000 __________ +""" + + + Correct solution: + values_k_2: """ + __________ -100.0000 0.0000 0.0000 0.0000 0.0000 __________ + 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 + __________ -100.0000 0.0000 0.0000 0.0000 0.0000 __________ +""" + + +Q-Values at iteration 2 for action south are correct. + Student/correct solution: + q_values_k_2_action_south: """ + __________ illegal illegal illegal illegal illegal __________ + illegal -76.5000 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + +Q-Values at iteration 2 for action north are NOT correct. + Student solution: + q_values_k_2_action_north: """ + __________ illegal illegal illegal illegal illegal __________ + illegal 0.0000 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + + Correct solution: + q_values_k_2_action_north: """ + __________ illegal illegal illegal illegal illegal __________ + illegal -76.5000 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + +Q-Values at iteration 2 for action exit are correct. + Student/correct solution: + q_values_k_2_action_exit: """ + __________ -100.0000 -100.0000 -100.0000 -100.0000 -100.0000 __________ + 1.0000 illegal illegal illegal illegal illegal 10.0000 + __________ -100.0000 -100.0000 -100.0000 -100.0000 -100.0000 __________ +""" + + +Q-Values at iteration 2 for action west are NOT correct. + Student solution: + q_values_k_2_action_west: """ + __________ illegal illegal illegal illegal illegal __________ + illegal -4.2500 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + + Correct solution: + q_values_k_2_action_west: """ + __________ illegal illegal illegal illegal illegal __________ + illegal -8.5000 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + +Q-Values at iteration 2 for action east are NOT correct. + Student solution: + q_values_k_2_action_east: """ + __________ illegal illegal illegal illegal illegal __________ + illegal -4.2500 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + + Correct solution: + q_values_k_2_action_east: """ + __________ illegal illegal illegal illegal illegal __________ + illegal -8.5000 0.0000 0.0000 0.0000 0.0000 illegal + __________ illegal illegal illegal illegal illegal __________ +""" + + diff --git a/test_cases/q5/4-discountgrid.test_output b/test_cases/q5/4-discountgrid.test_output new file mode 100644 index 0000000..eb83e9e --- /dev/null +++ b/test_cases/q5/4-discountgrid.test_output @@ -0,0 +1,238 @@ +Values at iteration 0 are correct. + Student/correct solution: + values_k_0: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 __________ 0.0000 + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 0.0000 0.0000 0.0000 0.0000 +""" + + +Q-Values at iteration 0 for action south are correct. + Student/correct solution: + q_values_k_0_action_south: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + 0.0000 0.0000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + +Q-Values at iteration 0 for action north are correct. + Student/correct solution: + q_values_k_0_action_north: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + 0.0000 0.0000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + +Q-Values at iteration 0 for action exit are correct. + Student/correct solution: + q_values_k_0_action_exit: """ + illegal illegal illegal illegal illegal + illegal __________ illegal illegal illegal + illegal __________ 1.0000 __________ 10.0000 + illegal illegal illegal illegal illegal + -10.0000 -10.0000 -10.0000 -10.0000 -10.0000 +""" + + +Q-Values at iteration 0 for action west are correct. + Student/correct solution: + q_values_k_0_action_west: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + 0.0000 0.0000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + +Q-Values at iteration 0 for action east are correct. + Student/correct solution: + q_values_k_0_action_east: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + 0.0000 0.0000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + +Values at iteration 1 are correct. + Student/correct solution: + values_k_1: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 __________ 0.0000 + 0.0000 0.0000 0.0000 0.0000 0.0000 + -10.0000 0.0000 0.0000 0.0000 0.0000 +""" + + +Q-Values at iteration 1 for action south are correct. + Student/correct solution: + q_values_k_1_action_south: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + -7.2000 0.0000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + +Q-Values at iteration 1 for action north are correct. + Student/correct solution: + q_values_k_1_action_north: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + 0.0000 0.0000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + +Q-Values at iteration 1 for action exit are correct. + Student/correct solution: + q_values_k_1_action_exit: """ + illegal illegal illegal illegal illegal + illegal __________ illegal illegal illegal + illegal __________ 1.0000 __________ 10.0000 + illegal illegal illegal illegal illegal + -10.0000 -10.0000 -10.0000 -10.0000 -10.0000 +""" + + +Q-Values at iteration 1 for action west are correct. + Student/correct solution: + q_values_k_1_action_west: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + -0.9000 0.0000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + +Q-Values at iteration 1 for action east are correct. + Student/correct solution: + q_values_k_1_action_east: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + -0.9000 0.0000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + +Values at iteration 2 are NOT correct. + Student solution: + values_k_2: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 __________ 0.0000 + 0.0000 0.0000 0.0000 0.0000 0.0000 + -10.0000 0.0000 0.0000 0.0000 0.0000 +""" + + + Correct solution: + values_k_2: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 __________ 0.0000 + 0.0000 0.0000 0.0000 0.0000 0.0000 + -10.0000 -10.0000 0.0000 0.0000 0.0000 +""" + + +Q-Values at iteration 2 for action south are NOT correct. + Student solution: + q_values_k_2_action_south: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + -7.2000 0.0000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + + Correct solution: + q_values_k_2_action_south: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + -7.2000 -7.2000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + +Q-Values at iteration 2 for action north are correct. + Student/correct solution: + q_values_k_2_action_north: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + 0.0000 0.0000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + +Q-Values at iteration 2 for action exit are correct. + Student/correct solution: + q_values_k_2_action_exit: """ + illegal illegal illegal illegal illegal + illegal __________ illegal illegal illegal + illegal __________ 1.0000 __________ 10.0000 + illegal illegal illegal illegal illegal + -10.0000 -10.0000 -10.0000 -10.0000 -10.0000 +""" + + +Q-Values at iteration 2 for action west are NOT correct. + Student solution: + q_values_k_2_action_west: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + -0.9000 0.0000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + + Correct solution: + q_values_k_2_action_west: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + -0.9000 -0.9000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + +Q-Values at iteration 2 for action east are NOT correct. + Student solution: + q_values_k_2_action_east: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + -0.9000 0.0000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + + Correct solution: + q_values_k_2_action_east: """ + 0.0000 0.0000 0.0000 0.0000 0.0000 + 0.0000 __________ 0.0000 0.0000 0.0000 + 0.0000 __________ illegal __________ illegal + -0.9000 -0.9000 0.0000 0.0000 0.0000 + illegal illegal illegal illegal illegal +""" + + diff --git a/valueIterationAgents.py b/valueIterationAgents.py index 1758a35..e7c7cff 100644 --- a/valueIterationAgents.py +++ b/valueIterationAgents.py @@ -174,6 +174,24 @@ class AsynchronousValueIterationAgent(ValueIterationAgent): def runValueIteration(self): "*** YOUR CODE HERE ***" + INF, NEG_INF = float("inf"), -float("inf") + + # MDP states + mdp_states = self.mdp.getStates() + + # Run through iterations + for i in range(self.iterations): + # copy function defined? + curr_state = mdp_states[i % len(mdp_states)] + + # not iterating all actions this time + if not self.mdp.isTerminal(curr_state): + options_actions = self.mdp.getPossibleActions(curr_state) + optimal = max([self.getQValue(curr_state, x) + for x in options_actions]) + + # add optimal to the policy + self.values[curr_state] = optimal class PrioritizedSweepingValueIterationAgent(AsynchronousValueIterationAgent): @@ -196,3 +214,76 @@ class PrioritizedSweepingValueIterationAgent(AsynchronousValueIterationAgent): def runValueIteration(self): "*** YOUR CODE HERE ***" + # initiliaze empty PQ + # Use priority queue from utils for algorithm order + hinge = util.PriorityQueue() + + dictPrev = {} + mdp_states = self.mdp.getStates() + + # computing the predecssors for all states + # For each non-terminal state, do: + + for curr_state in mdp_states: + # exit the iteration + if self.mdp.isTerminal(curr_state): + continue + + options_actions = self.mdp.getPossibleActions(curr_state) + for action in options_actions: + + all_transitions = self.mdp.getTransitionStatesAndProbs( + curr_state, action) + for new_state, prob in all_transitions: + + if new_state in dictPrev: + dictPrev[new_state].add(curr_state) + + else: + dictPrev[new_state] = {curr_state} + + mdp_states = self.mdp.getStates() + + # Find the absolute value of the difference between the current value of s in self.values and the highest Q-value across all possible actions from s (this represents what the value should be); call this number diff. Do NOT update self.values[s] in this step. + # Push s into the priority queue with priority -diff (note that this is negative). We use a negative because the priority queue is a min heap, but we want to prioritize updating states that have a higher error. + for curr_state in mdp_states: + if not self.mdp.isTerminal(curr_state): + options_actions = self.mdp.getPossibleActions(curr_state) + optimal = max([self.getQValue(curr_state, x) + for x in options_actions]) + # finding -diff + diff = abs(optimal - self.values[curr_state]) + hinge.update(curr_state, - diff) + + # For iterations + # For iteration in 0, 1, 2, ..., self.iterations - 1, do: + # If the priority queue is empty, then terminate. + # Pop a state s off the priority queue. + # Update s's value (if it is not a terminal state) in self.values. + # For each predecessor p of s, do: + # Find the absolute value of the difference between the current value of p in self.values and the highest Q-value across all possible actions from p (this represents what the value should be); call this number diff. Do NOT update self.values[p] in this step. + # If diff > theta, push p into the priority queue with priority -diff (note that this is negative), as long as it does not already exist in the priority queue with equal or lower priority. As before, we use a negative because the priority queue is a min heap, but we want to prioritize updating states that have a higher error. + for i in range(self.iterations): + # no processing to do + if hinge.isEmpty(): + break + + curr_state = hinge.pop() + if not self.mdp.isTerminal(curr_state): + options_actions = self.mdp.getPossibleActions(curr_state) + optimal = max([self.getQValue(curr_state, x) + for x in options_actions]) + + self.values[curr_state] = optimal + + for prev in dictPrev[curr_state]: + if self.mdp.isTerminal(prev): + continue + + options_actions = self.mdp.getPossibleActions(curr_state) + optimal = max([self.getQValue(curr_state, x) + for x in options_actions]) + diff = abs(optimal - self.values[prev]) + # difference large enough? + if diff > self.theta: + hinge.update(prev, -diff)