attempt at prioritized

analysis.py

@@ -4,7 +4,7 @@
 # educational purposes provided that (1) you do not distribute or publish
 # solutions, (2) you retain this notice, and (3) you provide clear
 # attribution to UC Berkeley, including a link to http://ai.berkeley.edu.
-# 
+#
 # Attribution Information: The Pacman AI projects were developed at UC Berkeley.
 # The core projects and autograders were primarily created by John DeNero
 # ([email protected]) and Dan Klein ([email protected]).
@@ -21,50 +21,57 @@
 
 def question2():
     answerDiscount = 0.9
-    answerNoise = 0.2
+    answerNoise = 0
     return answerDiscount, answerNoise
 
+
 def question3a():
-    answerDiscount = None
+    answerDiscount = .1
-    answerNoise = None
+    answerNoise = 0
-    answerLivingReward = None
+    answerLivingReward = .8
     return answerDiscount, answerNoise, answerLivingReward
     # If not possible, return 'NOT POSSIBLE'
 
+
 def question3b():
-    answerDiscount = None
+    answerDiscount = .1
-    answerNoise = None
+    answerNoise = .2
-    answerLivingReward = None
+    answerLivingReward = .8
     return answerDiscount, answerNoise, answerLivingReward
     # If not possible, return 'NOT POSSIBLE'
 
+
 def question3c():
-    answerDiscount = None
+    answerDiscount = .5
-    answerNoise = None
+    answerNoise = 0
-    answerLivingReward = None
+    answerLivingReward = .5
     return answerDiscount, answerNoise, answerLivingReward
     # If not possible, return 'NOT POSSIBLE'
 
+
 def question3d():
-    answerDiscount = None
+    answerDiscount = .9
-    answerNoise = None
+    answerNoise = .1
-    answerLivingReward = None
+    answerLivingReward = .2
     return answerDiscount, answerNoise, answerLivingReward
     # If not possible, return 'NOT POSSIBLE'
 
+
 def question3e():
-    answerDiscount = None
+    answerDiscount = 0
-    answerNoise = None
+    answerNoise = 0
-    answerLivingReward = None
+    answerLivingReward = 0
     return answerDiscount, answerNoise, answerLivingReward
     # If not possible, return 'NOT POSSIBLE'
 
+
 def question8():
     answerEpsilon = None
     answerLearningRate = None
     return answerEpsilon, answerLearningRate
     # If not possible, return 'NOT POSSIBLE'
 
+
 if __name__ == '__main__':
     print('Answers to analysis questions:')
     import analysis

test_cases/q5/1-tinygrid.test_output

@@ -0,0 +1,150 @@
+Values at iteration 0 are correct.
+   Student/correct solution:
+ values_k_0: """
+            0.0000       0.0000       0.0000
+"""
+
+
+Q-Values at iteration 0 for action south are correct.
+   Student/correct solution:
+ q_values_k_0_action_south: """
+           illegal       0.0000      illegal
+"""
+
+
+Q-Values at iteration 0 for action north are correct.
+   Student/correct solution:
+ q_values_k_0_action_north: """
+           illegal       0.0000      illegal
+"""
+
+
+Q-Values at iteration 0 for action exit are correct.
+   Student/correct solution:
+ q_values_k_0_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+
+Q-Values at iteration 0 for action west are correct.
+   Student/correct solution:
+ q_values_k_0_action_west: """
+           illegal       0.0000      illegal
+"""
+
+
+Q-Values at iteration 0 for action east are correct.
+   Student/correct solution:
+ q_values_k_0_action_east: """
+           illegal       0.0000      illegal
+"""
+
+
+Values at iteration 1 are correct.
+   Student/correct solution:
+ values_k_1: """
+           10.0000       0.0000       0.0000
+"""
+
+
+Q-Values at iteration 1 for action south are correct.
+   Student/correct solution:
+ q_values_k_1_action_south: """
+           illegal       0.0000      illegal
+"""
+
+
+Q-Values at iteration 1 for action north are correct.
+   Student/correct solution:
+ q_values_k_1_action_north: """
+           illegal       0.0000      illegal
+"""
+
+
+Q-Values at iteration 1 for action exit are correct.
+   Student/correct solution:
+ q_values_k_1_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+
+Q-Values at iteration 1 for action west are correct.
+   Student/correct solution:
+ q_values_k_1_action_west: """
+           illegal       5.0000      illegal
+"""
+
+
+Q-Values at iteration 1 for action east are correct.
+   Student/correct solution:
+ q_values_k_1_action_east: """
+           illegal       0.0000      illegal
+"""
+
+
+Values at iteration 2 are NOT correct.
+   Student solution:
+ values_k_2: """
+           10.0000       5.0000       0.0000
+"""
+
+
+   Correct solution:
+ values_k_2: """
+           10.0000       0.0000     -10.0000
+"""
+
+
+Q-Values at iteration 2 for action south are NOT correct.
+   Student solution:
+ q_values_k_2_action_south: """
+           illegal       2.5000      illegal
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_south: """
+           illegal       0.0000      illegal
+"""
+
+
+Q-Values at iteration 2 for action north are NOT correct.
+   Student solution:
+ q_values_k_2_action_north: """
+           illegal       2.5000      illegal
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_north: """
+           illegal       0.0000      illegal
+"""
+
+
+Q-Values at iteration 2 for action exit are correct.
+   Student/correct solution:
+ q_values_k_2_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+
+Q-Values at iteration 2 for action west are correct.
+   Student/correct solution:
+ q_values_k_2_action_west: """
+           illegal       5.0000      illegal
+"""
+
+
+Q-Values at iteration 2 for action east are NOT correct.
+   Student solution:
+ q_values_k_2_action_east: """
+           illegal       0.0000      illegal
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_east: """
+           illegal      -5.0000      illegal
+"""
+
+

test_cases/q5/2-tinygrid-noisy.test_output

@@ -0,0 +1,156 @@
+Values at iteration 0 are correct.
+   Student/correct solution:
+ values_k_0: """
+            0.0000       0.0000       0.0000
+"""
+
+
+Q-Values at iteration 0 for action south are correct.
+   Student/correct solution:
+ q_values_k_0_action_south: """
+           illegal       0.0000      illegal
+"""
+
+
+Q-Values at iteration 0 for action north are correct.
+   Student/correct solution:
+ q_values_k_0_action_north: """
+           illegal       0.0000      illegal
+"""
+
+
+Q-Values at iteration 0 for action exit are correct.
+   Student/correct solution:
+ q_values_k_0_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+
+Q-Values at iteration 0 for action west are correct.
+   Student/correct solution:
+ q_values_k_0_action_west: """
+           illegal       0.0000      illegal
+"""
+
+
+Q-Values at iteration 0 for action east are correct.
+   Student/correct solution:
+ q_values_k_0_action_east: """
+           illegal       0.0000      illegal
+"""
+
+
+Values at iteration 1 are correct.
+   Student/correct solution:
+ values_k_1: """
+           10.0000       0.0000       0.0000
+"""
+
+
+Q-Values at iteration 1 for action south are correct.
+   Student/correct solution:
+ q_values_k_1_action_south: """
+           illegal       0.9375      illegal
+"""
+
+
+Q-Values at iteration 1 for action north are correct.
+   Student/correct solution:
+ q_values_k_1_action_north: """
+           illegal       0.9375      illegal
+"""
+
+
+Q-Values at iteration 1 for action exit are correct.
+   Student/correct solution:
+ q_values_k_1_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+
+Q-Values at iteration 1 for action west are correct.
+   Student/correct solution:
+ q_values_k_1_action_west: """
+           illegal       5.6250      illegal
+"""
+
+
+Q-Values at iteration 1 for action east are correct.
+   Student/correct solution:
+ q_values_k_1_action_east: """
+           illegal       0.0000      illegal
+"""
+
+
+Values at iteration 2 are NOT correct.
+   Student solution:
+ values_k_2: """
+           10.0000       5.6250       0.0000
+"""
+
+
+   Correct solution:
+ values_k_2: """
+           10.0000       0.0000     -10.0000
+"""
+
+
+Q-Values at iteration 2 for action south are NOT correct.
+   Student solution:
+ q_values_k_2_action_south: """
+           illegal       4.1016      illegal
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_south: """
+           illegal       0.0000      illegal
+"""
+
+
+Q-Values at iteration 2 for action north are NOT correct.
+   Student solution:
+ q_values_k_2_action_north: """
+           illegal       4.1016      illegal
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_north: """
+           illegal       0.0000      illegal
+"""
+
+
+Q-Values at iteration 2 for action exit are correct.
+   Student/correct solution:
+ q_values_k_2_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+
+Q-Values at iteration 2 for action west are NOT correct.
+   Student solution:
+ q_values_k_2_action_west: """
+           illegal       6.6797      illegal
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_west: """
+           illegal       5.6250      illegal
+"""
+
+
+Q-Values at iteration 2 for action east are NOT correct.
+   Student solution:
+ q_values_k_2_action_east: """
+           illegal       1.0547      illegal
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_east: """
+           illegal      -5.6250      illegal
+"""
+
+

test_cases/q5/3-bridge.test_output

@@ -0,0 +1,194 @@
+Values at iteration 0 are correct.
+   Student/correct solution:
+ values_k_0: """
+        __________       0.0000       0.0000       0.0000       0.0000       0.0000   __________
+            0.0000       0.0000       0.0000       0.0000       0.0000       0.0000       0.0000
+        __________       0.0000       0.0000       0.0000       0.0000       0.0000   __________
+"""
+
+
+Q-Values at iteration 0 for action south are correct.
+   Student/correct solution:
+ q_values_k_0_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+Q-Values at iteration 0 for action north are correct.
+   Student/correct solution:
+ q_values_k_0_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+Q-Values at iteration 0 for action exit are correct.
+   Student/correct solution:
+ q_values_k_0_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+
+Q-Values at iteration 0 for action west are correct.
+   Student/correct solution:
+ q_values_k_0_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+Q-Values at iteration 0 for action east are correct.
+   Student/correct solution:
+ q_values_k_0_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+Values at iteration 1 are correct.
+   Student/correct solution:
+ values_k_1: """
+        __________       0.0000       0.0000       0.0000       0.0000       0.0000   __________
+            0.0000       0.0000       0.0000       0.0000       0.0000       0.0000       0.0000
+        __________    -100.0000       0.0000       0.0000       0.0000       0.0000   __________
+"""
+
+
+Q-Values at iteration 1 for action south are correct.
+   Student/correct solution:
+ q_values_k_1_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.5000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+Q-Values at iteration 1 for action north are correct.
+   Student/correct solution:
+ q_values_k_1_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+Q-Values at iteration 1 for action exit are correct.
+   Student/correct solution:
+ q_values_k_1_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+
+Q-Values at iteration 1 for action west are correct.
+   Student/correct solution:
+ q_values_k_1_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -4.2500       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+Q-Values at iteration 1 for action east are correct.
+   Student/correct solution:
+ q_values_k_1_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -4.2500       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+Values at iteration 2 are NOT correct.
+   Student solution:
+ values_k_2: """
+        __________       0.0000       0.0000       0.0000       0.0000       0.0000   __________
+            0.0000       0.0000       0.0000       0.0000       0.0000       0.0000       0.0000
+        __________    -100.0000       0.0000       0.0000       0.0000       0.0000   __________
+"""
+
+
+   Correct solution:
+ values_k_2: """
+        __________    -100.0000       0.0000       0.0000       0.0000       0.0000   __________
+            0.0000       0.0000       0.0000       0.0000       0.0000       0.0000       0.0000
+        __________    -100.0000       0.0000       0.0000       0.0000       0.0000   __________
+"""
+
+
+Q-Values at iteration 2 for action south are correct.
+   Student/correct solution:
+ q_values_k_2_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.5000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+Q-Values at iteration 2 for action north are NOT correct.
+   Student solution:
+ q_values_k_2_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.5000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+Q-Values at iteration 2 for action exit are correct.
+   Student/correct solution:
+ q_values_k_2_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+
+Q-Values at iteration 2 for action west are NOT correct.
+   Student solution:
+ q_values_k_2_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -4.2500       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -8.5000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+Q-Values at iteration 2 for action east are NOT correct.
+   Student solution:
+ q_values_k_2_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -4.2500       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -8.5000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+

test_cases/q5/4-discountgrid.test_output

@@ -0,0 +1,238 @@
+Values at iteration 0 are correct.
+   Student/correct solution:
+ values_k_0: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+"""
+
+
+Q-Values at iteration 0 for action south are correct.
+   Student/correct solution:
+ q_values_k_0_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+Q-Values at iteration 0 for action north are correct.
+   Student/correct solution:
+ q_values_k_0_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+Q-Values at iteration 0 for action exit are correct.
+   Student/correct solution:
+ q_values_k_0_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+
+Q-Values at iteration 0 for action west are correct.
+   Student/correct solution:
+ q_values_k_0_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+Q-Values at iteration 0 for action east are correct.
+   Student/correct solution:
+ q_values_k_0_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+Values at iteration 1 are correct.
+   Student/correct solution:
+ values_k_1: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+          -10.0000       0.0000       0.0000       0.0000       0.0000
+"""
+
+
+Q-Values at iteration 1 for action south are correct.
+   Student/correct solution:
+ q_values_k_1_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -7.2000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+Q-Values at iteration 1 for action north are correct.
+   Student/correct solution:
+ q_values_k_1_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+Q-Values at iteration 1 for action exit are correct.
+   Student/correct solution:
+ q_values_k_1_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+
+Q-Values at iteration 1 for action west are correct.
+   Student/correct solution:
+ q_values_k_1_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+Q-Values at iteration 1 for action east are correct.
+   Student/correct solution:
+ q_values_k_1_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+Values at iteration 2 are NOT correct.
+   Student solution:
+ values_k_2: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+          -10.0000       0.0000       0.0000       0.0000       0.0000
+"""
+
+
+   Correct solution:
+ values_k_2: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+          -10.0000     -10.0000       0.0000       0.0000       0.0000
+"""
+
+
+Q-Values at iteration 2 for action south are NOT correct.
+   Student solution:
+ q_values_k_2_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -7.2000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -7.2000      -7.2000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+Q-Values at iteration 2 for action north are correct.
+   Student/correct solution:
+ q_values_k_2_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+Q-Values at iteration 2 for action exit are correct.
+   Student/correct solution:
+ q_values_k_2_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+
+Q-Values at iteration 2 for action west are NOT correct.
+   Student solution:
+ q_values_k_2_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000      -0.9000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+Q-Values at iteration 2 for action east are NOT correct.
+   Student solution:
+ q_values_k_2_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+
+   Correct solution:
+ q_values_k_2_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000      -0.9000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+

valueIterationAgents.py

@@ -174,6 +174,24 @@ class AsynchronousValueIterationAgent(ValueIterationAgent):
 
     def runValueIteration(self):
         "*** YOUR CODE HERE ***"
+        INF, NEG_INF = float("inf"), -float("inf")
+
+        # MDP states
+        mdp_states = self.mdp.getStates()
+
+        # Run through iterations
+        for i in range(self.iterations):
+            # copy function defined?
+            curr_state = mdp_states[i % len(mdp_states)]
+
+            # not iterating all actions this time
+            if not self.mdp.isTerminal(curr_state):
+                options_actions = self.mdp.getPossibleActions(curr_state)
+                optimal = max([self.getQValue(curr_state, x)
+                               for x in options_actions])
+
+                # add optimal to the policy
+                self.values[curr_state] = optimal
 
 
 class PrioritizedSweepingValueIterationAgent(AsynchronousValueIterationAgent):
@@ -196,3 +214,76 @@ class PrioritizedSweepingValueIterationAgent(AsynchronousValueIterationAgent):
 
     def runValueIteration(self):
         "*** YOUR CODE HERE ***"
+        # initiliaze empty PQ
+        # Use priority queue from utils for algorithm order
+        hinge = util.PriorityQueue()
+
+        dictPrev = {}
+        mdp_states = self.mdp.getStates()
+
+        # computing the predecssors for all states
+        # For each non-terminal state, do:
+
+        for curr_state in mdp_states:
+            # exit the iteration
+            if self.mdp.isTerminal(curr_state):
+                continue
+
+            options_actions = self.mdp.getPossibleActions(curr_state)
+            for action in options_actions:
+
+                all_transitions = self.mdp.getTransitionStatesAndProbs(
+                    curr_state, action)
+                for new_state, prob in all_transitions:
+
+                    if new_state in dictPrev:
+                        dictPrev[new_state].add(curr_state)
+
+                    else:
+                        dictPrev[new_state] = {curr_state}
+
+        mdp_states = self.mdp.getStates()
+
+        # Find the absolute value of the difference between the current value of s in self.values and the highest Q-value across all possible actions from s (this represents what the value should be); call this number diff. Do NOT update self.values[s] in this step.
+        # Push s into the priority queue with priority -diff (note that this is negative). We use a negative because the priority queue is a min heap, but we want to prioritize updating states that have a higher error.
+        for curr_state in mdp_states:
+            if not self.mdp.isTerminal(curr_state):
+                options_actions = self.mdp.getPossibleActions(curr_state)
+                optimal = max([self.getQValue(curr_state, x)
+                               for x in options_actions])
+                # finding -diff
+                diff = abs(optimal - self.values[curr_state])
+                hinge.update(curr_state, - diff)
+
+        # For iterations
+        #         For iteration in 0, 1, 2, ..., self.iterations - 1, do:
+        # If the priority queue is empty, then terminate.
+        # Pop a state s off the priority queue.
+        # Update s's value (if it is not a terminal state) in self.values.
+        # For each predecessor p of s, do:
+        # Find the absolute value of the difference between the current value of p in self.values and the highest Q-value across all possible actions from p (this represents what the value should be); call this number diff. Do NOT update self.values[p] in this step.
+        # If diff > theta, push p into the priority queue with priority -diff (note that this is negative), as long as it does not already exist in the priority queue with equal or lower priority. As before, we use a negative because the priority queue is a min heap, but we want to prioritize updating states that have a higher error.
+        for i in range(self.iterations):
+            # no processing to do
+            if hinge.isEmpty():
+                break
+
+            curr_state = hinge.pop()
+            if not self.mdp.isTerminal(curr_state):
+                options_actions = self.mdp.getPossibleActions(curr_state)
+                optimal = max([self.getQValue(curr_state, x)
+                               for x in options_actions])
+
+                self.values[curr_state] = optimal
+
+            for prev in dictPrev[curr_state]:
+                if self.mdp.isTerminal(prev):
+                    continue
+
+                options_actions = self.mdp.getPossibleActions(curr_state)
+                optimal = max([self.getQValue(curr_state, x)
+                               for x in options_actions])
+                diff = abs(optimal - self.values[prev])
+                # difference large enough?
+                if diff > self.theta:
+                    hinge.update(prev, -diff)