Initial commit

test_cases/q4/1-tinygrid.solution

@@ -0,0 +1,276 @@
+values_k_0: """
+            0.0000       0.0000       0.0000
+"""
+
+q_values_k_0_action_north: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_0_action_east: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_0_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_0_action_south: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_0_action_west: """
+           illegal       0.0000      illegal
+"""
+
+values_k_1: """
+            0.0000       0.0000       0.0000
+"""
+
+q_values_k_1_action_north: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_1_action_east: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_1_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_1_action_south: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_1_action_west: """
+           illegal       0.0000      illegal
+"""
+
+values_k_2: """
+           10.0000       0.0000       0.0000
+"""
+
+q_values_k_2_action_north: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_2_action_east: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_2_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_2_action_south: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_2_action_west: """
+           illegal       5.0000      illegal
+"""
+
+values_k_3: """
+           10.0000       5.0000       0.0000
+"""
+
+q_values_k_3_action_north: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_3_action_east: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_3_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_3_action_south: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_3_action_west: """
+           illegal       5.0000      illegal
+"""
+
+values_k_4: """
+           10.0000       5.0000     -10.0000
+"""
+
+q_values_k_4_action_north: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_4_action_east: """
+           illegal      -5.0000      illegal
+"""
+
+q_values_k_4_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_4_action_south: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_4_action_west: """
+           illegal       5.0000      illegal
+"""
+
+values_k_5: """
+           10.0000       5.0000     -10.0000
+"""
+
+q_values_k_5_action_north: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_5_action_east: """
+           illegal      -5.0000      illegal
+"""
+
+q_values_k_5_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_5_action_south: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_5_action_west: """
+           illegal       5.0000      illegal
+"""
+
+values_k_6: """
+           10.0000       5.0000     -10.0000
+"""
+
+q_values_k_6_action_north: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_6_action_east: """
+           illegal      -5.0000      illegal
+"""
+
+q_values_k_6_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_6_action_south: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_6_action_west: """
+           illegal       5.0000      illegal
+"""
+
+values_k_7: """
+           10.0000       5.0000     -10.0000
+"""
+
+q_values_k_7_action_north: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_7_action_east: """
+           illegal      -5.0000      illegal
+"""
+
+q_values_k_7_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_7_action_south: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_7_action_west: """
+           illegal       5.0000      illegal
+"""
+
+values_k_8: """
+           10.0000       5.0000     -10.0000
+"""
+
+q_values_k_8_action_north: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_8_action_east: """
+           illegal      -5.0000      illegal
+"""
+
+q_values_k_8_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_8_action_south: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_8_action_west: """
+           illegal       5.0000      illegal
+"""
+
+values_k_9: """
+           10.0000       5.0000     -10.0000
+"""
+
+q_values_k_9_action_north: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_9_action_east: """
+           illegal      -5.0000      illegal
+"""
+
+q_values_k_9_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_9_action_south: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_9_action_west: """
+           illegal       5.0000      illegal
+"""
+
+values_k_1000: """
+           10.0000       5.0000     -10.0000
+"""
+
+q_values_k_1000_action_north: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_1000_action_east: """
+           illegal      -5.0000      illegal
+"""
+
+q_values_k_1000_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_1000_action_south: """
+           illegal       2.5000      illegal
+"""
+
+q_values_k_1000_action_west: """
+           illegal       5.0000      illegal
+"""
+
+policy: """
+        exit         west         exit
+"""
+
+actions: """
+north
+east
+exit
+south
+west
+"""
+

test_cases/q4/1-tinygrid.test

@@ -0,0 +1,21 @@
+class: "AsynchronousValueIterationTest"
+
+# GridWorld specification
+#    _ is empty space
+#    numbers are terminal states with that value
+#    # is a wall
+#    S is a start state
+#
+grid: """
+   10    S  -10
+"""
+discount: "0.5"
+noise: "0.0"
+livingReward: "0.0"
+epsilon: "0.5"
+learningRate: "0.1"
+numExperiences: "100"
+valueIterations: "1000"
+iterations: "10000"
+
+

test_cases/q4/2-tinygrid-noisy.solution

@@ -0,0 +1,276 @@
+values_k_0: """
+            0.0000       0.0000       0.0000
+"""
+
+q_values_k_0_action_north: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_0_action_east: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_0_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_0_action_south: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_0_action_west: """
+           illegal       0.0000      illegal
+"""
+
+values_k_1: """
+            0.0000       0.0000       0.0000
+"""
+
+q_values_k_1_action_north: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_1_action_east: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_1_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_1_action_south: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_1_action_west: """
+           illegal       0.0000      illegal
+"""
+
+values_k_2: """
+           10.0000       0.0000       0.0000
+"""
+
+q_values_k_2_action_north: """
+           illegal       0.9375      illegal
+"""
+
+q_values_k_2_action_east: """
+           illegal       0.0000      illegal
+"""
+
+q_values_k_2_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_2_action_south: """
+           illegal       0.9375      illegal
+"""
+
+q_values_k_2_action_west: """
+           illegal       5.6250      illegal
+"""
+
+values_k_3: """
+           10.0000       5.6250       0.0000
+"""
+
+q_values_k_3_action_north: """
+           illegal       4.1016      illegal
+"""
+
+q_values_k_3_action_east: """
+           illegal       1.0547      illegal
+"""
+
+q_values_k_3_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_3_action_south: """
+           illegal       4.1016      illegal
+"""
+
+q_values_k_3_action_west: """
+           illegal       6.6797      illegal
+"""
+
+values_k_4: """
+           10.0000       5.6250     -10.0000
+"""
+
+q_values_k_4_action_north: """
+           illegal       3.1641      illegal
+"""
+
+q_values_k_4_action_east: """
+           illegal      -4.5703      illegal
+"""
+
+q_values_k_4_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_4_action_south: """
+           illegal       3.1641      illegal
+"""
+
+q_values_k_4_action_west: """
+           illegal       6.6797      illegal
+"""
+
+values_k_5: """
+           10.0000       5.6250     -10.0000
+"""
+
+q_values_k_5_action_north: """
+           illegal       3.1641      illegal
+"""
+
+q_values_k_5_action_east: """
+           illegal      -4.5703      illegal
+"""
+
+q_values_k_5_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_5_action_south: """
+           illegal       3.1641      illegal
+"""
+
+q_values_k_5_action_west: """
+           illegal       6.6797      illegal
+"""
+
+values_k_6: """
+           10.0000       5.6250     -10.0000
+"""
+
+q_values_k_6_action_north: """
+           illegal       3.1641      illegal
+"""
+
+q_values_k_6_action_east: """
+           illegal      -4.5703      illegal
+"""
+
+q_values_k_6_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_6_action_south: """
+           illegal       3.1641      illegal
+"""
+
+q_values_k_6_action_west: """
+           illegal       6.6797      illegal
+"""
+
+values_k_7: """
+           10.0000       6.6797     -10.0000
+"""
+
+q_values_k_7_action_north: """
+           illegal       3.7573      illegal
+"""
+
+q_values_k_7_action_east: """
+           illegal      -4.3726      illegal
+"""
+
+q_values_k_7_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_7_action_south: """
+           illegal       3.7573      illegal
+"""
+
+q_values_k_7_action_west: """
+           illegal       6.8774      illegal
+"""
+
+values_k_8: """
+           10.0000       6.6797     -10.0000
+"""
+
+q_values_k_8_action_north: """
+           illegal       3.7573      illegal
+"""
+
+q_values_k_8_action_east: """
+           illegal      -4.3726      illegal
+"""
+
+q_values_k_8_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_8_action_south: """
+           illegal       3.7573      illegal
+"""
+
+q_values_k_8_action_west: """
+           illegal       6.8774      illegal
+"""
+
+values_k_9: """
+           10.0000       6.6797     -10.0000
+"""
+
+q_values_k_9_action_north: """
+           illegal       3.7573      illegal
+"""
+
+q_values_k_9_action_east: """
+           illegal      -4.3726      illegal
+"""
+
+q_values_k_9_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_9_action_south: """
+           illegal       3.7573      illegal
+"""
+
+q_values_k_9_action_west: """
+           illegal       6.8774      illegal
+"""
+
+values_k_1000: """
+           10.0000       6.9231     -10.0000
+"""
+
+q_values_k_1000_action_north: """
+           illegal       3.8942      illegal
+"""
+
+q_values_k_1000_action_east: """
+           illegal      -4.3269      illegal
+"""
+
+q_values_k_1000_action_exit: """
+           10.0000      illegal     -10.0000
+"""
+
+q_values_k_1000_action_south: """
+           illegal       3.8942      illegal
+"""
+
+q_values_k_1000_action_west: """
+           illegal       6.9231      illegal
+"""
+
+policy: """
+        exit         west         exit
+"""
+
+actions: """
+north
+east
+exit
+south
+west
+"""
+

test_cases/q4/2-tinygrid-noisy.test

@@ -0,0 +1,21 @@
+class: "AsynchronousValueIterationTest"
+
+# GridWorld specification
+#    _ is empty space
+#    numbers are terminal states with that value
+#    # is a wall
+#    S is a start state
+#
+grid: """
+   10    S  -10
+"""
+discount: "0.75"
+noise: "0.25"
+livingReward: "0.0"
+epsilon: "0.5"
+learningRate: "0.1"
+numExperiences: "100"
+valueIterations: "1000"
+iterations: "10000"
+
+

test_cases/q4/3-bridge.solution

@@ -0,0 +1,410 @@
+values_k_0: """
+        __________       0.0000       0.0000       0.0000       0.0000       0.0000   __________
+            0.0000       0.0000       0.0000       0.0000       0.0000       0.0000       0.0000
+        __________       0.0000       0.0000       0.0000       0.0000       0.0000   __________
+"""
+
+q_values_k_0_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_0_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_0_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+q_values_k_0_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_0_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+values_k_1: """
+        __________       0.0000       0.0000       0.0000       0.0000       0.0000   __________
+            0.0000       0.0000       0.0000       0.0000       0.0000       0.0000       0.0000
+        __________       0.0000       0.0000       0.0000       0.0000       0.0000   __________
+"""
+
+q_values_k_1_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_1_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_1_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+q_values_k_1_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_1_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+values_k_2: """
+        __________       0.0000       0.0000       0.0000       0.0000       0.0000   __________
+            1.0000       0.0000       0.0000       0.0000       0.0000       0.0000       0.0000
+        __________       0.0000       0.0000       0.0000       0.0000       0.0000   __________
+"""
+
+q_values_k_2_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0425       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_2_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_2_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+q_values_k_2_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0425       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_2_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.7650       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+values_k_3: """
+        __________       0.0000       0.0000       0.0000       0.0000       0.0000   __________
+            1.0000       0.0000       0.0000       0.0000       0.0000       0.0000       0.0000
+        __________    -100.0000       0.0000       0.0000       0.0000       0.0000   __________
+"""
+
+q_values_k_3_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0425       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_3_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -4.2500       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_3_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+q_values_k_3_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.4575       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_3_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -3.4850       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+values_k_4: """
+        __________       0.0000       0.0000       0.0000       0.0000       0.0000   __________
+            1.0000       0.0425       0.0000       0.0000       0.0000       0.0000       0.0000
+        __________    -100.0000       0.0000       0.0000       0.0000       0.0000   __________
+"""
+
+q_values_k_4_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal       0.0425       0.0018       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_4_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -4.2500       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_4_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+q_values_k_4_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.4575       0.0018       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_4_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -3.4850       0.0325       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+values_k_5: """
+        __________    -100.0000       0.0000       0.0000       0.0000       0.0000   __________
+            1.0000       0.0425       0.0000       0.0000       0.0000       0.0000       0.0000
+        __________    -100.0000       0.0000       0.0000       0.0000       0.0000   __________
+"""
+
+q_values_k_5_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.4575       0.0018       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_5_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -8.5000       0.0000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_5_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+q_values_k_5_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.4575       0.0018       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_5_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -7.7350       0.0325       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+values_k_6: """
+        __________    -100.0000       0.0000       0.0000       0.0000       0.0000   __________
+            1.0000       0.0425       0.0000       0.0000       0.0000       0.0000       0.0000
+        __________    -100.0000    -100.0000       0.0000       0.0000       0.0000   __________
+"""
+
+q_values_k_6_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.4575       0.0018       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_6_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -8.5000      -4.2500       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_6_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+q_values_k_6_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.4575     -76.4982       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_6_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -7.7350      -4.2175       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+values_k_7: """
+        __________    -100.0000       0.0000       0.0000       0.0000       0.0000   __________
+            1.0000       0.0425       0.0018       0.0000       0.0000       0.0000       0.0000
+        __________    -100.0000    -100.0000       0.0000       0.0000       0.0000   __________
+"""
+
+q_values_k_7_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.4574       0.0018       0.0001       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_7_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -8.4986      -4.2500       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_7_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+q_values_k_7_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.4574     -76.4982       0.0001       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_7_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -7.7350      -4.2175       0.0014       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+values_k_8: """
+        __________    -100.0000    -100.0000       0.0000       0.0000       0.0000   __________
+            1.0000       0.0425       0.0018       0.0000       0.0000       0.0000       0.0000
+        __________    -100.0000    -100.0000       0.0000       0.0000       0.0000   __________
+"""
+
+q_values_k_8_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.4574     -76.4982       0.0001       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_8_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -8.4986      -8.5000       0.0000       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_8_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+q_values_k_8_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.4574     -76.4982       0.0001       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_8_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -7.7350      -8.4675       0.0014       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+values_k_9: """
+        __________    -100.0000    -100.0000       0.0000       0.0000       0.0000   __________
+            1.0000       0.0425       0.0018       0.0000       0.0000       0.0000       0.0000
+        __________    -100.0000    -100.0000    -100.0000       0.0000       0.0000   __________
+"""
+
+q_values_k_9_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.4574     -76.4982       0.0001       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_9_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -8.4986      -8.5000      -4.2500       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_9_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+q_values_k_9_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -76.4574     -76.4982     -76.4999       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_9_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -7.7350      -8.4675      -4.2486       0.0000       0.0000      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+values_k_1000: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      -7.7350     -14.4173     -15.4999      -9.1502      -0.8500      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+q_values_k_1000_action_north: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -77.0702     -77.4875     -77.5016     -77.1949     -76.4639      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_1000_action_east: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -19.5292     -20.3575     -15.4999      -9.1502      -0.8500      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_1000_action_exit: """
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+            1.0000      illegal      illegal      illegal      illegal      illegal      10.0000
+        __________    -100.0000    -100.0000    -100.0000    -100.0000    -100.0000   __________
+"""
+
+q_values_k_1000_action_south: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal     -77.0702     -77.4875     -77.5016     -77.1949     -76.4639      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+q_values_k_1000_action_west: """
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+           illegal      -7.7350     -14.4173     -19.5292     -20.3575     -15.4999      illegal
+        __________      illegal      illegal      illegal      illegal      illegal   __________
+"""
+
+policy: """
+        __________   exit         exit         exit         exit         exit         __________
+        exit         west         west         east         east         east         exit      
+        __________   exit         exit         exit         exit         exit         __________
+"""
+
+actions: """
+north
+east
+exit
+south
+west
+"""
+

test_cases/q4/3-bridge.test

@@ -0,0 +1,24 @@
+class: "AsynchronousValueIterationTest"
+
+# GridWorld specification
+#    _ is empty space
+#    numbers are terminal states with that value
+#    # is a wall
+#    S is a start state
+#
+grid: """
+    # -100 -100 -100 -100 -100    #
+    1    S    _    _    _    _   10
+    # -100 -100 -100 -100 -100    #
+"""
+gridName: "bridgeGrid"
+discount: "0.85"
+noise: "0.1"
+livingReward: "0.0"
+epsilon: "0.5"
+learningRate: "0.1"
+numExperiences: "500"
+valueIterations: "1000"
+iterations: "10000"
+
+

test_cases/q4/4-discountgrid.solution

@@ -0,0 +1,544 @@
+values_k_0: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+"""
+
+q_values_k_0_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_0_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_0_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+q_values_k_0_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_0_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+values_k_1: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+"""
+
+q_values_k_1_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_1_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_1_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+q_values_k_1_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_1_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+values_k_2: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+          -10.0000       0.0000       0.0000       0.0000       0.0000
+"""
+
+q_values_k_2_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_2_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_2_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+q_values_k_2_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -7.2000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_2_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+values_k_3: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+          -10.0000       0.0000       0.0000       0.0000       0.0000
+"""
+
+q_values_k_3_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_3_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_3_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+q_values_k_3_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -7.2000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_3_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+values_k_4: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+          -10.0000       0.0000       0.0000       0.0000       0.0000
+"""
+
+q_values_k_4_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_4_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_4_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+q_values_k_4_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -7.2000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_4_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+values_k_5: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+          -10.0000       0.0000       0.0000       0.0000       0.0000
+"""
+
+q_values_k_5_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_5_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_5_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+q_values_k_5_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -7.2000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_5_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+values_k_6: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+          -10.0000       0.0000       0.0000       0.0000       0.0000
+"""
+
+q_values_k_6_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_6_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_6_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+q_values_k_6_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -7.2000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_6_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+values_k_7: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+          -10.0000     -10.0000       0.0000       0.0000       0.0000
+"""
+
+q_values_k_7_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_7_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000      -0.9000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_7_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+q_values_k_7_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -7.2000      -7.2000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_7_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000      -0.9000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+values_k_8: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+          -10.0000     -10.0000       0.0000       0.0000       0.0000
+"""
+
+q_values_k_8_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_8_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000      -0.9000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_8_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+q_values_k_8_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -7.2000      -7.2000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_8_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000      -0.9000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+values_k_9: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000   __________       0.0000
+            0.0000       0.0000       0.0000       0.0000       0.0000
+          -10.0000     -10.0000       0.0000       0.0000       0.0000
+"""
+
+q_values_k_9_action_north: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+            0.0000       0.0000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_9_action_east: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000      -0.9000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_9_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+q_values_k_9_action_south: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -7.2000      -7.2000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_9_action_west: """
+            0.0000       0.0000       0.0000       0.0000       0.0000
+            0.0000   __________       0.0000       0.0000       0.0000
+            0.0000   __________      illegal   __________      illegal
+           -0.9000      -0.9000       0.0000       0.0000       0.0000
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+values_k_1000: """
+            4.4765       5.1665       5.8841       6.6836       7.5087
+            3.9306   __________       6.0258       7.5087       8.6547
+            3.4513   __________       1.0000   __________      10.0000
+            2.9289       2.0045       3.3074       5.7186       8.4777
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+q_values_k_1000_action_north: """
+            4.0910       4.6523       5.3030       6.0175       6.6836
+            3.9306   __________       5.4546       6.1334       6.8610
+            3.4513   __________      illegal   __________      illegal
+            2.9289       2.0045       1.4151       5.1780       8.4777
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_1000_action_east: """
+            4.4765       5.1665       5.8841       6.6836       6.8610
+            3.5435   __________       6.0258       7.5087       7.8072
+            3.1023   __________      illegal   __________      illegal
+            0.8539       1.6617       3.3074       5.7186       6.1039
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_1000_action_exit: """
+           illegal      illegal      illegal      illegal      illegal
+           illegal   __________      illegal      illegal      illegal
+           illegal   __________       1.0000   __________      10.0000
+           illegal      illegal      illegal      illegal      illegal
+          -10.0000     -10.0000     -10.0000     -10.0000     -10.0000
+"""
+
+q_values_k_1000_action_south: """
+            3.6979       4.6523       5.4051       6.6116       7.5087
+            3.1924   __________       1.9381       6.7275       8.6547
+            2.7300   __________      illegal   __________      illegal
+           -6.7560      -6.6387      -6.5049      -6.1393      -5.9223
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+q_values_k_1000_action_west: """
+            3.9797       4.1531       4.7918       5.5138       6.2669
+            3.5435   __________       4.9582       5.6159       6.9820
+            3.1023   __________      illegal   __________      illegal
+            1.5194       1.3892       0.6333       1.9960       4.1174
+           illegal      illegal      illegal      illegal      illegal
+"""
+
+policy: """
+        east         east         east         east         south     
+        north        __________   east         east         south     
+        north        __________   exit         __________   exit      
+        north        north        east         east         north     
+        exit         exit         exit         exit         exit
+"""
+
+actions: """
+north
+east
+exit
+south
+west
+"""
+

test_cases/q4/4-discountgrid.test

@@ -0,0 +1,25 @@
+class: "AsynchronousValueIterationTest"
+
+# GridWorld specification
+#    _ is empty space
+#    numbers are terminal states with that value
+#    # is a wall
+#    S is a start state
+#
+grid: """
+    _    _    _    _    _
+    _    #    _    _    _
+    _    #    1    #   10
+    S    _    _    _    _
+  -10  -10  -10  -10  -10
+"""
+discount: "0.9"
+noise: "0.2"
+livingReward: "0.0"
+epsilon: "0.2"
+learningRate: "0.1"
+numExperiences: "3000"
+valueIterations: "1000"
+iterations: "10000"
+
+

test_cases/q4/CONFIG

@@ -0,0 +1,2 @@
+max_points: "1"
+class: "PassAllTestsQuestion"