adding in setup files

autograder.py

@@ -0,0 +1,579 @@
+# A custom autograder for this project
+
+################################################################################
+# A mini-framework for autograding
+################################################################################
+
+import optparse
+import pickle
+import random
+import sys
+import traceback
+
+class WritableNull:
+    def write(self, string):
+        pass
+
+    def flush(self):
+        pass
+
+class Tracker(object):
+    def __init__(self, questions, maxes, prereqs, mute_output):
+        self.questions = questions
+        self.maxes = maxes
+        self.prereqs = prereqs
+
+        self.points = {q: 0 for q in self.questions}
+
+        self.current_question = None
+
+        self.current_test = None
+        self.points_at_test_start = None
+        self.possible_points_remaining = None
+
+        self.mute_output = mute_output
+        self.original_stdout = None
+        self.muted = False
+
+    def mute(self):
+        if self.muted:
+            return
+
+        self.muted = True
+        self.original_stdout = sys.stdout
+        sys.stdout = WritableNull()
+
+    def unmute(self):
+        if not self.muted:
+            return
+
+        self.muted = False
+        sys.stdout = self.original_stdout
+
+    def begin_q(self, q):
+        assert q in self.questions
+        text = 'Question {}'.format(q)
+        print('\n' + text)
+        print('=' * len(text))
+
+        for prereq in sorted(self.prereqs[q]):
+            if self.points[prereq] < self.maxes[prereq]:
+                print("""*** NOTE: Make sure to complete Question {} before working on Question {},
+*** because Question {} builds upon your answer for Question {}.
+""".format(prereq, q, q, prereq))
+                return False
+
+        self.current_question = q
+        self.possible_points_remaining = self.maxes[q]
+        return True
+
+    def begin_test(self, test_name):
+        self.current_test = test_name
+        self.points_at_test_start = self.points[self.current_question]
+        print("*** {}) {}".format(self.current_question, self.current_test))
+        if self.mute_output:
+            self.mute()
+
+    def end_test(self, pts):
+        if self.mute_output:
+            self.unmute()
+        self.possible_points_remaining -= pts
+        if self.points[self.current_question] == self.points_at_test_start + pts:
+            print("*** PASS: {}".format(self.current_test))
+        elif self.points[self.current_question] == self.points_at_test_start:
+            print("*** FAIL")
+
+        self.current_test = None
+        self.points_at_test_start = None
+
+    def end_q(self):
+        assert self.current_question is not None
+        assert self.possible_points_remaining == 0
+        print('\n### Question {}: {}/{} ###'.format(
+            self.current_question,
+            self.points[self.current_question],
+            self.maxes[self.current_question]))
+
+        self.current_question = None
+        self.possible_points_remaining = None
+
+    def finalize(self):
+        import time
+        print('\nFinished at %d:%02d:%02d' % time.localtime()[3:6])
+        print("\nProvisional grades\n==================")
+
+        for q in self.questions:
+          print('Question %s: %d/%d' % (q, self.points[q], self.maxes[q]))
+        print('------------------')
+        print('Total: %d/%d' % (sum(self.points.values()),
+            sum([self.maxes[q] for q in self.questions])))
+
+        print("""
+Your grades are NOT yet registered.  To register your grades, make sure
+to follow your instructor's guidelines to receive credit on your project.
+""")
+
+    def add_points(self, pts):
+        self.points[self.current_question] += pts
+
+TESTS = []
+PREREQS = {}
+def add_prereq(q, pre):
+    if isinstance(pre, str):
+        pre = [pre]
+
+    if q not in PREREQS:
+        PREREQS[q] = set()
+    PREREQS[q] |= set(pre)
+
+def test(q, points):
+    def deco(fn):
+        TESTS.append((q, points, fn))
+        return fn
+    return deco
+
+def parse_options(argv):
+    parser = optparse.OptionParser(description = 'Run public tests on student code')
+    parser.set_defaults(
+        edx_output=False,
+        gs_output=False,
+        no_graphics=False,
+        mute_output=False,
+        check_dependencies=False,
+        )
+    parser.add_option('--edx-output',
+                        dest = 'edx_output',
+                        action = 'store_true',
+                        help = 'Ignored, present for compatibility only')
+    parser.add_option('--gradescope-output',
+                        dest = 'gs_output',
+                        action = 'store_true',
+                        help = 'Ignored, present for compatibility only')
+    parser.add_option('--question', '-q',
+                        dest = 'grade_question',
+                        default = None,
+                        help = 'Grade only one question (e.g. `-q q1`)')
+    parser.add_option('--no-graphics',
+                        dest = 'no_graphics',
+                        action = 'store_true',
+                        help = 'Do not display graphics (visualizing your implementation is highly recommended for debugging).')
+    parser.add_option('--mute',
+                        dest = 'mute_output',
+                        action = 'store_true',
+                        help = 'Mute output from executing tests')
+    parser.add_option('--check-dependencies',
+                        dest = 'check_dependencies',
+                        action = 'store_true',
+                        help = 'check that numpy and matplotlib are installed')
+    (options, args) = parser.parse_args(argv)
+    return options
+
+def main():
+    options = parse_options(sys.argv)
+    if options.check_dependencies:
+        check_dependencies()
+        return
+
+    if options.no_graphics:
+        disable_graphics()
+
+    questions = set()
+    maxes = {}
+    for q, points, fn in TESTS:
+        questions.add(q)
+        maxes[q] = maxes.get(q, 0) + points
+        if q not in PREREQS:
+            PREREQS[q] = set()
+
+    questions = list(sorted(questions))
+    if options.grade_question:
+        if options.grade_question not in questions:
+            print("ERROR: question {} does not exist".format(options.grade_question))
+            sys.exit(1)
+        else:
+            questions = [options.grade_question]
+            PREREQS[options.grade_question] = set()
+
+    tracker = Tracker(questions, maxes, PREREQS, options.mute_output)
+    for q in questions:
+        started = tracker.begin_q(q)
+        if not started:
+            continue
+
+        for testq, points, fn in TESTS:
+            if testq != q:
+                continue
+            tracker.begin_test(fn.__name__)
+            try:
+                fn(tracker)
+            except KeyboardInterrupt:
+                tracker.unmute()
+                print("\n\nCaught KeyboardInterrupt: aborting autograder")
+                tracker.finalize()
+                print("\n[autograder was interrupted before finishing]")
+                sys.exit(1)
+            except:
+                tracker.unmute()
+                print(traceback.format_exc())
+            tracker.end_test(points)
+        tracker.end_q()
+    tracker.finalize()
+
+################################################################################
+# Tests begin here
+################################################################################
+
+import numpy as np
+import matplotlib
+import contextlib
+
+import nn
+import backend
+
+def check_dependencies():
+    import matplotlib.pyplot as plt
+    import time
+    fig, ax = plt.subplots(1, 1)
+    ax.set_xlim([-1, 1])
+    ax.set_ylim([-1, 1])
+    line, = ax.plot([], [], color="black")
+    plt.show(block=False)
+
+    for t in range(400):
+        angle = t * 0.05
+        x = np.sin(angle)
+        y = np.cos(angle)
+        line.set_data([x,-x], [y,-y])
+        fig.canvas.draw_idle()
+        fig.canvas.start_event_loop(1e-3)
+
+def disable_graphics():
+    backend.use_graphics = False
+
+@contextlib.contextmanager
+def no_graphics():
+    old_use_graphics = backend.use_graphics
+    backend.use_graphics = False
+    yield
+    backend.use_graphics = old_use_graphics
+
+def verify_node(node, expected_type, expected_shape, method_name):
+    if expected_type == 'parameter':
+        assert node is not None, (
+            "{} should return an instance of nn.Parameter, not None".format(method_name))
+        assert isinstance(node, nn.Parameter), (
+            "{} should return an instance of nn.Parameter, instead got type {!r}".format(
+            method_name, type(node).__name__))
+    elif expected_type == 'loss':
+        assert node is not None, (
+            "{} should return an instance a loss node, not None".format(method_name))
+        assert isinstance(node, (nn.SquareLoss, nn.SoftmaxLoss)), (
+            "{} should return a loss node, instead got type {!r}".format(
+            method_name, type(node).__name__))
+    elif expected_type == 'node':
+        assert node is not None, (
+            "{} should return a node object, not None".format(method_name))
+        assert isinstance(node, nn.Node), (
+            "{} should return a node object, instead got type {!r}".format(
+            method_name, type(node).__name__))
+    else:
+        assert False, "If you see this message, please report a bug in the autograder"
+
+    if expected_type != 'loss':
+        assert all([(expected is '?' or actual == expected) for (actual, expected) in zip(node.data.shape, expected_shape)]), (
+            "{} should return an object with shape {}, got {}".format(
+                method_name, nn.format_shape(expected_shape), nn.format_shape(node.data.shape)))
+
+def trace_node(node_to_trace):
+    """
+    Returns a set containing the node and all ancestors in the computation graph
+    """
+    nodes = set()
+    tape = []
+
+    def visit(node):
+        if node not in nodes:
+            for parent in node.parents:
+                visit(parent)
+            nodes.add(node)
+            tape.append(node)
+
+    visit(node_to_trace)
+
+    return nodes
+
+@test('q1', points=6)
+def check_perceptron(tracker):
+    import models
+
+    print("Sanity checking perceptron...")
+    np_random = np.random.RandomState(0)
+    # Check that the perceptron weights are initialized to a vector with `dimensions` entries.
+    for dimensions in range(1, 10):
+        p = models.PerceptronModel(dimensions)
+        p_weights = p.get_weights()
+        verify_node(p_weights, 'parameter', (1, dimensions), "PerceptronModel.get_weights()")
+
+    # Check that run returns a node, and that the score in the node is correct
+    for dimensions in range(1, 10):
+        p = models.PerceptronModel(dimensions)
+        p_weights = p.get_weights()
+        verify_node(p_weights, 'parameter', (1, dimensions), "PerceptronModel.get_weights()")
+        point = np_random.uniform(-10, 10, (1, dimensions))
+        score = p.run(nn.Constant(point))
+        verify_node(score, 'node', (1, 1), "PerceptronModel.run()")
+        calculated_score = nn.as_scalar(score)
+        expected_score = float(np.dot(point.flatten(), p_weights.data.flatten()))
+        assert np.isclose(calculated_score, expected_score), (
+            "The score computed by PerceptronModel.run() ({:.4f}) does not match the expected score ({:.4f})".format(
+            calculated_score, expected_score))
+
+    # Check that get_prediction returns the correct values, including the
+    # case when a point lies exactly on the decision boundary
+    for dimensions in range(1, 10):
+        p = models.PerceptronModel(dimensions)
+        random_point = np_random.uniform(-10, 10, (1, dimensions))
+        for point in (random_point, np.zeros_like(random_point)):
+            prediction = p.get_prediction(nn.Constant(point))
+            assert prediction == 1 or prediction == -1, (
+                "PerceptronModel.get_prediction() should return 1 or -1, not {}".format(
+                prediction))
+
+            expected_prediction = np.asscalar(np.where(np.dot(point, p.get_weights().data.T) >= 0, 1, -1))
+            assert prediction == expected_prediction, (
+                "PerceptronModel.get_prediction() returned {}; expected {}".format(
+                    prediction, expected_prediction))
+
+    tracker.add_points(2) # Partial credit for passing sanity checks
+
+    print("Sanity checking perceptron weight updates...")
+
+    # Test weight updates. This involves constructing a dataset that
+    # requires 0 or 1 updates before convergence, and testing that weight
+    # values change as expected. Note that (multiplier < -1 or multiplier > 1)
+    # must be true for the testing code to be correct.
+    dimensions = 2
+    for multiplier in (-5, -2, 2, 5):
+        p = models.PerceptronModel(dimensions)
+        orig_weights = p.get_weights().data.reshape((1, dimensions)).copy()
+        if np.abs(orig_weights).sum() == 0.0:
+            # This autograder test doesn't work when weights are exactly zero
+            continue
+        point = multiplier * orig_weights
+        sanity_dataset = backend.Dataset(
+            x=np.tile(point, (500, 1)),
+            y=np.ones((500, 1)) * -1.0
+        )
+        p.train(sanity_dataset)
+        new_weights = p.get_weights().data.reshape((1, dimensions))
+
+        if multiplier < 0:
+            expected_weights = orig_weights
+        else:
+            expected_weights = orig_weights - point
+
+        if not np.all(new_weights == expected_weights):
+            print()
+            print("Initial perceptron weights were: [{:.4f}, {:.4f}]".format(
+                orig_weights[0,0], orig_weights[0,1]))
+            print("All data points in the dataset were identical and had:")
+            print("    x = [{:.4f}, {:.4f}]".format(
+                point[0,0], point[0,1]))
+            print("    y = -1")
+            print("Your trained weights were: [{:.4f}, {:.4f}]".format(
+                new_weights[0,0], new_weights[0,1]))
+            print("Expected weights after training: [{:.4f}, {:.4f}]".format(
+                expected_weights[0,0], expected_weights[0,1]))
+            print()
+            assert False, "Weight update sanity check failed"
+
+    print("Sanity checking complete. Now training perceptron")
+    model = models.PerceptronModel(3)
+    dataset = backend.PerceptronDataset(model)
+
+    model.train(dataset)
+    backend.maybe_sleep_and_close(1)
+
+    assert dataset.epoch != 0, "Perceptron code never iterated over the training data"
+
+    accuracy = np.mean(np.where(np.dot(dataset.x, model.get_weights().data.T) >= 0.0, 1.0, -1.0) == dataset.y)
+    if accuracy < 1.0:
+        print("The weights learned by your perceptron correctly classified {:.2%} of training examples".format(accuracy))
+        print("To receive full points for this question, your perceptron must converge to 100% accuracy")
+        return
+
+    tracker.add_points(4)
+
+@test('q2', points=6)
+def check_regression(tracker):
+    import models
+    model = models.RegressionModel()
+    dataset = backend.RegressionDataset(model)
+
+    detected_parameters = None
+    for batch_size in (1, 2, 4):
+        inp_x = nn.Constant(dataset.x[:batch_size])
+        inp_y = nn.Constant(dataset.y[:batch_size])
+        output_node = model.run(inp_x)
+        verify_node(output_node, 'node', (batch_size, 1), "RegressionModel.run()")
+        trace = trace_node(output_node)
+        assert inp_x in trace, "Node returned from RegressionModel.run() does not depend on the provided input (x)"
+
+        if detected_parameters is None:
+            detected_parameters = [node for node in trace if isinstance(node, nn.Parameter)]
+
+        for node in trace:
+            assert not isinstance(node, nn.Parameter) or node in detected_parameters, (
+                "Calling RegressionModel.run() multiple times should always re-use the same parameters, but a new nn.Parameter object was detected")
+
+    for batch_size in (1, 2, 4):
+        inp_x = nn.Constant(dataset.x[:batch_size])
+        inp_y = nn.Constant(dataset.y[:batch_size])
+        loss_node = model.get_loss(inp_x, inp_y)
+        verify_node(loss_node, 'loss', None, "RegressionModel.get_loss()")
+        trace = trace_node(loss_node)
+        assert inp_x in trace, "Node returned from RegressionModel.get_loss() does not depend on the provided input (x)"
+        assert inp_y in trace, "Node returned from RegressionModel.get_loss() does not depend on the provided labels (y)"
+
+        for node in trace:
+            assert not isinstance(node, nn.Parameter) or node in detected_parameters, (
+                "RegressionModel.get_loss() should not use additional parameters not used by RegressionModel.run()")
+
+    tracker.add_points(2) # Partial credit for passing sanity checks
+
+    model.train(dataset)
+    backend.maybe_sleep_and_close(1)
+
+    train_loss = model.get_loss(nn.Constant(dataset.x), nn.Constant(dataset.y))
+    verify_node(train_loss, 'loss', None, "RegressionModel.get_loss()")
+    train_loss = nn.as_scalar(train_loss)
+
+    # Re-compute the loss ourselves: otherwise get_loss() could be hard-coded
+    # to always return zero
+    train_predicted = model.run(nn.Constant(dataset.x))
+    verify_node(train_predicted, 'node', (dataset.x.shape[0], 1), "RegressionModel.run()")
+    sanity_loss = 0.5 * np.mean((train_predicted.data - dataset.y)**2)
+
+    assert np.isclose(train_loss, sanity_loss), (
+        "RegressionModel.get_loss() returned a loss of {:.4f}, "
+        "but the autograder computed a loss of {:.4f} "
+        "based on the output of RegressionModel.run()".format(
+            train_loss, sanity_loss))
+
+    loss_threshold = 0.02
+    if train_loss <= loss_threshold:
+        print("Your final loss is: {:f}".format(train_loss))
+        tracker.add_points(4)
+    else:
+        print("Your final loss ({:f}) must be no more than {:.4f} to receive full points for this question".format(train_loss, loss_threshold))
+
+@test('q3', points=6)
+def check_digit_classification(tracker):
+    import models
+    model = models.DigitClassificationModel()
+    dataset = backend.DigitClassificationDataset(model)
+
+    detected_parameters = None
+    for batch_size in (1, 2, 4):
+        inp_x = nn.Constant(dataset.x[:batch_size])
+        inp_y = nn.Constant(dataset.y[:batch_size])
+        output_node = model.run(inp_x)
+        verify_node(output_node, 'node', (batch_size, 10), "DigitClassificationModel.run()")
+        trace = trace_node(output_node)
+        assert inp_x in trace, "Node returned from DigitClassificationModel.run() does not depend on the provided input (x)"
+
+        if detected_parameters is None:
+            detected_parameters = [node for node in trace if isinstance(node, nn.Parameter)]
+
+        for node in trace:
+            assert not isinstance(node, nn.Parameter) or node in detected_parameters, (
+                "Calling DigitClassificationModel.run() multiple times should always re-use the same parameters, but a new nn.Parameter object was detected")
+
+    for batch_size in (1, 2, 4):
+        inp_x = nn.Constant(dataset.x[:batch_size])
+        inp_y = nn.Constant(dataset.y[:batch_size])
+        loss_node = model.get_loss(inp_x, inp_y)
+        verify_node(loss_node, 'loss', None, "DigitClassificationModel.get_loss()")
+        trace = trace_node(loss_node)
+        assert inp_x in trace, "Node returned from DigitClassificationModel.get_loss() does not depend on the provided input (x)"
+        assert inp_y in trace, "Node returned from DigitClassificationModel.get_loss() does not depend on the provided labels (y)"
+
+        for node in trace:
+            assert not isinstance(node, nn.Parameter) or node in detected_parameters, (
+                "DigitClassificationModel.get_loss() should not use additional parameters not used by DigitClassificationModel.run()")
+
+    tracker.add_points(2) # Partial credit for passing sanity checks
+
+    model.train(dataset)
+
+    test_logits = model.run(nn.Constant(dataset.test_images)).data
+    test_predicted = np.argmax(test_logits, axis=1)
+    test_accuracy = np.mean(test_predicted == dataset.test_labels)
+
+    accuracy_threshold = 0.97
+    if test_accuracy >= accuracy_threshold:
+        print("Your final test set accuracy is: {:%}".format(test_accuracy))
+        tracker.add_points(4)
+    else:
+        print("Your final test set accuracy ({:%}) must be at least {:.0%} to receive full points for this question".format(test_accuracy, accuracy_threshold))
+
+@test('q4', points=7)
+def check_lang_id(tracker):
+    import models
+    model = models.LanguageIDModel()
+    dataset = backend.LanguageIDDataset(model)
+
+    detected_parameters = None
+    for batch_size, word_length in ((1, 1), (2, 1), (2, 6), (4, 8)):
+        start = dataset.dev_buckets[-1, 0]
+        end = start + batch_size
+        inp_xs, inp_y = dataset._encode(dataset.dev_x[start:end], dataset.dev_y[start:end])
+        inp_xs = inp_xs[:word_length]
+
+        output_node = model.run(inp_xs)
+        verify_node(output_node, 'node', (batch_size, len(dataset.language_names)), "LanguageIDModel.run()")
+        trace = trace_node(output_node)
+        for inp_x in inp_xs:
+            assert inp_x in trace, "Node returned from LanguageIDModel.run() does not depend on all of the provided inputs (xs)"
+
+        # Word length 1 does not use parameters related to transferring the
+        # hidden state across timesteps, so initial parameter detection is only
+        # run for longer words
+        if word_length > 1:
+            if detected_parameters is None:
+                detected_parameters = [node for node in trace if isinstance(node, nn.Parameter)]
+
+            for node in trace:
+                assert not isinstance(node, nn.Parameter) or node in detected_parameters, (
+                    "Calling LanguageIDModel.run() multiple times should always re-use the same parameters, but a new nn.Parameter object was detected")
+
+    for batch_size, word_length in ((1, 1), (2, 1), (2, 6), (4, 8)):
+        start = dataset.dev_buckets[-1, 0]
+        end = start + batch_size
+        inp_xs, inp_y = dataset._encode(dataset.dev_x[start:end], dataset.dev_y[start:end])
+        inp_xs = inp_xs[:word_length]
+        loss_node = model.get_loss(inp_xs, inp_y)
+        trace = trace_node(loss_node)
+        for inp_x in inp_xs:
+            assert inp_x in trace, "Node returned from LanguageIDModel.run() does not depend on all of the provided inputs (xs)"
+        assert inp_y in trace, "Node returned from LanguageIDModel.get_loss() does not depend on the provided labels (y)"
+
+        for node in trace:
+            assert not isinstance(node, nn.Parameter) or node in detected_parameters, (
+                "LanguageIDModel.get_loss() should not use additional parameters not used by LanguageIDModel.run()")
+
+    tracker.add_points(2) # Partial credit for passing sanity checks
+
+    model.train(dataset)
+
+    test_predicted_probs, test_predicted, test_correct = dataset._predict('test')
+    test_accuracy = np.mean(test_predicted == test_correct)
+    accuracy_threshold = 0.81
+    if test_accuracy >= accuracy_threshold:
+        print("Your final test set accuracy is: {:%}".format(test_accuracy))
+        tracker.add_points(5)
+    else:
+        print("Your final test set accuracy ({:%}) must be at least {:.0%} to receive full points for this question".format(test_accuracy, accuracy_threshold))
+
+if __name__ == '__main__':
+    main()

backend.py

@@ -0,0 +1,447 @@
+import collections
+import os
+import time
+import os
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+import nn
+
+use_graphics = True
+
+def maybe_sleep_and_close(seconds):
+    if use_graphics and plt.get_fignums():
+        time.sleep(seconds)
+        for fignum in plt.get_fignums():
+            fig = plt.figure(fignum)
+            plt.close(fig)
+            try:
+                # This raises a TclError on some Windows machines
+                fig.canvas.start_event_loop(1e-3)
+            except:
+                pass
+
+def get_data_path(filename):
+    path = os.path.join(
+        os.path.dirname(__file__), os.pardir, "data", filename)
+    if not os.path.exists(path):
+        path = os.path.join(
+            os.path.dirname(__file__), "data", filename)
+    if not os.path.exists(path):
+        path = os.path.join(
+            os.path.dirname(__file__), filename)
+    if not os.path.exists(path):
+        raise Exception("Could not find data file: {}".format(filename))
+    return path
+
+class Dataset(object):
+    def __init__(self, x, y):
+        assert isinstance(x, np.ndarray)
+        assert isinstance(y, np.ndarray)
+        assert np.issubdtype(x.dtype, np.floating)
+        assert np.issubdtype(y.dtype, np.floating)
+        assert x.ndim == 2
+        assert y.ndim == 2
+        assert x.shape[0] == y.shape[0]
+        self.x = x
+        self.y = y
+
+    def iterate_once(self, batch_size):
+        assert isinstance(batch_size, int) and batch_size > 0, (
+            "Batch size should be a positive integer, got {!r}".format(
+                batch_size))
+        assert self.x.shape[0] % batch_size == 0, (
+            "Dataset size {:d} is not divisible by batch size {:d}".format(
+                self.x.shape[0], batch_size))
+        index = 0
+        while index < self.x.shape[0]:
+            x = self.x[index:index + batch_size]
+            y = self.y[index:index + batch_size]
+            yield nn.Constant(x), nn.Constant(y)
+            index += batch_size
+
+    def iterate_forever(self, batch_size):
+        while True:
+            yield from self.iterate_once(batch_size)
+
+    def get_validation_accuracy(self):
+        raise NotImplementedError(
+            "No validation data is available for this dataset. "
+            "In this assignment, only the Digit Classification and Language "
+            "Identification datasets have validation data.")
+
+class PerceptronDataset(Dataset):
+    def __init__(self, model):
+        points = 500
+        x = np.hstack([np.random.randn(points, 2), np.ones((points, 1))])
+        y = np.where(x[:, 0] + 2 * x[:, 1] - 1 >= 0, 1.0, -1.0)
+        super().__init__(x, np.expand_dims(y, axis=1))
+
+        self.model = model
+        self.epoch = 0
+
+        if use_graphics:
+            fig, ax = plt.subplots(1, 1)
+            limits = np.array([-3.0, 3.0])
+            ax.set_xlim(limits)
+            ax.set_ylim(limits)
+            positive = ax.scatter(*x[y == 1, :-1].T, color="red", marker="+")
+            negative = ax.scatter(*x[y == -1, :-1].T, color="blue", marker="_")
+            line, = ax.plot([], [], color="black")
+            text = ax.text(0.03, 0.97, "", transform=ax.transAxes, va="top")
+            ax.legend([positive, negative], [1, -1])
+            plt.show(block=False)
+
+            self.fig = fig
+            self.limits = limits
+            self.line = line
+            self.text = text
+            self.last_update = time.time()
+
+    def iterate_once(self, batch_size):
+        self.epoch += 1
+
+        for i, (x, y) in enumerate(super().iterate_once(batch_size)):
+            yield x, y
+
+            if use_graphics and time.time() - self.last_update > 0.01:
+                w = self.model.get_weights().data.flatten()
+                limits = self.limits
+                if w[1] != 0:
+                    self.line.set_data(limits, (-w[0] * limits - w[2]) / w[1])
+                elif w[0] != 0:
+                    self.line.set_data(np.full(2, -w[2] / w[0]), limits)
+                else:
+                    self.line.set_data([], [])
+                self.text.set_text(
+                    "epoch: {:,}\npoint: {:,}/{:,}\nweights: {}".format(
+                        self.epoch, i * batch_size + 1, len(self.x), w))
+                self.fig.canvas.draw_idle()
+                self.fig.canvas.start_event_loop(1e-3)
+                self.last_update = time.time()
+
+class RegressionDataset(Dataset):
+    def __init__(self, model):
+        x = np.expand_dims(np.linspace(-2 * np.pi, 2 * np.pi, num=200), axis=1)
+        np.random.RandomState(0).shuffle(x)
+        self.argsort_x = np.argsort(x.flatten())
+        y = np.sin(x)
+        super().__init__(x, y)
+
+        self.model = model
+        self.processed = 0
+
+        if use_graphics:
+            fig, ax = plt.subplots(1, 1)
+            ax.set_xlim(-2 * np.pi, 2 * np.pi)
+            ax.set_ylim(-1.4, 1.4)
+            real, = ax.plot(x[self.argsort_x], y[self.argsort_x], color="blue")
+            learned, = ax.plot([], [], color="red")
+            text = ax.text(0.03, 0.97, "", transform=ax.transAxes, va="top")
+            ax.legend([real, learned], ["real", "learned"])
+            plt.show(block=False)
+
+            self.fig = fig
+            self.learned = learned
+            self.text = text
+            self.last_update = time.time()
+
+    def iterate_once(self, batch_size):
+        for x, y in super().iterate_once(batch_size):
+            yield x, y
+            self.processed += batch_size
+
+            if use_graphics and time.time() - self.last_update > 0.1:
+                predicted = self.model.run(nn.Constant(self.x)).data
+                loss = self.model.get_loss(
+                    nn.Constant(self.x), nn.Constant(self.y)).data
+                self.learned.set_data(self.x[self.argsort_x], predicted[self.argsort_x])
+                self.text.set_text("processed: {:,}\nloss: {:.6f}".format(
+                   self.processed, loss))
+                self.fig.canvas.draw_idle()
+                self.fig.canvas.start_event_loop(1e-3)
+                self.last_update = time.time()
+
+class DigitClassificationDataset(Dataset):
+    def __init__(self, model):
+        mnist_path = get_data_path("mnist.npz")
+
+        with np.load(mnist_path) as data:
+            train_images = data["train_images"]
+            train_labels = data["train_labels"]
+            test_images = data["test_images"]
+            test_labels = data["test_labels"]
+            assert len(train_images) == len(train_labels) == 60000
+            assert len(test_images) == len(test_labels) == 10000
+            self.dev_images = test_images[0::2]
+            self.dev_labels = test_labels[0::2]
+            self.test_images = test_images[1::2]
+            self.test_labels = test_labels[1::2]
+
+        train_labels_one_hot = np.zeros((len(train_images), 10))
+        train_labels_one_hot[range(len(train_images)), train_labels] = 1
+
+        super().__init__(train_images, train_labels_one_hot)
+
+        self.model = model
+        self.epoch = 0
+
+        if use_graphics:
+            width = 20  # Width of each row expressed as a multiple of image width
+            samples = 100  # Number of images to display per label
+            fig = plt.figure()
+            ax = {}
+            images = collections.defaultdict(list)
+            texts = collections.defaultdict(list)
+            for i in reversed(range(10)):
+                ax[i] = plt.subplot2grid((30, 1), (3 * i, 0), 2, 1,
+                                         sharex=ax.get(9))
+                plt.setp(ax[i].get_xticklabels(), visible=i == 9)
+                ax[i].set_yticks([])
+                ax[i].text(-0.03, 0.5, i, transform=ax[i].transAxes,
+                           va="center")
+                ax[i].set_xlim(0, 28 * width)
+                ax[i].set_ylim(0, 28)
+                for j in range(samples):
+                    images[i].append(ax[i].imshow(
+                        np.zeros((28, 28)), vmin=0, vmax=1, cmap="Greens",
+                        alpha=0.3))
+                    texts[i].append(ax[i].text(
+                        0, 0, "", ha="center", va="top", fontsize="smaller"))
+            ax[9].set_xticks(np.linspace(0, 28 * width, 11))
+            ax[9].set_xticklabels(
+                ["{:.1f}".format(num) for num in np.linspace(0, 1, 11)])
+            ax[9].tick_params(axis="x", pad=16)
+            ax[9].set_xlabel("Probability of Correct Label")
+            status = ax[0].text(
+                0.5, 1.5, "", transform=ax[0].transAxes, ha="center",
+                va="bottom")
+            plt.show(block=False)
+
+            self.width = width
+            self.samples = samples
+            self.fig = fig
+            self.images = images
+            self.texts = texts
+            self.status = status
+            self.last_update = time.time()
+
+    def iterate_once(self, batch_size):
+        self.epoch += 1
+
+        for i, (x, y) in enumerate(super().iterate_once(batch_size)):
+            yield x, y
+
+            if use_graphics and time.time() - self.last_update > 1:
+                dev_logits = self.model.run(nn.Constant(self.dev_images)).data
+                dev_predicted = np.argmax(dev_logits, axis=1)
+                dev_probs = np.exp(nn.SoftmaxLoss.log_softmax(dev_logits))
+                dev_accuracy = np.mean(dev_predicted == self.dev_labels)
+
+                self.status.set_text(
+                    "epoch: {:d}, batch: {:d}/{:d}, validation accuracy: "
+                    "{:.2%}".format(
+                        self.epoch, i, len(self.x) // batch_size, dev_accuracy))
+                for i in range(10):
+                    predicted = dev_predicted[self.dev_labels == i]
+                    probs = dev_probs[self.dev_labels == i][:, i]
+                    linspace = np.linspace(
+                        0, len(probs) - 1, self.samples).astype(int)
+                    indices = probs.argsort()[linspace]
+                    for j, (prob, image) in enumerate(zip(
+                            probs[indices],
+                            self.dev_images[self.dev_labels == i][indices])):
+                        self.images[i][j].set_data(image.reshape((28, 28)))
+                        left = prob * (self.width - 1) * 28
+                        if predicted[indices[j]] == i:
+                            self.images[i][j].set_cmap("Greens")
+                            self.texts[i][j].set_text("")
+                        else:
+                            self.images[i][j].set_cmap("Reds")
+                            self.texts[i][j].set_text(predicted[indices[j]])
+                            self.texts[i][j].set_x(left + 14)
+                        self.images[i][j].set_extent([left, left + 28, 0, 28])
+                self.fig.canvas.draw_idle()
+                self.fig.canvas.start_event_loop(1e-3)
+                self.last_update = time.time()
+
+    def get_validation_accuracy(self):
+        dev_logits = self.model.run(nn.Constant(self.dev_images)).data
+        dev_predicted = np.argmax(dev_logits, axis=1)
+        dev_accuracy = np.mean(dev_predicted == self.dev_labels)
+        return dev_accuracy
+
+class LanguageIDDataset(Dataset):
+    def __init__(self, model):
+        self.model = model
+
+        data_path = get_data_path("lang_id.npz")
+
+        with np.load(data_path) as data:
+            self.chars = data['chars']
+            self.language_codes = data['language_codes']
+            self.language_names = data['language_names']
+
+            self.train_x = data['train_x']
+            self.train_y = data['train_y']
+            self.train_buckets = data['train_buckets']
+            self.dev_x = data['dev_x']
+            self.dev_y = data['dev_y']
+            self.dev_buckets = data['dev_buckets']
+            self.test_x = data['test_x']
+            self.test_y = data['test_y']
+            self.test_buckets = data['test_buckets']
+
+        self.epoch = 0
+        self.bucket_weights = self.train_buckets[:,1] - self.train_buckets[:,0]
+        self.bucket_weights = self.bucket_weights / float(self.bucket_weights.sum())
+
+        self.chars_print = self.chars
+        try:
+            print(u"Alphabet: {}".format(u"".join(self.chars)))
+        except UnicodeEncodeError:
+            self.chars_print = "abcdefghijklmnopqrstuvwxyzaaeeeeiinoouuacelnszz"
+            print("Alphabet: " + self.chars_print)
+            self.chars_print = list(self.chars_print)
+            print("""
+NOTE: Your terminal does not appear to support printing Unicode characters.
+For the purposes of printing to the terminal, some of the letters in the
+alphabet above have been substituted with ASCII symbols.""".strip())
+        print("")
+
+        # Select some examples to spotlight in the monitoring phase (3 per language)
+        spotlight_idxs = []
+        for i in range(len(self.language_names)):
+            idxs_lang_i = np.nonzero(self.dev_y == i)[0]
+            idxs_lang_i = np.random.choice(idxs_lang_i, size=3, replace=False)
+            spotlight_idxs.extend(list(idxs_lang_i))
+        self.spotlight_idxs = np.array(spotlight_idxs, dtype=int)
+
+        # Templates for printing updates as training progresses
+        max_word_len = self.dev_x.shape[1]
+        max_lang_len = max([len(x) for x in self.language_names])
+
+        self.predicted_template = u"Pred: {:<NUM}".replace('NUM',
+            str(max_lang_len))
+
+        self.word_template = u"  "
+        self.word_template += u"{:<NUM} ".replace('NUM', str(max_word_len))
+        self.word_template += u"{:<NUM} ({:6.1%})".replace('NUM', str(max_lang_len))
+        self.word_template += u" {:<NUM} ".replace('NUM',
+            str(max_lang_len + len('Pred: ')))
+        for i in range(len(self.language_names)):
+            self.word_template += u"|{}".format(self.language_codes[i])
+            self.word_template += "{probs[" + str(i) + "]:4.0%}"
+
+        self.last_update = time.time()
+
+    def _encode(self, inp_x, inp_y):
+        xs = []
+        for i in range(inp_x.shape[1]):
+            if np.all(inp_x[:,i] == -1):
+                break
+            assert not np.any(inp_x[:,i] == -1), (
+                "Please report this error in the project: batching by length was done incorrectly in the provided code")
+            x = np.eye(len(self.chars))[inp_x[:,i]]
+            xs.append(nn.Constant(x))
+        y = np.eye(len(self.language_names))[inp_y]
+        y = nn.Constant(y)
+        return xs, y
+
+    def _softmax(self, x):
+        exp = np.exp(x - np.max(x, axis=-1, keepdims=True))
+        return exp / np.sum(exp, axis=-1, keepdims=True)
+
+    def _predict(self, split='dev'):
+        if split == 'dev':
+            data_x = self.dev_x
+            data_y = self.dev_y
+            buckets = self.dev_buckets
+        else:
+            data_x = self.test_x
+            data_y = self.test_y
+            buckets = self.test_buckets
+
+        all_predicted = []
+        all_correct = []
+        for bucket_id in range(buckets.shape[0]):
+            start, end = buckets[bucket_id]
+            xs, y = self._encode(data_x[start:end], data_y[start:end])
+            predicted = self.model.run(xs)
+
+            all_predicted.extend(list(predicted.data))
+            all_correct.extend(list(data_y[start:end]))
+
+        all_predicted_probs = self._softmax(np.asarray(all_predicted))
+        all_predicted = np.asarray(all_predicted).argmax(axis=-1)
+        all_correct = np.asarray(all_correct)
+
+        return all_predicted_probs, all_predicted, all_correct
+
+    def iterate_once(self, batch_size):
+        assert isinstance(batch_size, int) and batch_size > 0, (
+            "Batch size should be a positive integer, got {!r}".format(
+                batch_size))
+        assert self.train_x.shape[0] >= batch_size, (
+            "Dataset size {:d} is smaller than the batch size {:d}".format(
+                self.train_x.shape[0], batch_size))
+
+        self.epoch += 1
+
+        for iteration in range(self.train_x.shape[0] // batch_size):
+            bucket_id = np.random.choice(self.bucket_weights.shape[0], p=self.bucket_weights)
+            example_ids = self.train_buckets[bucket_id, 0] + np.random.choice(
+                self.train_buckets[bucket_id, 1] - self.train_buckets[bucket_id, 0],
+                size=batch_size)
+
+            yield self._encode(self.train_x[example_ids], self.train_y[example_ids])
+
+            if use_graphics and time.time() - self.last_update > 0.5:
+                dev_predicted_probs, dev_predicted, dev_correct = self._predict()
+                dev_accuracy = np.mean(dev_predicted == dev_correct)
+
+                print("epoch {:,} iteration {:,} validation-accuracy {:.1%}".format(
+                    self.epoch, iteration, dev_accuracy))
+
+                for idx in self.spotlight_idxs:
+                    correct = (dev_predicted[idx] == dev_correct[idx])
+                    word = u"".join([self.chars_print[ch] for ch in self.dev_x[idx] if ch != -1])
+
+                    print(self.word_template.format(
+                        word,
+                        self.language_names[dev_correct[idx]],
+                        dev_predicted_probs[idx, dev_correct[idx]],
+                        "" if correct else self.predicted_template.format(
+                            self.language_names[dev_predicted[idx]]),
+                        probs=dev_predicted_probs[idx,:],
+                    ))
+
+                self.last_update = time.time()
+
+    def get_validation_accuracy(self):
+        dev_predicted_probs, dev_predicted, dev_correct = self._predict()
+        dev_accuracy = np.mean(dev_predicted == dev_correct)
+        return dev_accuracy
+
+
+def main():
+    import models
+    model = models.PerceptronModel(3)
+    dataset = PerceptronDataset(model)
+    model.train(dataset)
+
+    model = models.RegressionModel()
+    dataset = RegressionDataset(model)
+    model.train(dataset)
+
+    model = models.DigitClassificationModel()
+    dataset = DigitClassificationDataset(model)
+    model.train(dataset)
+
+    model = models.LanguageIDModel()
+    dataset = LanguageIDDataset(model)
+    model.train(dataset)
+
+if __name__ == "__main__":
+    main()

models.py

@@ -0,0 +1,282 @@
+import nn
+import backend
+import numpy as np
+
+
+class PerceptronModel(object):
+    def __init__(self, dimensions):
+        """
+        Initialize a new Perceptron instance.
+
+        A perceptron classifies data points as either belonging to a particular
+        class (+1) or not (-1). `dimensions` is the dimensionality of the data.
+        For example, dimensions=2 would mean that the perceptron must classify
+        2D points.
+        """
+        self.w = nn.Parameter(1, dimensions)
+
+    def get_weights(self):
+        """
+        Return a Parameter instance with the current weights of the perceptron.
+        """
+        return self.w
+
+    def run(self, x):
+        """
+        Calculates the score assigned by the perceptron to a data point x.
+
+        Inputs:
+            x: a node with shape (1 x dimensions)
+        Returns: a node containing a single number (the score)
+        """
+        "*** YOUR CODE HERE ***"
+
+    def get_prediction(self, x):
+        """
+        Calculates the predicted class for a single data point `x`.
+
+        Returns: 1 or -1
+        """
+        "*** YOUR CODE HERE ***"
+
+    def train(self, dataset):
+        """
+        Train the perceptron until convergence.
+        """
+        "*** YOUR CODE HERE ***"
+
+
+class RegressionModel(object):
+    """
+    A neural network model for approximating a function that maps from real
+    numbers to real numbers. The network should be sufficiently large to be able
+    to approximate sin(x) on the interval [-2pi, 2pi] to reasonable precision.
+    """
+
+    def __init__(self):
+        # Initialize your model parameters here
+        "*** YOUR CODE HERE ***"
+        model = object.__init__(self)
+        self.get_data_and_monitor = backend.RegressionDataset(model)
+
+        # Remember to set self.learning_rate!
+        # You may use any learning rate that works well for your architecture
+        "*** YOUR CODE HERE ***"
+        self.learning_rate = 0.1
+        self.hidden_size = 300
+
+        self.w1 = nn.Parameter(1, self.hidden_size)
+        self.w2 = nn.Parameter(self.hidden_size, self.hidden_size)
+        self.w3 = nn.Parameter(self.hidden_size, 1)
+        self.b1 = nn.Parameter(self.hidden_size)
+        self.b2 = nn.Parameter(self.hidden_size)
+        self.b3 = nn.Parameter(1)
+
+    def run(self, x):
+        """
+        Runs the model for a batch of examples.
+
+        Inputs:
+            x: a node with shape (batch_size x 1)
+        Returns:
+            A node with shape (batch_size x 1) containing predicted y-values
+        """
+        "*** YOUR CODE HERE ***"
+        self.graph = nn.Graph(
+            [self.w1, self.w2, self.w3, self.b1, self.b2, self.b3])
+
+        if y is not None:
+            # At training time, the correct output `y` is known.
+            # Here, you should construct a loss node, and return the nn.Graph
+            # that the node belongs to. The loss node must be the last node
+            # added to the graph.
+            "*** YOUR CODE HERE ***"
+            input_x = nn.Input(self.graph, x)
+            input_y = nn.Input(self.graph, y)
+            xw1 = nn.MatrixMultiply(self.graph, input_x, self.w1)
+            xw1_plus_b1 = nn.MatrixVectorAdd(self.graph, xw1, self.b1)
+            l1 = nn.ReLU(self.graph, xw1_plus_b1)
+            l1w2 = nn.MatrixMultiply(self.graph, l1, self.w2)
+            l2w2_plus_b2 = nn.MatrixVectorAdd(self.graph, l1w2, self.b2)
+            l2 = nn.ReLU(self.graph, l2w2_plus_b2)
+            l2w3 = nn.MatrixMultiply(self.graph, l2, self.w3)
+            l2w3_plus_b3 = nn.MatrixVectorAdd(self.graph, l2w3, self.b3)
+            loss = nn.SquareLoss(self.graph, l2w3_plus_b3, input_y)
+
+            return self.graph
+
+        else:
+            # At test time, the correct output is unknown.
+            # You should instead return your model's prediction as a numpy array
+            "*** YOUR CODE HERE ***"
+            input_x = nn.Input(self.graph, x)
+
+            xw1 = nn.MatrixMultiply(self.graph, input_x, self.w1)
+            xw1_plus_b1 = nn.MatrixVectorAdd(self.graph, xw1, self.b1)
+            l1 = nn.ReLU(self.graph, xw1_plus_b1)
+            l1w2 = nn.MatrixMultiply(self.graph, l1, self.w2)
+            l2w2_plus_b2 = nn.MatrixVectorAdd(self.graph, l1w2, self.b2)
+            l2 = nn.ReLU(self.graph, l2w2_plus_b2)
+            l2w3 = nn.MatrixMultiply(self.graph, l2, self.w3)
+            l2w3_plus_b3 = nn.MatrixVectorAdd(self.graph, l2w3, self.b3)
+
+            return self.graph.get_output(l2w3_plus_b3)
+
+    def get_loss(self, x, y):
+        """
+        Computes the loss for a batch of examples.
+
+        Inputs:
+            x: a node with shape (batch_size x 1)
+            y: a node with shape (batch_size x 1), containing the true y-values
+                to be used for training
+        Returns: a loss node
+        """
+        "*** YOUR CODE HERE ***"
+
+    def train(self, dataset):
+        """
+        Trains the model.
+        """
+        "*** YOUR CODE HERE ***"
+
+
+class DigitClassificationModel(object):
+    """
+    A model for handwritten digit classification using the MNIST dataset.
+
+    Each handwritten digit is a 28x28 pixel grayscale image, which is flattened
+    into a 784-dimensional vector for the purposes of this model. Each entry in
+    the vector is a floating point number between 0 and 1.
+
+    The goal is to sort each digit into one of 10 classes (number 0 through 9).
+
+    (See RegressionModel for more information about the APIs of different
+    methods here. We recommend that you implement the RegressionModel before
+    working on this part of the project.)
+    """
+
+    def __init__(self):
+        # Initialize your model parameters here
+        "*** YOUR CODE HERE ***"
+        object.__init__(self)
+        self.get_data_and_monitor = backend.DigitClassificationDataset
+        self.learning_rate = 0.15
+        self.hidden_size = 300
+        self.w1 = nn.Parameter(784, self.hidden_size)
+        self.w2 = nn.Parameter(self.hidden_size, self.hidden_size)
+        self.w3 = nn.Parameter(self.hidden_size, 10)
+        self.b1 = nn.Parameter(1, self.hidden_size)
+        self.b2 = nn.Parameter(1, self.hidden_size)
+        self.b3 = nn.Parameter(1, 10)
+
+    def run(self, x):
+        """
+        Runs the model for a batch of examples.
+
+        Your model should predict a node with shape (batch_size x 10),
+        containing scores. Higher scores correspond to greater probability of
+        the image belonging to a particular class.
+
+        Inputs:
+            x: a node with shape (batch_size x 784)
+        Output:
+            A node with shape (batch_size x 10) containing predicted scores
+                (also called logits)
+        """
+        "*** YOUR CODE HERE ***"
+
+    def get_loss(self, x, y):
+        """
+        Computes the loss for a batch of examples.
+
+        The correct labels `y` are represented as a node with shape
+        (batch_size x 10). Each row is a one-hot vector encoding the correct
+        digit class (0-9).
+
+        Inputs:
+            x: a node with shape (batch_size x 784)
+            y: a node with shape (batch_size x 10)
+        Returns: a loss node
+        """
+        "*** YOUR CODE HERE ***"
+
+    def train(self, dataset):
+        """
+        Trains the model.
+        """
+        "*** YOUR CODE HERE ***"
+
+
+class LanguageIDModel(object):
+    """
+    A model for language identification at a single-word granularity.
+
+    (See RegressionModel for more information about the APIs of different
+    methods here. We recommend that you implement the RegressionModel before
+    working on this part of the project.)
+    """
+
+    def __init__(self):
+        # Our dataset contains words from five different languages, and the
+        # combined alphabets of the five languages contain a total of 47 unique
+        # characters.
+        # You can refer to self.num_chars or len(self.languages) in your code
+        self.num_chars = 47
+        self.languages = ["English", "Spanish", "Finnish", "Dutch", "Polish"]
+
+        # Initialize your model parameters here
+        "*** YOUR CODE HERE ***"
+
+    def run(self, xs):
+        """
+        Runs the model for a batch of examples.
+
+        Although words have different lengths, our data processing guarantees
+        that within a single batch, all words will be of the same length (L).
+
+        Here `xs` will be a list of length L. Each element of `xs` will be a
+        node with shape (batch_size x self.num_chars), where every row in the
+        array is a one-hot vector encoding of a character. For example, if we
+        have a batch of 8 three-letter words where the last word is "cat", then
+        xs[1] will be a node that contains a 1 at position (7, 0). Here the
+        index 7 reflects the fact that "cat" is the last word in the batch, and
+        the index 0 reflects the fact that the letter "a" is the inital (0th)
+        letter of our combined alphabet for this task.
+
+        Your model should use a Recurrent Neural Network to summarize the list
+        `xs` into a single node of shape (batch_size x hidden_size), for your
+        choice of hidden_size. It should then calculate a node of shape
+        (batch_size x 5) containing scores, where higher scores correspond to
+        greater probability of the word originating from a particular language.
+
+        Inputs:
+            xs: a list with L elements (one per character), where each element
+                is a node with shape (batch_size x self.num_chars)
+        Returns:
+            A node with shape (batch_size x 5) containing predicted scores
+                (also called logits)
+        """
+        "*** YOUR CODE HERE ***"
+
+    def get_loss(self, xs, y):
+        """
+        Computes the loss for a batch of examples.
+
+        The correct labels `y` are represented as a node with shape
+        (batch_size x 5). Each row is a one-hot vector encoding the correct
+        language.
+
+        Inputs:
+            xs: a list with L elements (one per character), where each element
+                is a node with shape (batch_size x self.num_chars)
+            y: a node with shape (batch_size x 5)
+        Returns: a loss node
+        """
+        "*** YOUR CODE HERE ***"
+
+    def train(self, dataset):
+        """
+        Trains the model.
+        """
+        "*** YOUR CODE HERE ***"

nn.py

@@ -0,0 +1,392 @@
+import numpy as np
+
+def format_shape(shape):
+    return "x".join(map(str, shape)) if shape else "()"
+
+class Node(object):
+    def __repr__(self):
+        return "<{} shape={} at {}>".format(
+            type(self).__name__, format_shape(self.data.shape), hex(id(self)))
+
+class DataNode(Node):
+    """
+    DataNode is the parent class for Parameter and Constant nodes.
+
+    You should not need to use this class directly.
+    """
+    def __init__(self, data):
+        self.parents = []
+        self.data = data
+
+    def _forward(self, *inputs):
+        return self.data
+
+    @staticmethod
+    def _backward(gradient, *inputs):
+        return []
+
+class Parameter(DataNode):
+    """
+    A Parameter node stores parameters used in a neural network (or perceptron).
+
+    Use the the `update` method to update parameters when training the
+    perceptron or neural network.
+    """
+    def __init__(self, *shape):
+        assert len(shape) == 2, (
+            "Shape must have 2 dimensions, instead has {}".format(len(shape)))
+        assert all(isinstance(dim, int) and dim > 0 for dim in shape), (
+            "Shape must consist of positive integers, got {!r}".format(shape))
+        limit = np.sqrt(3.0 / np.mean(shape))
+        data = np.random.uniform(low=-limit, high=limit, size=shape)
+        super().__init__(data)
+
+    def update(self, direction, multiplier):
+        assert isinstance(direction, Constant), (
+            "Update direction must be a {} node, instead has type {!r}".format(
+                Constant.__name__, type(direction).__name__))
+        assert direction.data.shape == self.data.shape, (
+            "Update direction shape {} does not match parameter shape "
+            "{}".format(
+                format_shape(direction.data.shape),
+                format_shape(self.data.shape)))
+        assert isinstance(multiplier, (int, float)), (
+            "Multiplier must be a Python scalar, instead has type {!r}".format(
+                type(multiplier).__name__))
+        self.data += multiplier * direction.data
+        assert np.all(np.isfinite(self.data)), (
+            "Parameter contains NaN or infinity after update, cannot continue")
+
+class Constant(DataNode):
+    """
+    A Constant node is used to represent:
+    * Input features
+    * Output labels
+    * Gradients computed by back-propagation
+
+    You should not need to construct any Constant nodes directly; they will
+    instead be provided by either the dataset or when you call `nn.gradients`.
+    """
+    def __init__(self, data):
+        assert isinstance(data, np.ndarray), (
+            "Data should be a numpy array, instead has type {!r}".format(
+                type(data).__name__))
+        assert np.issubdtype(data.dtype, np.floating), (
+            "Data should be a float array, instead has data type {!r}".format(
+                data.dtype))
+        super().__init__(data)
+
+class FunctionNode(Node):
+    """
+    A FunctionNode represents a value that is computed based on other nodes.
+    The FunctionNode class performs necessary book-keeping to compute gradients.
+    """
+    def __init__(self, *parents):
+        assert all(isinstance(parent, Node) for parent in parents), (
+            "Inputs must be node objects, instead got types {!r}".format(
+                tuple(type(parent).__name__ for parent in parents)))
+        self.parents = parents
+        self.data = self._forward(*(parent.data for parent in parents))
+
+class Add(FunctionNode):
+    """
+    Adds matrices element-wise.
+
+    Usage: nn.Add(x, y)
+    Inputs:
+        x: a Node with shape (batch_size x num_features)
+        y: a Node with the same shape as x
+    Output:
+        a Node with shape (batch_size x num_features)
+    """
+    @staticmethod
+    def _forward(*inputs):
+        assert len(inputs) == 2, "Expected 2 inputs, got {}".format(len(inputs))
+        assert inputs[0].ndim == 2, (
+            "First input should have 2 dimensions, instead has {}".format(
+                inputs[0].ndim))
+        assert inputs[1].ndim == 2, (
+            "Second input should have 2 dimensions, instead has {}".format(
+                inputs[1].ndim))
+        assert inputs[0].shape == inputs[1].shape, (
+            "Input shapes should match, instead got {} and {}".format(
+                format_shape(inputs[0].shape), format_shape(inputs[1].shape)))
+        return inputs[0] + inputs[1]
+
+    @staticmethod
+    def _backward(gradient, *inputs):
+        assert gradient.shape == inputs[0].shape
+        return [gradient, gradient]
+
+class AddBias(FunctionNode):
+    """
+    Adds a bias vector to each feature vector
+
+    Usage: nn.AddBias(features, bias)
+    Inputs:
+        features: a Node with shape (batch_size x num_features)
+        bias: a Node with shape (1 x num_features)
+    Output:
+        a Node with shape (batch_size x num_features)
+    """
+    @staticmethod
+    def _forward(*inputs):
+        assert len(inputs) == 2, "Expected 2 inputs, got {}".format(len(inputs))
+        assert inputs[0].ndim == 2, (
+            "First input should have 2 dimensions, instead has {}".format(
+                inputs[0].ndim))
+        assert inputs[1].ndim == 2, (
+            "Second input should have 2 dimensions, instead has {}".format(
+                inputs[1].ndim))
+        assert inputs[1].shape[0] == 1, (
+            "First dimension of second input should be 1, instead got shape "
+            "{}".format(format_shape(inputs[1].shape)))
+        assert inputs[0].shape[1] == inputs[1].shape[1], (
+            "Second dimension of inputs should match, instead got shapes {} "
+            "and {}".format(
+                format_shape(inputs[0].shape), format_shape(inputs[1].shape)))
+        return inputs[0] + inputs[1]
+
+    @staticmethod
+    def _backward(gradient, *inputs):
+        assert gradient.shape == inputs[0].shape
+        return [gradient, np.sum(gradient, axis=0, keepdims=True)]
+
+class DotProduct(FunctionNode):
+    """
+    Batched dot product
+
+    Usage: nn.DotProduct(features, weights)
+    Inputs:
+        features: a Node with shape (batch_size x num_features)
+        weights: a Node with shape (1 x num_features)
+    Output: a Node with shape (batch_size x 1)
+    """
+    @staticmethod
+    def _forward(*inputs):
+        assert len(inputs) == 2, "Expected 2 inputs, got {}".format(len(inputs))
+        assert inputs[0].ndim == 2, (
+            "First input should have 2 dimensions, instead has {}".format(
+                inputs[0].ndim))
+        assert inputs[1].ndim == 2, (
+            "Second input should have 2 dimensions, instead has {}".format(
+                inputs[1].ndim))
+        assert inputs[1].shape[0] == 1, (
+            "First dimension of second input should be 1, instead got shape "
+            "{}".format(format_shape(inputs[1].shape)))
+        assert inputs[0].shape[1] == inputs[1].shape[1], (
+            "Second dimension of inputs should match, instead got shapes {} "
+            "and {}".format(
+                format_shape(inputs[0].shape), format_shape(inputs[1].shape)))
+        return np.dot(inputs[0], inputs[1].T)
+
+    @staticmethod
+    def _backward(gradient, *inputs):
+        # assert gradient.shape[0] == inputs[0].shape[0]
+        # assert gradient.shape[1] == 1
+        # return [np.dot(gradient, inputs[1]), np.dot(gradient.T, inputs[0])]
+        raise NotImplementedError(
+            "Backpropagation through DotProduct nodes is not needed in this "
+            "assignment")
+
+class Linear(FunctionNode):
+    """
+    Applies a linear transformation (matrix multiplication) to the input
+
+    Usage: nn.Linear(features, weights)
+    Inputs:
+        features: a Node with shape (batch_size x input_features)
+        weights: a Node with shape (input_features x output_features)
+    Output: a node with shape (batch_size x input_features)
+    """
+    @staticmethod
+    def _forward(*inputs):
+        assert len(inputs) == 2, "Expected 2 inputs, got {}".format(len(inputs))
+        assert inputs[0].ndim == 2, (
+            "First input should have 2 dimensions, instead has {}".format(
+                inputs[0].ndim))
+        assert inputs[1].ndim == 2, (
+            "Second input should have 2 dimensions, instead has {}".format(
+                inputs[1].ndim))
+        assert inputs[0].shape[1] == inputs[1].shape[0], (
+            "Second dimension of first input should match first dimension of "
+            "second input, instead got shapes {} and {}".format(
+                format_shape(inputs[0].shape), format_shape(inputs[1].shape)))
+        return np.dot(inputs[0], inputs[1])
+
+    @staticmethod
+    def _backward(gradient, *inputs):
+        assert gradient.shape[0] == inputs[0].shape[0]
+        assert gradient.shape[1] == inputs[1].shape[1]
+        return [np.dot(gradient, inputs[1].T), np.dot(inputs[0].T, gradient)]
+
+class ReLU(FunctionNode):
+    """
+    An element-wise Rectified Linear Unit nonlinearity: max(x, 0).
+    This nonlinearity replaces all negative entries in its input with zeros.
+
+    Usage: nn.ReLU(x)
+    Input:
+        x: a Node with shape (batch_size x num_features)
+    Output: a Node with the same shape as x, but no negative entries
+    """
+    @staticmethod
+    def _forward(*inputs):
+        assert len(inputs) == 1, "Expected 1 input, got {}".format(len(inputs))
+        assert inputs[0].ndim == 2, (
+            "Input should have 2 dimensions, instead has {}".format(
+                inputs[0].ndim))
+        return np.maximum(inputs[0], 0)
+
+    @staticmethod
+    def _backward(gradient, *inputs):
+        assert gradient.shape == inputs[0].shape
+        return [gradient * np.where(inputs[0] > 0, 1.0, 0.0)]
+
+class SquareLoss(FunctionNode):
+    """
+    This node first computes 0.5 * (a[i,j] - b[i,j])**2 at all positions (i,j)
+    in the inputs, which creates a (batch_size x dim) matrix. It then calculates
+    and returns the mean of all elements in this matrix.
+
+    Usage: nn.SquareLoss(a, b)
+    Inputs:
+        a: a Node with shape (batch_size x dim)
+        b: a Node with shape (batch_size x dim)
+    Output: a scalar Node (containing a single floating-point number)
+    """
+    @staticmethod
+    def _forward(*inputs):
+        assert len(inputs) == 2, "Expected 2 inputs, got {}".format(len(inputs))
+        assert inputs[0].ndim == 2, (
+            "First input should have 2 dimensions, instead has {}".format(
+                inputs[0].ndim))
+        assert inputs[1].ndim == 2, (
+            "Second input should have 2 dimensions, instead has {}".format(
+                inputs[1].ndim))
+        assert inputs[0].shape == inputs[1].shape, (
+            "Input shapes should match, instead got {} and {}".format(
+                format_shape(inputs[0].shape), format_shape(inputs[1].shape)))
+        return np.mean(np.square(inputs[0] - inputs[1]) / 2)
+
+    @staticmethod
+    def _backward(gradient, *inputs):
+        assert np.asarray(gradient).ndim == 0
+        return [
+            gradient * (inputs[0] - inputs[1]) / inputs[0].size,
+            gradient * (inputs[1] - inputs[0]) / inputs[0].size
+        ]
+
+class SoftmaxLoss(FunctionNode):
+    """
+    A batched softmax loss, used for classification problems.
+
+    IMPORTANT: do not swap the order of the inputs to this node!
+
+    Usage: nn.SoftmaxLoss(logits, labels)
+    Inputs:
+        logits: a Node with shape (batch_size x num_classes). Each row
+            represents the scores associated with that example belonging to a
+            particular class. A score can be an arbitrary real number.
+        labels: a Node with shape (batch_size x num_classes) that encodes the
+            correct labels for the examples. All entries must be non-negative
+            and the sum of values along each row should be 1.
+    Output: a scalar Node (containing a single floating-point number)
+    """
+    @staticmethod
+    def log_softmax(logits):
+        log_probs = logits - np.max(logits, axis=1, keepdims=True)
+        log_probs -= np.log(np.sum(np.exp(log_probs), axis=1, keepdims=True))
+        return log_probs
+
+    @staticmethod
+    def _forward(*inputs):
+        assert len(inputs) == 2, "Expected 2 inputs, got {}".format(len(inputs))
+        assert inputs[0].ndim == 2, (
+            "First input should have 2 dimensions, instead has {}".format(
+                inputs[0].ndim))
+        assert inputs[1].ndim == 2, (
+            "Second input should have 2 dimensions, instead has {}".format(
+                inputs[1].ndim))
+        assert inputs[0].shape == inputs[1].shape, (
+            "Input shapes should match, instead got {} and {}".format(
+                format_shape(inputs[0].shape), format_shape(inputs[1].shape)))
+        assert np.all(inputs[1] >= 0), (
+            "All entries in the labels input must be non-negative")
+        assert np.allclose(np.sum(inputs[1], axis=1), 1), (
+            "Labels input must sum to 1 along each row")
+        log_probs = SoftmaxLoss.log_softmax(inputs[0])
+        return np.mean(-np.sum(inputs[1] * log_probs, axis=1))
+
+    @staticmethod
+    def _backward(gradient, *inputs):
+        assert np.asarray(gradient).ndim == 0
+        log_probs = SoftmaxLoss.log_softmax(inputs[0])
+        return [
+            gradient * (np.exp(log_probs) - inputs[1]) / inputs[0].shape[0],
+            gradient * -log_probs / inputs[0].shape[0]
+        ]
+
+def gradients(loss, parameters):
+    """
+    Computes and returns the gradient of the loss with respect to the provided
+    parameters.
+
+    Usage: nn.gradients(loss, parameters)
+    Inputs:
+        loss: a SquareLoss or SoftmaxLoss node
+        parameters: a list (or iterable) containing Parameter nodes
+    Output: a list of Constant objects, representing the gradient of the loss
+        with respect to each provided parameter.
+    """
+
+    assert isinstance(loss, (SquareLoss, SoftmaxLoss)), (
+        "Loss must be a loss node, instead has type {!r}".format(
+            type(loss).__name__))
+    assert all(isinstance(parameter, Parameter) for parameter in parameters), (
+        "Parameters must all have type {}, instead got types {!r}".format(
+            Parameter.__name__,
+            tuple(type(parameter).__name__ for parameter in parameters)))
+    assert not hasattr(loss, "used"), (
+        "Loss node has already been used for backpropagation, cannot reuse")
+
+    loss.used = True
+
+    nodes = set()
+    tape = []
+
+    def visit(node):
+        if node not in nodes:
+            for parent in node.parents:
+                visit(parent)
+            nodes.add(node)
+            tape.append(node)
+
+    visit(loss)
+    nodes |= set(parameters)
+
+    grads = {node: np.zeros_like(node.data) for node in nodes}
+    grads[loss] = 1.0
+
+    for node in reversed(tape):
+        parent_grads = node._backward(
+            grads[node], *(parent.data for parent in node.parents))
+        for parent, parent_grad in zip(node.parents, parent_grads):
+            grads[parent] += parent_grad
+
+    return [Constant(grads[parameter]) for parameter in parameters]
+
+def as_scalar(node):
+    """
+    Returns the value of a Node as a standard Python number. This only works
+    for nodes with one element (e.g. SquareLoss and SoftmaxLoss, as well as
+    DotProduct with a batch size of 1 element).
+    """
+
+    assert isinstance(node, Node), (
+        "Input must be a node object, instead has type {!r}".format(
+            type(node).__name__))
+    assert node.data.size == 1, (
+        "Node has shape {}, cannot convert to a scalar".format(
+            format_shape(node.data.shape)))
+    return np.asscalar(node.data)