diff --git a/__pycache__/backend.cpython-37.pyc b/__pycache__/backend.cpython-37.pyc new file mode 100644 index 0000000..41042d9 Binary files /dev/null and b/__pycache__/backend.cpython-37.pyc differ diff --git a/__pycache__/nn.cpython-37.pyc b/__pycache__/nn.cpython-37.pyc new file mode 100644 index 0000000..e525f0a Binary files /dev/null and b/__pycache__/nn.cpython-37.pyc differ diff --git a/autograder.py b/autograder.py new file mode 100644 index 0000000..3763fbe --- /dev/null +++ b/autograder.py @@ -0,0 +1,579 @@ +# A custom autograder for this project + +################################################################################ +# A mini-framework for autograding +################################################################################ + +import optparse +import pickle +import random +import sys +import traceback + +class WritableNull: + def write(self, string): + pass + + def flush(self): + pass + +class Tracker(object): + def __init__(self, questions, maxes, prereqs, mute_output): + self.questions = questions + self.maxes = maxes + self.prereqs = prereqs + + self.points = {q: 0 for q in self.questions} + + self.current_question = None + + self.current_test = None + self.points_at_test_start = None + self.possible_points_remaining = None + + self.mute_output = mute_output + self.original_stdout = None + self.muted = False + + def mute(self): + if self.muted: + return + + self.muted = True + self.original_stdout = sys.stdout + sys.stdout = WritableNull() + + def unmute(self): + if not self.muted: + return + + self.muted = False + sys.stdout = self.original_stdout + + def begin_q(self, q): + assert q in self.questions + text = 'Question {}'.format(q) + print('\n' + text) + print('=' * len(text)) + + for prereq in sorted(self.prereqs[q]): + if self.points[prereq] < self.maxes[prereq]: + print("""*** NOTE: Make sure to complete Question {} before working on Question {}, +*** because Question {} builds upon your answer for Question {}. +""".format(prereq, q, q, prereq)) + return False + + self.current_question = q + self.possible_points_remaining = self.maxes[q] + return True + + def begin_test(self, test_name): + self.current_test = test_name + self.points_at_test_start = self.points[self.current_question] + print("*** {}) {}".format(self.current_question, self.current_test)) + if self.mute_output: + self.mute() + + def end_test(self, pts): + if self.mute_output: + self.unmute() + self.possible_points_remaining -= pts + if self.points[self.current_question] == self.points_at_test_start + pts: + print("*** PASS: {}".format(self.current_test)) + elif self.points[self.current_question] == self.points_at_test_start: + print("*** FAIL") + + self.current_test = None + self.points_at_test_start = None + + def end_q(self): + assert self.current_question is not None + assert self.possible_points_remaining == 0 + print('\n### Question {}: {}/{} ###'.format( + self.current_question, + self.points[self.current_question], + self.maxes[self.current_question])) + + self.current_question = None + self.possible_points_remaining = None + + def finalize(self): + import time + print('\nFinished at %d:%02d:%02d' % time.localtime()[3:6]) + print("\nProvisional grades\n==================") + + for q in self.questions: + print('Question %s: %d/%d' % (q, self.points[q], self.maxes[q])) + print('------------------') + print('Total: %d/%d' % (sum(self.points.values()), + sum([self.maxes[q] for q in self.questions]))) + + print(""" +Your grades are NOT yet registered. To register your grades, make sure +to follow your instructor's guidelines to receive credit on your project. +""") + + def add_points(self, pts): + self.points[self.current_question] += pts + +TESTS = [] +PREREQS = {} +def add_prereq(q, pre): + if isinstance(pre, str): + pre = [pre] + + if q not in PREREQS: + PREREQS[q] = set() + PREREQS[q] |= set(pre) + +def test(q, points): + def deco(fn): + TESTS.append((q, points, fn)) + return fn + return deco + +def parse_options(argv): + parser = optparse.OptionParser(description = 'Run public tests on student code') + parser.set_defaults( + edx_output=False, + gs_output=False, + no_graphics=False, + mute_output=False, + check_dependencies=False, + ) + parser.add_option('--edx-output', + dest = 'edx_output', + action = 'store_true', + help = 'Ignored, present for compatibility only') + parser.add_option('--gradescope-output', + dest = 'gs_output', + action = 'store_true', + help = 'Ignored, present for compatibility only') + parser.add_option('--question', '-q', + dest = 'grade_question', + default = None, + help = 'Grade only one question (e.g. `-q q1`)') + parser.add_option('--no-graphics', + dest = 'no_graphics', + action = 'store_true', + help = 'Do not display graphics (visualizing your implementation is highly recommended for debugging).') + parser.add_option('--mute', + dest = 'mute_output', + action = 'store_true', + help = 'Mute output from executing tests') + parser.add_option('--check-dependencies', + dest = 'check_dependencies', + action = 'store_true', + help = 'check that numpy and matplotlib are installed') + (options, args) = parser.parse_args(argv) + return options + +def main(): + options = parse_options(sys.argv) + if options.check_dependencies: + check_dependencies() + return + + if options.no_graphics: + disable_graphics() + + questions = set() + maxes = {} + for q, points, fn in TESTS: + questions.add(q) + maxes[q] = maxes.get(q, 0) + points + if q not in PREREQS: + PREREQS[q] = set() + + questions = list(sorted(questions)) + if options.grade_question: + if options.grade_question not in questions: + print("ERROR: question {} does not exist".format(options.grade_question)) + sys.exit(1) + else: + questions = [options.grade_question] + PREREQS[options.grade_question] = set() + + tracker = Tracker(questions, maxes, PREREQS, options.mute_output) + for q in questions: + started = tracker.begin_q(q) + if not started: + continue + + for testq, points, fn in TESTS: + if testq != q: + continue + tracker.begin_test(fn.__name__) + try: + fn(tracker) + except KeyboardInterrupt: + tracker.unmute() + print("\n\nCaught KeyboardInterrupt: aborting autograder") + tracker.finalize() + print("\n[autograder was interrupted before finishing]") + sys.exit(1) + except: + tracker.unmute() + print(traceback.format_exc()) + tracker.end_test(points) + tracker.end_q() + tracker.finalize() + +################################################################################ +# Tests begin here +################################################################################ + +import numpy as np +import matplotlib +import contextlib + +import nn +import backend + +def check_dependencies(): + import matplotlib.pyplot as plt + import time + fig, ax = plt.subplots(1, 1) + ax.set_xlim([-1, 1]) + ax.set_ylim([-1, 1]) + line, = ax.plot([], [], color="black") + plt.show(block=False) + + for t in range(400): + angle = t * 0.05 + x = np.sin(angle) + y = np.cos(angle) + line.set_data([x,-x], [y,-y]) + fig.canvas.draw_idle() + fig.canvas.start_event_loop(1e-3) + +def disable_graphics(): + backend.use_graphics = False + +@contextlib.contextmanager +def no_graphics(): + old_use_graphics = backend.use_graphics + backend.use_graphics = False + yield + backend.use_graphics = old_use_graphics + +def verify_node(node, expected_type, expected_shape, method_name): + if expected_type == 'parameter': + assert node is not None, ( + "{} should return an instance of nn.Parameter, not None".format(method_name)) + assert isinstance(node, nn.Parameter), ( + "{} should return an instance of nn.Parameter, instead got type {!r}".format( + method_name, type(node).__name__)) + elif expected_type == 'loss': + assert node is not None, ( + "{} should return an instance a loss node, not None".format(method_name)) + assert isinstance(node, (nn.SquareLoss, nn.SoftmaxLoss)), ( + "{} should return a loss node, instead got type {!r}".format( + method_name, type(node).__name__)) + elif expected_type == 'node': + assert node is not None, ( + "{} should return a node object, not None".format(method_name)) + assert isinstance(node, nn.Node), ( + "{} should return a node object, instead got type {!r}".format( + method_name, type(node).__name__)) + else: + assert False, "If you see this message, please report a bug in the autograder" + + if expected_type != 'loss': + assert all([(expected is '?' or actual == expected) for (actual, expected) in zip(node.data.shape, expected_shape)]), ( + "{} should return an object with shape {}, got {}".format( + method_name, nn.format_shape(expected_shape), nn.format_shape(node.data.shape))) + +def trace_node(node_to_trace): + """ + Returns a set containing the node and all ancestors in the computation graph + """ + nodes = set() + tape = [] + + def visit(node): + if node not in nodes: + for parent in node.parents: + visit(parent) + nodes.add(node) + tape.append(node) + + visit(node_to_trace) + + return nodes + +@test('q1', points=6) +def check_perceptron(tracker): + import models + + print("Sanity checking perceptron...") + np_random = np.random.RandomState(0) + # Check that the perceptron weights are initialized to a vector with `dimensions` entries. + for dimensions in range(1, 10): + p = models.PerceptronModel(dimensions) + p_weights = p.get_weights() + verify_node(p_weights, 'parameter', (1, dimensions), "PerceptronModel.get_weights()") + + # Check that run returns a node, and that the score in the node is correct + for dimensions in range(1, 10): + p = models.PerceptronModel(dimensions) + p_weights = p.get_weights() + verify_node(p_weights, 'parameter', (1, dimensions), "PerceptronModel.get_weights()") + point = np_random.uniform(-10, 10, (1, dimensions)) + score = p.run(nn.Constant(point)) + verify_node(score, 'node', (1, 1), "PerceptronModel.run()") + calculated_score = nn.as_scalar(score) + expected_score = float(np.dot(point.flatten(), p_weights.data.flatten())) + assert np.isclose(calculated_score, expected_score), ( + "The score computed by PerceptronModel.run() ({:.4f}) does not match the expected score ({:.4f})".format( + calculated_score, expected_score)) + + # Check that get_prediction returns the correct values, including the + # case when a point lies exactly on the decision boundary + for dimensions in range(1, 10): + p = models.PerceptronModel(dimensions) + random_point = np_random.uniform(-10, 10, (1, dimensions)) + for point in (random_point, np.zeros_like(random_point)): + prediction = p.get_prediction(nn.Constant(point)) + assert prediction == 1 or prediction == -1, ( + "PerceptronModel.get_prediction() should return 1 or -1, not {}".format( + prediction)) + + expected_prediction = np.asscalar(np.where(np.dot(point, p.get_weights().data.T) >= 0, 1, -1)) + assert prediction == expected_prediction, ( + "PerceptronModel.get_prediction() returned {}; expected {}".format( + prediction, expected_prediction)) + + tracker.add_points(2) # Partial credit for passing sanity checks + + print("Sanity checking perceptron weight updates...") + + # Test weight updates. This involves constructing a dataset that + # requires 0 or 1 updates before convergence, and testing that weight + # values change as expected. Note that (multiplier < -1 or multiplier > 1) + # must be true for the testing code to be correct. + dimensions = 2 + for multiplier in (-5, -2, 2, 5): + p = models.PerceptronModel(dimensions) + orig_weights = p.get_weights().data.reshape((1, dimensions)).copy() + if np.abs(orig_weights).sum() == 0.0: + # This autograder test doesn't work when weights are exactly zero + continue + point = multiplier * orig_weights + sanity_dataset = backend.Dataset( + x=np.tile(point, (500, 1)), + y=np.ones((500, 1)) * -1.0 + ) + p.train(sanity_dataset) + new_weights = p.get_weights().data.reshape((1, dimensions)) + + if multiplier < 0: + expected_weights = orig_weights + else: + expected_weights = orig_weights - point + + if not np.all(new_weights == expected_weights): + print() + print("Initial perceptron weights were: [{:.4f}, {:.4f}]".format( + orig_weights[0,0], orig_weights[0,1])) + print("All data points in the dataset were identical and had:") + print(" x = [{:.4f}, {:.4f}]".format( + point[0,0], point[0,1])) + print(" y = -1") + print("Your trained weights were: [{:.4f}, {:.4f}]".format( + new_weights[0,0], new_weights[0,1])) + print("Expected weights after training: [{:.4f}, {:.4f}]".format( + expected_weights[0,0], expected_weights[0,1])) + print() + assert False, "Weight update sanity check failed" + + print("Sanity checking complete. Now training perceptron") + model = models.PerceptronModel(3) + dataset = backend.PerceptronDataset(model) + + model.train(dataset) + backend.maybe_sleep_and_close(1) + + assert dataset.epoch != 0, "Perceptron code never iterated over the training data" + + accuracy = np.mean(np.where(np.dot(dataset.x, model.get_weights().data.T) >= 0.0, 1.0, -1.0) == dataset.y) + if accuracy < 1.0: + print("The weights learned by your perceptron correctly classified {:.2%} of training examples".format(accuracy)) + print("To receive full points for this question, your perceptron must converge to 100% accuracy") + return + + tracker.add_points(4) + +@test('q2', points=6) +def check_regression(tracker): + import models + model = models.RegressionModel() + dataset = backend.RegressionDataset(model) + + detected_parameters = None + for batch_size in (1, 2, 4): + inp_x = nn.Constant(dataset.x[:batch_size]) + inp_y = nn.Constant(dataset.y[:batch_size]) + output_node = model.run(inp_x) + verify_node(output_node, 'node', (batch_size, 1), "RegressionModel.run()") + trace = trace_node(output_node) + assert inp_x in trace, "Node returned from RegressionModel.run() does not depend on the provided input (x)" + + if detected_parameters is None: + detected_parameters = [node for node in trace if isinstance(node, nn.Parameter)] + + for node in trace: + assert not isinstance(node, nn.Parameter) or node in detected_parameters, ( + "Calling RegressionModel.run() multiple times should always re-use the same parameters, but a new nn.Parameter object was detected") + + for batch_size in (1, 2, 4): + inp_x = nn.Constant(dataset.x[:batch_size]) + inp_y = nn.Constant(dataset.y[:batch_size]) + loss_node = model.get_loss(inp_x, inp_y) + verify_node(loss_node, 'loss', None, "RegressionModel.get_loss()") + trace = trace_node(loss_node) + assert inp_x in trace, "Node returned from RegressionModel.get_loss() does not depend on the provided input (x)" + assert inp_y in trace, "Node returned from RegressionModel.get_loss() does not depend on the provided labels (y)" + + for node in trace: + assert not isinstance(node, nn.Parameter) or node in detected_parameters, ( + "RegressionModel.get_loss() should not use additional parameters not used by RegressionModel.run()") + + tracker.add_points(2) # Partial credit for passing sanity checks + + model.train(dataset) + backend.maybe_sleep_and_close(1) + + train_loss = model.get_loss(nn.Constant(dataset.x), nn.Constant(dataset.y)) + verify_node(train_loss, 'loss', None, "RegressionModel.get_loss()") + train_loss = nn.as_scalar(train_loss) + + # Re-compute the loss ourselves: otherwise get_loss() could be hard-coded + # to always return zero + train_predicted = model.run(nn.Constant(dataset.x)) + verify_node(train_predicted, 'node', (dataset.x.shape[0], 1), "RegressionModel.run()") + sanity_loss = 0.5 * np.mean((train_predicted.data - dataset.y)**2) + + assert np.isclose(train_loss, sanity_loss), ( + "RegressionModel.get_loss() returned a loss of {:.4f}, " + "but the autograder computed a loss of {:.4f} " + "based on the output of RegressionModel.run()".format( + train_loss, sanity_loss)) + + loss_threshold = 0.02 + if train_loss <= loss_threshold: + print("Your final loss is: {:f}".format(train_loss)) + tracker.add_points(4) + else: + print("Your final loss ({:f}) must be no more than {:.4f} to receive full points for this question".format(train_loss, loss_threshold)) + +@test('q3', points=6) +def check_digit_classification(tracker): + import models + model = models.DigitClassificationModel() + dataset = backend.DigitClassificationDataset(model) + + detected_parameters = None + for batch_size in (1, 2, 4): + inp_x = nn.Constant(dataset.x[:batch_size]) + inp_y = nn.Constant(dataset.y[:batch_size]) + output_node = model.run(inp_x) + verify_node(output_node, 'node', (batch_size, 10), "DigitClassificationModel.run()") + trace = trace_node(output_node) + assert inp_x in trace, "Node returned from DigitClassificationModel.run() does not depend on the provided input (x)" + + if detected_parameters is None: + detected_parameters = [node for node in trace if isinstance(node, nn.Parameter)] + + for node in trace: + assert not isinstance(node, nn.Parameter) or node in detected_parameters, ( + "Calling DigitClassificationModel.run() multiple times should always re-use the same parameters, but a new nn.Parameter object was detected") + + for batch_size in (1, 2, 4): + inp_x = nn.Constant(dataset.x[:batch_size]) + inp_y = nn.Constant(dataset.y[:batch_size]) + loss_node = model.get_loss(inp_x, inp_y) + verify_node(loss_node, 'loss', None, "DigitClassificationModel.get_loss()") + trace = trace_node(loss_node) + assert inp_x in trace, "Node returned from DigitClassificationModel.get_loss() does not depend on the provided input (x)" + assert inp_y in trace, "Node returned from DigitClassificationModel.get_loss() does not depend on the provided labels (y)" + + for node in trace: + assert not isinstance(node, nn.Parameter) or node in detected_parameters, ( + "DigitClassificationModel.get_loss() should not use additional parameters not used by DigitClassificationModel.run()") + + tracker.add_points(2) # Partial credit for passing sanity checks + + model.train(dataset) + + test_logits = model.run(nn.Constant(dataset.test_images)).data + test_predicted = np.argmax(test_logits, axis=1) + test_accuracy = np.mean(test_predicted == dataset.test_labels) + + accuracy_threshold = 0.97 + if test_accuracy >= accuracy_threshold: + print("Your final test set accuracy is: {:%}".format(test_accuracy)) + tracker.add_points(4) + else: + print("Your final test set accuracy ({:%}) must be at least {:.0%} to receive full points for this question".format(test_accuracy, accuracy_threshold)) + +@test('q4', points=7) +def check_lang_id(tracker): + import models + model = models.LanguageIDModel() + dataset = backend.LanguageIDDataset(model) + + detected_parameters = None + for batch_size, word_length in ((1, 1), (2, 1), (2, 6), (4, 8)): + start = dataset.dev_buckets[-1, 0] + end = start + batch_size + inp_xs, inp_y = dataset._encode(dataset.dev_x[start:end], dataset.dev_y[start:end]) + inp_xs = inp_xs[:word_length] + + output_node = model.run(inp_xs) + verify_node(output_node, 'node', (batch_size, len(dataset.language_names)), "LanguageIDModel.run()") + trace = trace_node(output_node) + for inp_x in inp_xs: + assert inp_x in trace, "Node returned from LanguageIDModel.run() does not depend on all of the provided inputs (xs)" + + # Word length 1 does not use parameters related to transferring the + # hidden state across timesteps, so initial parameter detection is only + # run for longer words + if word_length > 1: + if detected_parameters is None: + detected_parameters = [node for node in trace if isinstance(node, nn.Parameter)] + + for node in trace: + assert not isinstance(node, nn.Parameter) or node in detected_parameters, ( + "Calling LanguageIDModel.run() multiple times should always re-use the same parameters, but a new nn.Parameter object was detected") + + for batch_size, word_length in ((1, 1), (2, 1), (2, 6), (4, 8)): + start = dataset.dev_buckets[-1, 0] + end = start + batch_size + inp_xs, inp_y = dataset._encode(dataset.dev_x[start:end], dataset.dev_y[start:end]) + inp_xs = inp_xs[:word_length] + loss_node = model.get_loss(inp_xs, inp_y) + trace = trace_node(loss_node) + for inp_x in inp_xs: + assert inp_x in trace, "Node returned from LanguageIDModel.run() does not depend on all of the provided inputs (xs)" + assert inp_y in trace, "Node returned from LanguageIDModel.get_loss() does not depend on the provided labels (y)" + + for node in trace: + assert not isinstance(node, nn.Parameter) or node in detected_parameters, ( + "LanguageIDModel.get_loss() should not use additional parameters not used by LanguageIDModel.run()") + + tracker.add_points(2) # Partial credit for passing sanity checks + + model.train(dataset) + + test_predicted_probs, test_predicted, test_correct = dataset._predict('test') + test_accuracy = np.mean(test_predicted == test_correct) + accuracy_threshold = 0.81 + if test_accuracy >= accuracy_threshold: + print("Your final test set accuracy is: {:%}".format(test_accuracy)) + tracker.add_points(5) + else: + print("Your final test set accuracy ({:%}) must be at least {:.0%} to receive full points for this question".format(test_accuracy, accuracy_threshold)) + +if __name__ == '__main__': + main() diff --git a/backend.py b/backend.py new file mode 100644 index 0000000..c283736 --- /dev/null +++ b/backend.py @@ -0,0 +1,447 @@ +import collections +import os +import time +import os + +import matplotlib.pyplot as plt +import numpy as np + +import nn + +use_graphics = True + +def maybe_sleep_and_close(seconds): + if use_graphics and plt.get_fignums(): + time.sleep(seconds) + for fignum in plt.get_fignums(): + fig = plt.figure(fignum) + plt.close(fig) + try: + # This raises a TclError on some Windows machines + fig.canvas.start_event_loop(1e-3) + except: + pass + +def get_data_path(filename): + path = os.path.join( + os.path.dirname(__file__), os.pardir, "data", filename) + if not os.path.exists(path): + path = os.path.join( + os.path.dirname(__file__), "data", filename) + if not os.path.exists(path): + path = os.path.join( + os.path.dirname(__file__), filename) + if not os.path.exists(path): + raise Exception("Could not find data file: {}".format(filename)) + return path + +class Dataset(object): + def __init__(self, x, y): + assert isinstance(x, np.ndarray) + assert isinstance(y, np.ndarray) + assert np.issubdtype(x.dtype, np.floating) + assert np.issubdtype(y.dtype, np.floating) + assert x.ndim == 2 + assert y.ndim == 2 + assert x.shape[0] == y.shape[0] + self.x = x + self.y = y + + def iterate_once(self, batch_size): + assert isinstance(batch_size, int) and batch_size > 0, ( + "Batch size should be a positive integer, got {!r}".format( + batch_size)) + assert self.x.shape[0] % batch_size == 0, ( + "Dataset size {:d} is not divisible by batch size {:d}".format( + self.x.shape[0], batch_size)) + index = 0 + while index < self.x.shape[0]: + x = self.x[index:index + batch_size] + y = self.y[index:index + batch_size] + yield nn.Constant(x), nn.Constant(y) + index += batch_size + + def iterate_forever(self, batch_size): + while True: + yield from self.iterate_once(batch_size) + + def get_validation_accuracy(self): + raise NotImplementedError( + "No validation data is available for this dataset. " + "In this assignment, only the Digit Classification and Language " + "Identification datasets have validation data.") + +class PerceptronDataset(Dataset): + def __init__(self, model): + points = 500 + x = np.hstack([np.random.randn(points, 2), np.ones((points, 1))]) + y = np.where(x[:, 0] + 2 * x[:, 1] - 1 >= 0, 1.0, -1.0) + super().__init__(x, np.expand_dims(y, axis=1)) + + self.model = model + self.epoch = 0 + + if use_graphics: + fig, ax = plt.subplots(1, 1) + limits = np.array([-3.0, 3.0]) + ax.set_xlim(limits) + ax.set_ylim(limits) + positive = ax.scatter(*x[y == 1, :-1].T, color="red", marker="+") + negative = ax.scatter(*x[y == -1, :-1].T, color="blue", marker="_") + line, = ax.plot([], [], color="black") + text = ax.text(0.03, 0.97, "", transform=ax.transAxes, va="top") + ax.legend([positive, negative], [1, -1]) + plt.show(block=False) + + self.fig = fig + self.limits = limits + self.line = line + self.text = text + self.last_update = time.time() + + def iterate_once(self, batch_size): + self.epoch += 1 + + for i, (x, y) in enumerate(super().iterate_once(batch_size)): + yield x, y + + if use_graphics and time.time() - self.last_update > 0.01: + w = self.model.get_weights().data.flatten() + limits = self.limits + if w[1] != 0: + self.line.set_data(limits, (-w[0] * limits - w[2]) / w[1]) + elif w[0] != 0: + self.line.set_data(np.full(2, -w[2] / w[0]), limits) + else: + self.line.set_data([], []) + self.text.set_text( + "epoch: {:,}\npoint: {:,}/{:,}\nweights: {}".format( + self.epoch, i * batch_size + 1, len(self.x), w)) + self.fig.canvas.draw_idle() + self.fig.canvas.start_event_loop(1e-3) + self.last_update = time.time() + +class RegressionDataset(Dataset): + def __init__(self, model): + x = np.expand_dims(np.linspace(-2 * np.pi, 2 * np.pi, num=200), axis=1) + np.random.RandomState(0).shuffle(x) + self.argsort_x = np.argsort(x.flatten()) + y = np.sin(x) + super().__init__(x, y) + + self.model = model + self.processed = 0 + + if use_graphics: + fig, ax = plt.subplots(1, 1) + ax.set_xlim(-2 * np.pi, 2 * np.pi) + ax.set_ylim(-1.4, 1.4) + real, = ax.plot(x[self.argsort_x], y[self.argsort_x], color="blue") + learned, = ax.plot([], [], color="red") + text = ax.text(0.03, 0.97, "", transform=ax.transAxes, va="top") + ax.legend([real, learned], ["real", "learned"]) + plt.show(block=False) + + self.fig = fig + self.learned = learned + self.text = text + self.last_update = time.time() + + def iterate_once(self, batch_size): + for x, y in super().iterate_once(batch_size): + yield x, y + self.processed += batch_size + + if use_graphics and time.time() - self.last_update > 0.1: + predicted = self.model.run(nn.Constant(self.x)).data + loss = self.model.get_loss( + nn.Constant(self.x), nn.Constant(self.y)).data + self.learned.set_data(self.x[self.argsort_x], predicted[self.argsort_x]) + self.text.set_text("processed: {:,}\nloss: {:.6f}".format( + self.processed, loss)) + self.fig.canvas.draw_idle() + self.fig.canvas.start_event_loop(1e-3) + self.last_update = time.time() + +class DigitClassificationDataset(Dataset): + def __init__(self, model): + mnist_path = get_data_path("mnist.npz") + + with np.load(mnist_path) as data: + train_images = data["train_images"] + train_labels = data["train_labels"] + test_images = data["test_images"] + test_labels = data["test_labels"] + assert len(train_images) == len(train_labels) == 60000 + assert len(test_images) == len(test_labels) == 10000 + self.dev_images = test_images[0::2] + self.dev_labels = test_labels[0::2] + self.test_images = test_images[1::2] + self.test_labels = test_labels[1::2] + + train_labels_one_hot = np.zeros((len(train_images), 10)) + train_labels_one_hot[range(len(train_images)), train_labels] = 1 + + super().__init__(train_images, train_labels_one_hot) + + self.model = model + self.epoch = 0 + + if use_graphics: + width = 20 # Width of each row expressed as a multiple of image width + samples = 100 # Number of images to display per label + fig = plt.figure() + ax = {} + images = collections.defaultdict(list) + texts = collections.defaultdict(list) + for i in reversed(range(10)): + ax[i] = plt.subplot2grid((30, 1), (3 * i, 0), 2, 1, + sharex=ax.get(9)) + plt.setp(ax[i].get_xticklabels(), visible=i == 9) + ax[i].set_yticks([]) + ax[i].text(-0.03, 0.5, i, transform=ax[i].transAxes, + va="center") + ax[i].set_xlim(0, 28 * width) + ax[i].set_ylim(0, 28) + for j in range(samples): + images[i].append(ax[i].imshow( + np.zeros((28, 28)), vmin=0, vmax=1, cmap="Greens", + alpha=0.3)) + texts[i].append(ax[i].text( + 0, 0, "", ha="center", va="top", fontsize="smaller")) + ax[9].set_xticks(np.linspace(0, 28 * width, 11)) + ax[9].set_xticklabels( + ["{:.1f}".format(num) for num in np.linspace(0, 1, 11)]) + ax[9].tick_params(axis="x", pad=16) + ax[9].set_xlabel("Probability of Correct Label") + status = ax[0].text( + 0.5, 1.5, "", transform=ax[0].transAxes, ha="center", + va="bottom") + plt.show(block=False) + + self.width = width + self.samples = samples + self.fig = fig + self.images = images + self.texts = texts + self.status = status + self.last_update = time.time() + + def iterate_once(self, batch_size): + self.epoch += 1 + + for i, (x, y) in enumerate(super().iterate_once(batch_size)): + yield x, y + + if use_graphics and time.time() - self.last_update > 1: + dev_logits = self.model.run(nn.Constant(self.dev_images)).data + dev_predicted = np.argmax(dev_logits, axis=1) + dev_probs = np.exp(nn.SoftmaxLoss.log_softmax(dev_logits)) + dev_accuracy = np.mean(dev_predicted == self.dev_labels) + + self.status.set_text( + "epoch: {:d}, batch: {:d}/{:d}, validation accuracy: " + "{:.2%}".format( + self.epoch, i, len(self.x) // batch_size, dev_accuracy)) + for i in range(10): + predicted = dev_predicted[self.dev_labels == i] + probs = dev_probs[self.dev_labels == i][:, i] + linspace = np.linspace( + 0, len(probs) - 1, self.samples).astype(int) + indices = probs.argsort()[linspace] + for j, (prob, image) in enumerate(zip( + probs[indices], + self.dev_images[self.dev_labels == i][indices])): + self.images[i][j].set_data(image.reshape((28, 28))) + left = prob * (self.width - 1) * 28 + if predicted[indices[j]] == i: + self.images[i][j].set_cmap("Greens") + self.texts[i][j].set_text("") + else: + self.images[i][j].set_cmap("Reds") + self.texts[i][j].set_text(predicted[indices[j]]) + self.texts[i][j].set_x(left + 14) + self.images[i][j].set_extent([left, left + 28, 0, 28]) + self.fig.canvas.draw_idle() + self.fig.canvas.start_event_loop(1e-3) + self.last_update = time.time() + + def get_validation_accuracy(self): + dev_logits = self.model.run(nn.Constant(self.dev_images)).data + dev_predicted = np.argmax(dev_logits, axis=1) + dev_accuracy = np.mean(dev_predicted == self.dev_labels) + return dev_accuracy + +class LanguageIDDataset(Dataset): + def __init__(self, model): + self.model = model + + data_path = get_data_path("lang_id.npz") + + with np.load(data_path) as data: + self.chars = data['chars'] + self.language_codes = data['language_codes'] + self.language_names = data['language_names'] + + self.train_x = data['train_x'] + self.train_y = data['train_y'] + self.train_buckets = data['train_buckets'] + self.dev_x = data['dev_x'] + self.dev_y = data['dev_y'] + self.dev_buckets = data['dev_buckets'] + self.test_x = data['test_x'] + self.test_y = data['test_y'] + self.test_buckets = data['test_buckets'] + + self.epoch = 0 + self.bucket_weights = self.train_buckets[:,1] - self.train_buckets[:,0] + self.bucket_weights = self.bucket_weights / float(self.bucket_weights.sum()) + + self.chars_print = self.chars + try: + print(u"Alphabet: {}".format(u"".join(self.chars))) + except UnicodeEncodeError: + self.chars_print = "abcdefghijklmnopqrstuvwxyzaaeeeeiinoouuacelnszz" + print("Alphabet: " + self.chars_print) + self.chars_print = list(self.chars_print) + print(""" +NOTE: Your terminal does not appear to support printing Unicode characters. +For the purposes of printing to the terminal, some of the letters in the +alphabet above have been substituted with ASCII symbols.""".strip()) + print("") + + # Select some examples to spotlight in the monitoring phase (3 per language) + spotlight_idxs = [] + for i in range(len(self.language_names)): + idxs_lang_i = np.nonzero(self.dev_y == i)[0] + idxs_lang_i = np.random.choice(idxs_lang_i, size=3, replace=False) + spotlight_idxs.extend(list(idxs_lang_i)) + self.spotlight_idxs = np.array(spotlight_idxs, dtype=int) + + # Templates for printing updates as training progresses + max_word_len = self.dev_x.shape[1] + max_lang_len = max([len(x) for x in self.language_names]) + + self.predicted_template = u"Pred: {: 0, ( + "Batch size should be a positive integer, got {!r}".format( + batch_size)) + assert self.train_x.shape[0] >= batch_size, ( + "Dataset size {:d} is smaller than the batch size {:d}".format( + self.train_x.shape[0], batch_size)) + + self.epoch += 1 + + for iteration in range(self.train_x.shape[0] // batch_size): + bucket_id = np.random.choice(self.bucket_weights.shape[0], p=self.bucket_weights) + example_ids = self.train_buckets[bucket_id, 0] + np.random.choice( + self.train_buckets[bucket_id, 1] - self.train_buckets[bucket_id, 0], + size=batch_size) + + yield self._encode(self.train_x[example_ids], self.train_y[example_ids]) + + if use_graphics and time.time() - self.last_update > 0.5: + dev_predicted_probs, dev_predicted, dev_correct = self._predict() + dev_accuracy = np.mean(dev_predicted == dev_correct) + + print("epoch {:,} iteration {:,} validation-accuracy {:.1%}".format( + self.epoch, iteration, dev_accuracy)) + + for idx in self.spotlight_idxs: + correct = (dev_predicted[idx] == dev_correct[idx]) + word = u"".join([self.chars_print[ch] for ch in self.dev_x[idx] if ch != -1]) + + print(self.word_template.format( + word, + self.language_names[dev_correct[idx]], + dev_predicted_probs[idx, dev_correct[idx]], + "" if correct else self.predicted_template.format( + self.language_names[dev_predicted[idx]]), + probs=dev_predicted_probs[idx,:], + )) + + self.last_update = time.time() + + def get_validation_accuracy(self): + dev_predicted_probs, dev_predicted, dev_correct = self._predict() + dev_accuracy = np.mean(dev_predicted == dev_correct) + return dev_accuracy + + +def main(): + import models + model = models.PerceptronModel(3) + dataset = PerceptronDataset(model) + model.train(dataset) + + model = models.RegressionModel() + dataset = RegressionDataset(model) + model.train(dataset) + + model = models.DigitClassificationModel() + dataset = DigitClassificationDataset(model) + model.train(dataset) + + model = models.LanguageIDModel() + dataset = LanguageIDDataset(model) + model.train(dataset) + +if __name__ == "__main__": + main() diff --git a/data/lang_id.npz b/data/lang_id.npz new file mode 100644 index 0000000..3974849 Binary files /dev/null and b/data/lang_id.npz differ diff --git a/data/mnist.npz b/data/mnist.npz new file mode 100644 index 0000000..abf960a Binary files /dev/null and b/data/mnist.npz differ diff --git a/models.py b/models.py new file mode 100644 index 0000000..f4fde16 --- /dev/null +++ b/models.py @@ -0,0 +1,282 @@ +import nn +import backend +import numpy as np + + +class PerceptronModel(object): + def __init__(self, dimensions): + """ + Initialize a new Perceptron instance. + + A perceptron classifies data points as either belonging to a particular + class (+1) or not (-1). `dimensions` is the dimensionality of the data. + For example, dimensions=2 would mean that the perceptron must classify + 2D points. + """ + self.w = nn.Parameter(1, dimensions) + + def get_weights(self): + """ + Return a Parameter instance with the current weights of the perceptron. + """ + return self.w + + def run(self, x): + """ + Calculates the score assigned by the perceptron to a data point x. + + Inputs: + x: a node with shape (1 x dimensions) + Returns: a node containing a single number (the score) + """ + "*** YOUR CODE HERE ***" + + def get_prediction(self, x): + """ + Calculates the predicted class for a single data point `x`. + + Returns: 1 or -1 + """ + "*** YOUR CODE HERE ***" + + def train(self, dataset): + """ + Train the perceptron until convergence. + """ + "*** YOUR CODE HERE ***" + + +class RegressionModel(object): + """ + A neural network model for approximating a function that maps from real + numbers to real numbers. The network should be sufficiently large to be able + to approximate sin(x) on the interval [-2pi, 2pi] to reasonable precision. + """ + + def __init__(self): + # Initialize your model parameters here + "*** YOUR CODE HERE ***" + model = object.__init__(self) + self.get_data_and_monitor = backend.RegressionDataset(model) + + # Remember to set self.learning_rate! + # You may use any learning rate that works well for your architecture + "*** YOUR CODE HERE ***" + self.learning_rate = 0.1 + self.hidden_size = 300 + + self.w1 = nn.Parameter(1, self.hidden_size) + self.w2 = nn.Parameter(self.hidden_size, self.hidden_size) + self.w3 = nn.Parameter(self.hidden_size, 1) + self.b1 = nn.Parameter(self.hidden_size) + self.b2 = nn.Parameter(self.hidden_size) + self.b3 = nn.Parameter(1) + + def run(self, x): + """ + Runs the model for a batch of examples. + + Inputs: + x: a node with shape (batch_size x 1) + Returns: + A node with shape (batch_size x 1) containing predicted y-values + """ + "*** YOUR CODE HERE ***" + self.graph = nn.Graph( + [self.w1, self.w2, self.w3, self.b1, self.b2, self.b3]) + + if y is not None: + # At training time, the correct output `y` is known. + # Here, you should construct a loss node, and return the nn.Graph + # that the node belongs to. The loss node must be the last node + # added to the graph. + "*** YOUR CODE HERE ***" + input_x = nn.Input(self.graph, x) + input_y = nn.Input(self.graph, y) + xw1 = nn.MatrixMultiply(self.graph, input_x, self.w1) + xw1_plus_b1 = nn.MatrixVectorAdd(self.graph, xw1, self.b1) + l1 = nn.ReLU(self.graph, xw1_plus_b1) + l1w2 = nn.MatrixMultiply(self.graph, l1, self.w2) + l2w2_plus_b2 = nn.MatrixVectorAdd(self.graph, l1w2, self.b2) + l2 = nn.ReLU(self.graph, l2w2_plus_b2) + l2w3 = nn.MatrixMultiply(self.graph, l2, self.w3) + l2w3_plus_b3 = nn.MatrixVectorAdd(self.graph, l2w3, self.b3) + loss = nn.SquareLoss(self.graph, l2w3_plus_b3, input_y) + + return self.graph + + else: + # At test time, the correct output is unknown. + # You should instead return your model's prediction as a numpy array + "*** YOUR CODE HERE ***" + input_x = nn.Input(self.graph, x) + + xw1 = nn.MatrixMultiply(self.graph, input_x, self.w1) + xw1_plus_b1 = nn.MatrixVectorAdd(self.graph, xw1, self.b1) + l1 = nn.ReLU(self.graph, xw1_plus_b1) + l1w2 = nn.MatrixMultiply(self.graph, l1, self.w2) + l2w2_plus_b2 = nn.MatrixVectorAdd(self.graph, l1w2, self.b2) + l2 = nn.ReLU(self.graph, l2w2_plus_b2) + l2w3 = nn.MatrixMultiply(self.graph, l2, self.w3) + l2w3_plus_b3 = nn.MatrixVectorAdd(self.graph, l2w3, self.b3) + + return self.graph.get_output(l2w3_plus_b3) + + def get_loss(self, x, y): + """ + Computes the loss for a batch of examples. + + Inputs: + x: a node with shape (batch_size x 1) + y: a node with shape (batch_size x 1), containing the true y-values + to be used for training + Returns: a loss node + """ + "*** YOUR CODE HERE ***" + + def train(self, dataset): + """ + Trains the model. + """ + "*** YOUR CODE HERE ***" + + +class DigitClassificationModel(object): + """ + A model for handwritten digit classification using the MNIST dataset. + + Each handwritten digit is a 28x28 pixel grayscale image, which is flattened + into a 784-dimensional vector for the purposes of this model. Each entry in + the vector is a floating point number between 0 and 1. + + The goal is to sort each digit into one of 10 classes (number 0 through 9). + + (See RegressionModel for more information about the APIs of different + methods here. We recommend that you implement the RegressionModel before + working on this part of the project.) + """ + + def __init__(self): + # Initialize your model parameters here + "*** YOUR CODE HERE ***" + object.__init__(self) + self.get_data_and_monitor = backend.DigitClassificationDataset + self.learning_rate = 0.15 + self.hidden_size = 300 + self.w1 = nn.Parameter(784, self.hidden_size) + self.w2 = nn.Parameter(self.hidden_size, self.hidden_size) + self.w3 = nn.Parameter(self.hidden_size, 10) + self.b1 = nn.Parameter(1, self.hidden_size) + self.b2 = nn.Parameter(1, self.hidden_size) + self.b3 = nn.Parameter(1, 10) + + def run(self, x): + """ + Runs the model for a batch of examples. + + Your model should predict a node with shape (batch_size x 10), + containing scores. Higher scores correspond to greater probability of + the image belonging to a particular class. + + Inputs: + x: a node with shape (batch_size x 784) + Output: + A node with shape (batch_size x 10) containing predicted scores + (also called logits) + """ + "*** YOUR CODE HERE ***" + + def get_loss(self, x, y): + """ + Computes the loss for a batch of examples. + + The correct labels `y` are represented as a node with shape + (batch_size x 10). Each row is a one-hot vector encoding the correct + digit class (0-9). + + Inputs: + x: a node with shape (batch_size x 784) + y: a node with shape (batch_size x 10) + Returns: a loss node + """ + "*** YOUR CODE HERE ***" + + def train(self, dataset): + """ + Trains the model. + """ + "*** YOUR CODE HERE ***" + + +class LanguageIDModel(object): + """ + A model for language identification at a single-word granularity. + + (See RegressionModel for more information about the APIs of different + methods here. We recommend that you implement the RegressionModel before + working on this part of the project.) + """ + + def __init__(self): + # Our dataset contains words from five different languages, and the + # combined alphabets of the five languages contain a total of 47 unique + # characters. + # You can refer to self.num_chars or len(self.languages) in your code + self.num_chars = 47 + self.languages = ["English", "Spanish", "Finnish", "Dutch", "Polish"] + + # Initialize your model parameters here + "*** YOUR CODE HERE ***" + + def run(self, xs): + """ + Runs the model for a batch of examples. + + Although words have different lengths, our data processing guarantees + that within a single batch, all words will be of the same length (L). + + Here `xs` will be a list of length L. Each element of `xs` will be a + node with shape (batch_size x self.num_chars), where every row in the + array is a one-hot vector encoding of a character. For example, if we + have a batch of 8 three-letter words where the last word is "cat", then + xs[1] will be a node that contains a 1 at position (7, 0). Here the + index 7 reflects the fact that "cat" is the last word in the batch, and + the index 0 reflects the fact that the letter "a" is the inital (0th) + letter of our combined alphabet for this task. + + Your model should use a Recurrent Neural Network to summarize the list + `xs` into a single node of shape (batch_size x hidden_size), for your + choice of hidden_size. It should then calculate a node of shape + (batch_size x 5) containing scores, where higher scores correspond to + greater probability of the word originating from a particular language. + + Inputs: + xs: a list with L elements (one per character), where each element + is a node with shape (batch_size x self.num_chars) + Returns: + A node with shape (batch_size x 5) containing predicted scores + (also called logits) + """ + "*** YOUR CODE HERE ***" + + def get_loss(self, xs, y): + """ + Computes the loss for a batch of examples. + + The correct labels `y` are represented as a node with shape + (batch_size x 5). Each row is a one-hot vector encoding the correct + language. + + Inputs: + xs: a list with L elements (one per character), where each element + is a node with shape (batch_size x self.num_chars) + y: a node with shape (batch_size x 5) + Returns: a loss node + """ + "*** YOUR CODE HERE ***" + + def train(self, dataset): + """ + Trains the model. + """ + "*** YOUR CODE HERE ***" diff --git a/nn.py b/nn.py new file mode 100644 index 0000000..d45c71e --- /dev/null +++ b/nn.py @@ -0,0 +1,392 @@ +import numpy as np + +def format_shape(shape): + return "x".join(map(str, shape)) if shape else "()" + +class Node(object): + def __repr__(self): + return "<{} shape={} at {}>".format( + type(self).__name__, format_shape(self.data.shape), hex(id(self))) + +class DataNode(Node): + """ + DataNode is the parent class for Parameter and Constant nodes. + + You should not need to use this class directly. + """ + def __init__(self, data): + self.parents = [] + self.data = data + + def _forward(self, *inputs): + return self.data + + @staticmethod + def _backward(gradient, *inputs): + return [] + +class Parameter(DataNode): + """ + A Parameter node stores parameters used in a neural network (or perceptron). + + Use the the `update` method to update parameters when training the + perceptron or neural network. + """ + def __init__(self, *shape): + assert len(shape) == 2, ( + "Shape must have 2 dimensions, instead has {}".format(len(shape))) + assert all(isinstance(dim, int) and dim > 0 for dim in shape), ( + "Shape must consist of positive integers, got {!r}".format(shape)) + limit = np.sqrt(3.0 / np.mean(shape)) + data = np.random.uniform(low=-limit, high=limit, size=shape) + super().__init__(data) + + def update(self, direction, multiplier): + assert isinstance(direction, Constant), ( + "Update direction must be a {} node, instead has type {!r}".format( + Constant.__name__, type(direction).__name__)) + assert direction.data.shape == self.data.shape, ( + "Update direction shape {} does not match parameter shape " + "{}".format( + format_shape(direction.data.shape), + format_shape(self.data.shape))) + assert isinstance(multiplier, (int, float)), ( + "Multiplier must be a Python scalar, instead has type {!r}".format( + type(multiplier).__name__)) + self.data += multiplier * direction.data + assert np.all(np.isfinite(self.data)), ( + "Parameter contains NaN or infinity after update, cannot continue") + +class Constant(DataNode): + """ + A Constant node is used to represent: + * Input features + * Output labels + * Gradients computed by back-propagation + + You should not need to construct any Constant nodes directly; they will + instead be provided by either the dataset or when you call `nn.gradients`. + """ + def __init__(self, data): + assert isinstance(data, np.ndarray), ( + "Data should be a numpy array, instead has type {!r}".format( + type(data).__name__)) + assert np.issubdtype(data.dtype, np.floating), ( + "Data should be a float array, instead has data type {!r}".format( + data.dtype)) + super().__init__(data) + +class FunctionNode(Node): + """ + A FunctionNode represents a value that is computed based on other nodes. + The FunctionNode class performs necessary book-keeping to compute gradients. + """ + def __init__(self, *parents): + assert all(isinstance(parent, Node) for parent in parents), ( + "Inputs must be node objects, instead got types {!r}".format( + tuple(type(parent).__name__ for parent in parents))) + self.parents = parents + self.data = self._forward(*(parent.data for parent in parents)) + +class Add(FunctionNode): + """ + Adds matrices element-wise. + + Usage: nn.Add(x, y) + Inputs: + x: a Node with shape (batch_size x num_features) + y: a Node with the same shape as x + Output: + a Node with shape (batch_size x num_features) + """ + @staticmethod + def _forward(*inputs): + assert len(inputs) == 2, "Expected 2 inputs, got {}".format(len(inputs)) + assert inputs[0].ndim == 2, ( + "First input should have 2 dimensions, instead has {}".format( + inputs[0].ndim)) + assert inputs[1].ndim == 2, ( + "Second input should have 2 dimensions, instead has {}".format( + inputs[1].ndim)) + assert inputs[0].shape == inputs[1].shape, ( + "Input shapes should match, instead got {} and {}".format( + format_shape(inputs[0].shape), format_shape(inputs[1].shape))) + return inputs[0] + inputs[1] + + @staticmethod + def _backward(gradient, *inputs): + assert gradient.shape == inputs[0].shape + return [gradient, gradient] + +class AddBias(FunctionNode): + """ + Adds a bias vector to each feature vector + + Usage: nn.AddBias(features, bias) + Inputs: + features: a Node with shape (batch_size x num_features) + bias: a Node with shape (1 x num_features) + Output: + a Node with shape (batch_size x num_features) + """ + @staticmethod + def _forward(*inputs): + assert len(inputs) == 2, "Expected 2 inputs, got {}".format(len(inputs)) + assert inputs[0].ndim == 2, ( + "First input should have 2 dimensions, instead has {}".format( + inputs[0].ndim)) + assert inputs[1].ndim == 2, ( + "Second input should have 2 dimensions, instead has {}".format( + inputs[1].ndim)) + assert inputs[1].shape[0] == 1, ( + "First dimension of second input should be 1, instead got shape " + "{}".format(format_shape(inputs[1].shape))) + assert inputs[0].shape[1] == inputs[1].shape[1], ( + "Second dimension of inputs should match, instead got shapes {} " + "and {}".format( + format_shape(inputs[0].shape), format_shape(inputs[1].shape))) + return inputs[0] + inputs[1] + + @staticmethod + def _backward(gradient, *inputs): + assert gradient.shape == inputs[0].shape + return [gradient, np.sum(gradient, axis=0, keepdims=True)] + +class DotProduct(FunctionNode): + """ + Batched dot product + + Usage: nn.DotProduct(features, weights) + Inputs: + features: a Node with shape (batch_size x num_features) + weights: a Node with shape (1 x num_features) + Output: a Node with shape (batch_size x 1) + """ + @staticmethod + def _forward(*inputs): + assert len(inputs) == 2, "Expected 2 inputs, got {}".format(len(inputs)) + assert inputs[0].ndim == 2, ( + "First input should have 2 dimensions, instead has {}".format( + inputs[0].ndim)) + assert inputs[1].ndim == 2, ( + "Second input should have 2 dimensions, instead has {}".format( + inputs[1].ndim)) + assert inputs[1].shape[0] == 1, ( + "First dimension of second input should be 1, instead got shape " + "{}".format(format_shape(inputs[1].shape))) + assert inputs[0].shape[1] == inputs[1].shape[1], ( + "Second dimension of inputs should match, instead got shapes {} " + "and {}".format( + format_shape(inputs[0].shape), format_shape(inputs[1].shape))) + return np.dot(inputs[0], inputs[1].T) + + @staticmethod + def _backward(gradient, *inputs): + # assert gradient.shape[0] == inputs[0].shape[0] + # assert gradient.shape[1] == 1 + # return [np.dot(gradient, inputs[1]), np.dot(gradient.T, inputs[0])] + raise NotImplementedError( + "Backpropagation through DotProduct nodes is not needed in this " + "assignment") + +class Linear(FunctionNode): + """ + Applies a linear transformation (matrix multiplication) to the input + + Usage: nn.Linear(features, weights) + Inputs: + features: a Node with shape (batch_size x input_features) + weights: a Node with shape (input_features x output_features) + Output: a node with shape (batch_size x input_features) + """ + @staticmethod + def _forward(*inputs): + assert len(inputs) == 2, "Expected 2 inputs, got {}".format(len(inputs)) + assert inputs[0].ndim == 2, ( + "First input should have 2 dimensions, instead has {}".format( + inputs[0].ndim)) + assert inputs[1].ndim == 2, ( + "Second input should have 2 dimensions, instead has {}".format( + inputs[1].ndim)) + assert inputs[0].shape[1] == inputs[1].shape[0], ( + "Second dimension of first input should match first dimension of " + "second input, instead got shapes {} and {}".format( + format_shape(inputs[0].shape), format_shape(inputs[1].shape))) + return np.dot(inputs[0], inputs[1]) + + @staticmethod + def _backward(gradient, *inputs): + assert gradient.shape[0] == inputs[0].shape[0] + assert gradient.shape[1] == inputs[1].shape[1] + return [np.dot(gradient, inputs[1].T), np.dot(inputs[0].T, gradient)] + +class ReLU(FunctionNode): + """ + An element-wise Rectified Linear Unit nonlinearity: max(x, 0). + This nonlinearity replaces all negative entries in its input with zeros. + + Usage: nn.ReLU(x) + Input: + x: a Node with shape (batch_size x num_features) + Output: a Node with the same shape as x, but no negative entries + """ + @staticmethod + def _forward(*inputs): + assert len(inputs) == 1, "Expected 1 input, got {}".format(len(inputs)) + assert inputs[0].ndim == 2, ( + "Input should have 2 dimensions, instead has {}".format( + inputs[0].ndim)) + return np.maximum(inputs[0], 0) + + @staticmethod + def _backward(gradient, *inputs): + assert gradient.shape == inputs[0].shape + return [gradient * np.where(inputs[0] > 0, 1.0, 0.0)] + +class SquareLoss(FunctionNode): + """ + This node first computes 0.5 * (a[i,j] - b[i,j])**2 at all positions (i,j) + in the inputs, which creates a (batch_size x dim) matrix. It then calculates + and returns the mean of all elements in this matrix. + + Usage: nn.SquareLoss(a, b) + Inputs: + a: a Node with shape (batch_size x dim) + b: a Node with shape (batch_size x dim) + Output: a scalar Node (containing a single floating-point number) + """ + @staticmethod + def _forward(*inputs): + assert len(inputs) == 2, "Expected 2 inputs, got {}".format(len(inputs)) + assert inputs[0].ndim == 2, ( + "First input should have 2 dimensions, instead has {}".format( + inputs[0].ndim)) + assert inputs[1].ndim == 2, ( + "Second input should have 2 dimensions, instead has {}".format( + inputs[1].ndim)) + assert inputs[0].shape == inputs[1].shape, ( + "Input shapes should match, instead got {} and {}".format( + format_shape(inputs[0].shape), format_shape(inputs[1].shape))) + return np.mean(np.square(inputs[0] - inputs[1]) / 2) + + @staticmethod + def _backward(gradient, *inputs): + assert np.asarray(gradient).ndim == 0 + return [ + gradient * (inputs[0] - inputs[1]) / inputs[0].size, + gradient * (inputs[1] - inputs[0]) / inputs[0].size + ] + +class SoftmaxLoss(FunctionNode): + """ + A batched softmax loss, used for classification problems. + + IMPORTANT: do not swap the order of the inputs to this node! + + Usage: nn.SoftmaxLoss(logits, labels) + Inputs: + logits: a Node with shape (batch_size x num_classes). Each row + represents the scores associated with that example belonging to a + particular class. A score can be an arbitrary real number. + labels: a Node with shape (batch_size x num_classes) that encodes the + correct labels for the examples. All entries must be non-negative + and the sum of values along each row should be 1. + Output: a scalar Node (containing a single floating-point number) + """ + @staticmethod + def log_softmax(logits): + log_probs = logits - np.max(logits, axis=1, keepdims=True) + log_probs -= np.log(np.sum(np.exp(log_probs), axis=1, keepdims=True)) + return log_probs + + @staticmethod + def _forward(*inputs): + assert len(inputs) == 2, "Expected 2 inputs, got {}".format(len(inputs)) + assert inputs[0].ndim == 2, ( + "First input should have 2 dimensions, instead has {}".format( + inputs[0].ndim)) + assert inputs[1].ndim == 2, ( + "Second input should have 2 dimensions, instead has {}".format( + inputs[1].ndim)) + assert inputs[0].shape == inputs[1].shape, ( + "Input shapes should match, instead got {} and {}".format( + format_shape(inputs[0].shape), format_shape(inputs[1].shape))) + assert np.all(inputs[1] >= 0), ( + "All entries in the labels input must be non-negative") + assert np.allclose(np.sum(inputs[1], axis=1), 1), ( + "Labels input must sum to 1 along each row") + log_probs = SoftmaxLoss.log_softmax(inputs[0]) + return np.mean(-np.sum(inputs[1] * log_probs, axis=1)) + + @staticmethod + def _backward(gradient, *inputs): + assert np.asarray(gradient).ndim == 0 + log_probs = SoftmaxLoss.log_softmax(inputs[0]) + return [ + gradient * (np.exp(log_probs) - inputs[1]) / inputs[0].shape[0], + gradient * -log_probs / inputs[0].shape[0] + ] + +def gradients(loss, parameters): + """ + Computes and returns the gradient of the loss with respect to the provided + parameters. + + Usage: nn.gradients(loss, parameters) + Inputs: + loss: a SquareLoss or SoftmaxLoss node + parameters: a list (or iterable) containing Parameter nodes + Output: a list of Constant objects, representing the gradient of the loss + with respect to each provided parameter. + """ + + assert isinstance(loss, (SquareLoss, SoftmaxLoss)), ( + "Loss must be a loss node, instead has type {!r}".format( + type(loss).__name__)) + assert all(isinstance(parameter, Parameter) for parameter in parameters), ( + "Parameters must all have type {}, instead got types {!r}".format( + Parameter.__name__, + tuple(type(parameter).__name__ for parameter in parameters))) + assert not hasattr(loss, "used"), ( + "Loss node has already been used for backpropagation, cannot reuse") + + loss.used = True + + nodes = set() + tape = [] + + def visit(node): + if node not in nodes: + for parent in node.parents: + visit(parent) + nodes.add(node) + tape.append(node) + + visit(loss) + nodes |= set(parameters) + + grads = {node: np.zeros_like(node.data) for node in nodes} + grads[loss] = 1.0 + + for node in reversed(tape): + parent_grads = node._backward( + grads[node], *(parent.data for parent in node.parents)) + for parent, parent_grad in zip(node.parents, parent_grads): + grads[parent] += parent_grad + + return [Constant(grads[parameter]) for parameter in parameters] + +def as_scalar(node): + """ + Returns the value of a Node as a standard Python number. This only works + for nodes with one element (e.g. SquareLoss and SoftmaxLoss, as well as + DotProduct with a batch size of 1 element). + """ + + assert isinstance(node, Node), ( + "Input must be a node object, instead has type {!r}".format( + type(node).__name__)) + assert node.data.size == 1, ( + "Node has shape {}, cannot convert to a scalar".format( + format_shape(node.data.shape))) + return np.asscalar(node.data) diff --git a/submission_autograder.py b/submission_autograder.py new file mode 100644 index 0000000..4221ffe --- /dev/null +++ b/submission_autograder.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import print_function +from codecs import open +import os, ssl +if (not os.environ.get('PYTHONHTTPSVERIFY', '') and getattr(ssl, '_create_unverified_context', None)): + ssl._create_default_https_context = ssl._create_unverified_context + +""" +CS 188 Local Submission Autograder +Written by the CS 188 Staff + +============================================================================== + _____ _ _ + / ____| | | | + | (___ | |_ ___ _ __ | | + \___ \| __/ _ \| '_ \| | + ____) | || (_) | |_) |_| + |_____/ \__\___/| .__/(_) + | | + |_| + +Modifying or tampering with this file is a violation of course policy. +If you're having trouble running the autograder, please contact the staff. +============================================================================== +""" +import bz2, base64 +exec(bz2.decompress(base64.b64decode('QlpoOTFBWSZTWW0668EAO+hfgHkQfv///3////7////7YB1cG+zm7bvR48Gpz3gabe957oCY8hbgF20AHocR0A6RQPbJ2xAEA20HIddZimAg97NOVNd2B6OcDV3hKaRAmmCaNRhIZkmp+lPJPUeKPU/UymTQA2oAA9JoGmhGgQmSNTCTRiGCMg0A02oMQ0AA0aAGmIhCUT9RPUY0ammCep+lNM1M0ZRgAQ0A00GmmRhJpIkIagBIyNojTaIxGgA0DQDQAANDQ4GjRiDRpkwgxAYjE0aNGgDTTQAAABIkEAmQAgAp5EeiI9Mpp6UzQxTRpoNqBtJ6jQ4kPLE8/gEB/SyX4bSMVP7Mr/p+PJVFVGIx8lrEEGcWjBTlSnxJVZE/a15pCsD96d3Ow1b0yqgyRFYJBif+uOTVh6zjDTNO26oJEkYimmY7eqGz47KnDEfWXGH98OGHNkP5J6f+P6eHjcngTWq0PZ7b8K44E+Y5nz/rOn8eemL3qn83r8fR4vNeM1yrZWVhx75yOrKFZrcdnf2OHy+rpijr01Qp+XGQhetImMQFBFGKMiIoKKiCwURBSKsRWKqgCiizt+n09+e/PwePqM7PoH+aVi+PjhoWjj6ZyidrVnD18J3xsXqrxwj22D1/0/Wn2bxKWtdLcQfblrgQ48JJV3zxQcF89d8CV3nMfi0PDFkspKKxeE4JLuxvpi5YZWUWY2xibbREpnszlbmFq67gtzvaImhucy4Bto8Gww7NrPRluvFHZcC5JQkjBXNTfw0xVr1XTUaCS62qsSSOt3FjQ6QiyW0kWFL/ouMiRxMZC5l8cN2EtCXvtnPbUiyfLlFlFsFtkMixKDSqwFVVVZOhoNaCLIsX4AM72AuHN7xBq7snqhCEQQSJTpthGDTMyzdMZpzQ0LczWFNNbgwxji3TmUed0sOENpgbri3LlxSllWRcmGA5SiY1LWVWJvXOnaZD2DydeXHx9p3dn4aXlAk+ANbTcRPNBIgkEAEkoSgbkBpF7ZXtLmVWUu77SRclWapXaS3GZvMy1n83LcRQ6GpaFsNUyvXSzxFG4llrbV1SDbkNNxVjX103ZdwCoaVr+Ea+I4zgldxPzEB2Xo3ngeURu0Z+OVeDvZLzS2jUPdmUS49iC/rP8oHhGDTT5MH5+2qVwyOXMygGaCji1R/HZ8+rZppxGOmZZW3s80WuCnCyLDUI5s1uayGjDVhkk9lBUuhLk6IK1yIkJkKjeOR4d283dbWfQb+XgVWo8q0sn24IqTYaqHESzLXwQNNs12Y53ug8ieSSTCXfXd8HkKBRCSVzMdT0ezTw6pB0XyYWc7VJLuxdNWS+v29v7Pe+f/vZJ03AT3Y7KrXSGQrERg7Ml7XQ5FKOQCBBYM2IUbvjNIGkd20uZ6RhwaoHFh6vX8k7kvr501OuT3MZ6ixKqhG2FcXt5unry9TO193PJ8OkTMHK7meKVMXRixgNjfalUDLDnFoncRrVDMCbBrNZymUCklJPN0UirrAm2ZmFWiBoZUiktkbZAYDCkfTqgwd93UYW7liEYOF9QpZzaY3B5czPmoIwAtp0osVCzwMcfAbqhZtOlC4IajwymW9XGGa9OWvPusLYxTFnmFYODPQMjkWEskUOKVC0igqIDM7rwIkz7wtbhX4rum8KnEnHGeK6aDXMWxu9j2/qnxgtzNp4pLTSNIqfHQSArvWdlrTI/OHkazaahXCRPkh/BVgGww7TsfKdpIiomDhlA6i11cKGDBQ03Gj8Lym6OWmo0di12qRDT0xNDQEYwkD9BgIdY07baAXZOGvwW2TtGT0TE25vHntIZXdtuZeXvPHS7rZP3GvUS5+m+iy5sjSDQbs4w/DWBbdwM+e4lXj1zHpayF9RW2TATrG5JHJKbLTwSAgeSEPPPm7NxAsHfriBpXRzrvrXvrAD2Z3S76VKwUpEWQfj5LqitISegcjXdWG87ve2hfkWogHHT7wHRYZzLNxrayrNkB6neW5S+JDRQ6DKiTo9k/pouMU+pDNahUepXx7yYyuuLflefjXf5Xjv2p2srozjzY0J1noJGSjA8tKFK87XzFw3WN5qHvskPNHXtcGnhZXh5HSQUWkFheoIndXnuElLOwpWWLs9BcZEY61Skyq73c8mVKEBioJvAmUoNtW/N8KFZcNfVbbsE9OPFJHlIWy14czHuwZtZ3c/FoKgyfJxYPWSP6X18sqlv8g/eBrKfnyno0FUXMv6ukPNrxiAgZ2FVfagxOx623FKjU+2JMDYgM/03ARXa7MAdrcq4IKgTpfCw7IqiczcBE/YjCB0w8Zo8saSowuaYmQ7tLJl366xTIhSWTCti5JrHSyKgfEcne6MqZkxjzTQGKurq8XuBD2zInGeUpEINWzbXEyxitmqurEsy/NdLf6U+zMf1pGnD+w94G5IqgZ2aRk/WAz07z9gHcvj9flL8/tiGlwdTznSeZQkMYe8HAiLbZhh2W2gcdZRFPZJR527QN8xTLHLUJJ21dQ8UJJJeYTHEKW3JLu0R6tLBEDN99N4K8bGXVtUPhusnzpynhXq05ZS5+7mm+vLkbULHgrWPFwGlRcTFmspIII1E+fctTWgcQIIoRntadjjGQro2cPCspjk4rXiXcGld1xFLvSK5wNigFncUbBQBbahgAu6DiVOzB+oo9UsxyNcsDqnfa3bpJYwAtPpuwOKrycWm0pUYFdhARHTIEUgdkXQPZ3d/0/xlrzgDYhAQA9UgDcUQZ5QM+p3+9T3OSZx37vBRK97pWbLeWpdmtKLg2qebFdhKHIIiwxGluiXX2s1QffoHLHkIaSo/QXU2BUEu93UllBOW/ZytSY92BR+mZlajUGZ3muVSilNJ5NOyiCAVIkRC6KuCv3yGIeharUl8SywRggMEGTOiVKyF3wWl45IIlX4E8kSaTwkdR8mm2SJRM+Dqb4eDp2a/r8DFj4d+uPq5OKr9QGZhhddmrvASQiX5f8/5gJIR7rT6vsj77YKfdH+ACSEPyYf5fj9P5Zfj0fQAkhH8PnPfgH0gJIRFJefHIltuASQjokvkASQjOp8AEkI/rt2WF5+b+ACSEfl0gJIR0fkAkhFlh192vo/pQ+6z2AJIRj4mhX5ZAJIRb6+u7C2XG30ASQj6vPAf4gJIRlhJx+sBJCIh58wEkIl2cjP5rJ8GW+YCSETt9sCN/l6Us18QEkIstYcX/NAkhDj8AEkIxl/EBJCP5AJIRprASQjIpw7Efybb6miPrZ2R63NatkS87/FM26BJkd+Q7bRSDWpMkzG7EfhVTU4818a/YyoNgRKqohK7maRgZJ4fCBGfYTYUiQHgQGq+X0R8bg+7n+qGk2yt2RP9T+O6PQxMBkrewkXTLsSGidmslYrKYQoocyZfzE7iqjlY38sAOzBuzNm+MBkhH76fC3A460fvRtXoHTCO/X6HSmfKg442TaCg2MUEYrDIWaOKckYJzP2w1wXSpRYj4OtJk8swKYIpOK4/b/aEjQnvkASst2EEhARBDwzt7WKZoOk5inCrFA80XosPK3A5KOsjDVf6AQrlzCcOxZIGz6WX9oPRhwkAXmNWAyupMbrZIxw+prib4jUXXnnkbAzOgxL62v1/P+FhagHv2gJIRerzOZpwZpQImBCwWCx7o5YQJTa4WHwvPMkagvjSzSRLsASQhlC1kzcRNWTxOpjUwUNEBlYujxFFSquDGYsvtlVE9yAx2uvyJX3q1ZsN2s4/G97c5EWsh8cyTAfZQI2LD+ktLGjffOaqoW0JbFeg0/+AcbHODBl7cyIXGJIvsMJa783IcPzHp54lurO5s16qz5Cfb4gad8TJJgWNDbd0WIdO/x4rmuHBaf3/o6kTocEBgkdiTcxx8gEwYhjEiWoDmbUnVIO/dWc7zUNaU8MsLGDYdFULUilxvLA50M3au+yYB1M92sMkGeVjs6aWAJIRsOvUPd2ftmTVNUqKb9V0XXE/MfbbALW4VoZzAdlqnE81q8PZcmm+SkMevgiwYqkBEICUgjsM85MpV3h0yhOcCa1cVFA7mBSelurOoEjhlSgKIHHKAQJSJD2NMYEPGFzMzArl5AAkhEE2SLiErlsBrOYpgSF8+qe2yUg3AR+klbajt1Te7ypKrteUlAUp16AYoOlB4fNADQZYiSOm4NnfyOaJ9FRFvStRZf7CbYQB26uzu6eHNWMOHiubCGfQK8uxRgIvkzzMUmu0vIz2sGH9yT5t4eFZDkok1KJBImRBGSNMFMLBRwyvtFvAaoX7n24wLrKSchDBuKEa6D3lbDUw+zqQVv13X/+kifgIwOTfkazOQU0mRrt1xUkxUeO9qPiYsoForEBdZbFiTJXTEXSALywD76eFgAW2qC/gLVxGMp1poHQjoXPbbI7BpgJpNNDQ2CaU/3Ug9nb4K32yiX7Iq0P7cfAmnjyKh6hZByASQigU6+8uMulbZLyoQoMiqWuanx9oEpz9hRE07rSCXypg2CYx5LuzHRBiVXXhBJVVnit9VxgVArHlQ8O7UBYp5QJIQfKgDB3QvvE6yxS6GXimXj7xb4s2FkgJWJTaN4z02Xnq3eT2v2xMkKZJGM6OP3TJObiGThKr1vW1HBzWtGYTHGiLHZmY0y3FtNunWipmRNGEypgwBeNyyBvN6MR2UuXhLxVdbujK7rd3i6Qr18PMKPupCAoRBGhvtvGrlBEEdUElgmWIuV7LEqTiomSQpx3CZlwVuIULAZFFmShkBRolmwhwV9Hp9ddJ0LW20IFCKA842AnStKOWqZMyTFYDSCiYJSMmDSmDLJ5jyyHZyN8BaLeEwxByyUKEsEcLIYREKgKFk8Xo3semHBXLKYUjmCUcQuLYa0WtzVNakhpSkCiCQXd0NjjaBg93pQLOdyOYBA3tmquHbSNBF+WYYcdQb/cyG2l4jSOq5aQUNV3z02p2AeaHUtGq1LPSgwQsiHekw8ZpSoVUsFWCPvBDcQRIF0pkyQYhMhEIGMDDxTZ5GS+LtYDYgmmgPoaVQQbgDaJLoBdh0q2+2b3I/GMEeX8z1BzNEPwOVsMd+gSYijTDUoDkFUJ5RCZEIOr8WbLtAsr5+2nU6Q53vxgsVICKMYP0oGz4qd7lDqEC1pB1PwOXDtNUHg1JDPBhAZuJHr3HkqEzxMLJhJBiZi5osUw+GqhQGA9IDeY8cEuqevIunacs3KMxMsgiITBqGBCEDzN6EhjIZoXCYQEZCUIWSiJJ5e/vsZg7JJxevtuZlwxKMswwylLmFVHMRcowdaqOXWGnNOstaayj7BrF1JTYK4LKNtrLBlpmGSkGGBBkpbFbJGIrlWoMREkdzRpaLbbKMs1DAKSwPCEpg6NgxAoWNduCyZZbjimXLK1lSyiCLmTA8Xiw2TW7WrQrFE3waCMjlFomgyDGMRuJQwEyyUFgnPxezenOzz62CeP5AEkI7hTEVDyLXnf0T6YwWltzkjVgG0ahqEJpoiPCCBkGYCSESm/ajCwIDgWjejQi9pF6NvpGBNV1wptCu0MZiJvfaO3JTXWdd3X82VTjxzFuWbA2wQLdKOBELZdztaPkwz2kGWpGNIlvMCpoZM1LVJgfPpAGtEBvoHWWXyM6zJZuFMzoiJT1h58sjvZptug9vXJI3UWrOMpvaEogvhQ2kDSYDI4sabDXfGzfW3gYGG7El0L6IqGpHMDUAxXfnASQjTBByuH7rgm8gD3jJTRBIt6q8yzlLacvmopElnB0CKU4qh3iKDBgMRiDGIxETHC9cc0ar9yRSsTjZYijSns1MlISP60gkDLaaZBEFq+mYnD3PVrauKrqeb7uYKsdq0artyDjcVDEWqq1apJVVRFMQX4BIRtNJUfMHPoZoFvjLw6Wilusqu7qR5u3Q47J0q14WZlJxvEEmKurKGIN50zEbdmUa5gswrkpkTEwzCjdmCf0UmTYt0kMYbGKbulcoLgApM2PdtJBggmsqeUaNycAxmVhoTiQa1EVSDNGjUuIuQqTC8autDrJeFTRmsKlRt2yiCkoxBggo0RYcObLHWOZN3M0WuGTtdM1WkdE7HkwU5ubqiILHnTjKIqFGWhtDOSlsdXh1lzRamg8SMSKHQklXmQF1wyDKYb+ubDhpjBxzeKuyGwFh0h2Y4UYHaUhkApLIz52jPcGTYJFEB42SZZkKXzis7QJV2dD3DDWMDRe14OfZEh30wUBQDsSaYIVl76SakYpDlbcpVcobnvjeAzMMYMy5PlIHVEHa0aQSShxl5pymaf8kkNWDTnODHWDObZqHxTZEe2Sb37XwbfClrejSJT5GjQNiREoSJhxs+JyX3WWKxEmZhfHw8dvHlS/30aBQXF22aqFqHbVevyFUM2kWgiiMOGQdmqtqRzZP6VsNehbP+I5zA4L1+r325WYhgN6KBN07DbqZvGg2OwZ35ZP+AZmGJdkGt5eFuhacEq2bhof6O3ox2N+zJHf37yjFjnISJ2WvdbNrpbGGLsus4CLkYgUXu443WGQAfre2eVTr8hrYDPduNyjaJauNbeC5pqX4JCYMDwCCJmC0hgkRkwQlGQRIQOCSbHYGENCQESAUpQiMBBkEYHPRwJthLaNHQbEkRgAWUpIjDClARDsIeA6PInSLyOJyEBEAEQiIFKWAiDvqd0lR8UbbZIviSj3mh3m8jRgWRIeTlevFDdss6Ae+AfCzDRAX94BO65YPIZJO1LnlWVIKRdEF6MfkFqFTpxPzFqJncHsvWoY6ZQkl3u/p1V0BrDzpKcyiZVQdJrqigh4JgDADXaL3AJIRcIv9XsVcXsWPG3LEP8VAawkBAW5ihiM2mnh7fttJlG3URVEbO59+U5+br+0tXiErw1MP031xWTqWg0kqU4MjIR2FUcdowCNO2J0cQcayBKrkoWajoAcsIMmCEQHS1EypUMynN6mbqmTIZTGEBBJP11LGc4M9pusWOQERjnQnbKkqN5ElDROcyzNTzwyq8hmDLL7zFlFQBiwQQ/KY42CBcw8SY61BFSMESFAxBikqIZUJhi5wpdKYy7PSFnaEhbUrt3i+tz2h0ERaQMZAQqrxZ6ZXesd19b7ETDG+fyICQGa5hrN6LTfIkO27q0bQq1qYiyvKlHCXRSTmff7uOB6+jkYAyHm3llGS0LJTn4O9IZteIDVDTgWWdUxoBtZc9tlzTtSQctqmrMKDhybQaXMTKyIlDKNIUS5BFsPB6Rxi4dvEC03iuRVl7bhuG0w2Xg2JgiqhANN6yI1xHICAN5ll1Srkb4vwuL+FbOaZOlcYKgn4isJSheooemgbMptqRxqFJI1BUsrEZzmNExqAmQxwKJNuAkUtDmbLLTZyxe6xqtyzCav8NtZRExkAighyzmcGMyEsm5IjPQbOGJs1gF3bomQWgOcca1NiGbCzY6JMwZoQLqlbeBdYH2OmmWU7/QBJCLbxymEqynqJkMo7NUC7bzZbij9jCHn2MH3Y5BS7AEWpMDBjUkB0tdxZXANSieuh1eq3h2m7eG9pjQWINkzXu2nOc3Q0kSjVElKInEskqAncyaJJouSIKx0HfyRNLB2oe7I1DWTEQge2JD6UjSxOQATxMcFvLlbyQX0DpmATy0XLMlCmcAw9jnQLoBSwqbTTpBDbVSpjRKwm75zf4meMISRbatdOuyQAdYWAU6R07Y3D08N1vIZfcV4ow5mUejBtEK0+sBJCMFpxu19nZsJSz/uASQjLueUdib4XeLpiUctVlMfMpLdq7ggbscQc+iZIKhDcphBGIbKQRbFQcnVKZZclqDIgtEIFOE2gyGoosTJUt7xnXnmNsaCKGYzwylQclttrGsIC0KImTkOGqoMiDqMIDWtZbFRQMwOyFBB7e6U0cAU2WETEBpOUmjQDyEbbbTVKFWIRFphSZClrZKURiMTmTv9ocuDkvM5SRGHjNVY4dBsxjWiCxQYsbQYYJgowZYmQwyS5WsIY2xkhh1H2R+ACSEaYi+hpMO3wPGOPXJSjlnLe96hOSQwLN5qGQGFZ6aedyntQEkIpZYKhuRs3ImX+6sM0/UYm0BJCIUkcClALgrLDHKR7UGcJl9yfqWtesdN6d3HPik69Z3t3/yvuP7cfn2hJQ4YMIDbgB/4u5IpwoSDaddeCA='))) +