getting right starter files
This commit is contained in:
Arjun Patel
+79
-68
@@ -4,7 +4,7 @@
|
||||
# educational purposes provided that (1) you do not distribute or publish
|
||||
# solutions, (2) you retain this notice, and (3) you provide clear
|
||||
# attribution to UC Berkeley, including a link to http://ai.berkeley.edu.
|
||||
#
|
||||
#
|
||||
# Attribution Information: The Pacman AI projects were developed at UC Berkeley.
|
||||
# The core projects and autograders were primarily created by John DeNero
|
||||
# (denero@cs.berkeley.edu) and Dan Klein (klein@cs.berkeley.edu).
|
||||
@@ -22,63 +22,65 @@ import sys
|
||||
import projectParams
|
||||
import random
|
||||
random.seed(0)
|
||||
try:
|
||||
try:
|
||||
from pacman import GameState
|
||||
except:
|
||||
pass
|
||||
|
||||
# register arguments and set default values
|
||||
def readCommand(argv):
|
||||
parser = optparse.OptionParser(description = 'Run public tests on student code')
|
||||
parser.set_defaults(generateSolutions=False, edxOutput=False, gsOutput=False, muteOutput=False, printTestCase=False, noGraphics=False)
|
||||
parser = optparse.OptionParser(
|
||||
description='Run public tests on student code')
|
||||
parser.set_defaults(generateSolutions=False, edxOutput=False, gsOutput=False,
|
||||
muteOutput=False, printTestCase=False, noGraphics=False)
|
||||
parser.add_option('--test-directory',
|
||||
dest = 'testRoot',
|
||||
default = 'test_cases',
|
||||
help = 'Root test directory which contains subdirectories corresponding to each question')
|
||||
dest='testRoot',
|
||||
default='test_cases',
|
||||
help='Root test directory which contains subdirectories corresponding to each question')
|
||||
parser.add_option('--student-code',
|
||||
dest = 'studentCode',
|
||||
default = projectParams.STUDENT_CODE_DEFAULT,
|
||||
help = 'comma separated list of student code files')
|
||||
dest='studentCode',
|
||||
default=projectParams.STUDENT_CODE_DEFAULT,
|
||||
help='comma separated list of student code files')
|
||||
parser.add_option('--code-directory',
|
||||
dest = 'codeRoot',
|
||||
default = "",
|
||||
help = 'Root directory containing the student and testClass code')
|
||||
dest='codeRoot',
|
||||
default="",
|
||||
help='Root directory containing the student and testClass code')
|
||||
parser.add_option('--test-case-code',
|
||||
dest = 'testCaseCode',
|
||||
default = projectParams.PROJECT_TEST_CLASSES,
|
||||
help = 'class containing testClass classes for this project')
|
||||
dest='testCaseCode',
|
||||
default=projectParams.PROJECT_TEST_CLASSES,
|
||||
help='class containing testClass classes for this project')
|
||||
parser.add_option('--generate-solutions',
|
||||
dest = 'generateSolutions',
|
||||
action = 'store_true',
|
||||
help = 'Write solutions generated to .solution file')
|
||||
dest='generateSolutions',
|
||||
action='store_true',
|
||||
help='Write solutions generated to .solution file')
|
||||
parser.add_option('--edx-output',
|
||||
dest = 'edxOutput',
|
||||
action = 'store_true',
|
||||
help = 'Generate edX output files')
|
||||
dest='edxOutput',
|
||||
action='store_true',
|
||||
help='Generate edX output files')
|
||||
parser.add_option('--gradescope-output',
|
||||
dest = 'gsOutput',
|
||||
action = 'store_true',
|
||||
help = 'Generate GradeScope output files')
|
||||
dest='gsOutput',
|
||||
action='store_true',
|
||||
help='Generate GradeScope output files')
|
||||
parser.add_option('--mute',
|
||||
dest = 'muteOutput',
|
||||
action = 'store_true',
|
||||
help = 'Mute output from executing tests')
|
||||
dest='muteOutput',
|
||||
action='store_true',
|
||||
help='Mute output from executing tests')
|
||||
parser.add_option('--print-tests', '-p',
|
||||
dest = 'printTestCase',
|
||||
action = 'store_true',
|
||||
help = 'Print each test case before running them.')
|
||||
dest='printTestCase',
|
||||
action='store_true',
|
||||
help='Print each test case before running them.')
|
||||
parser.add_option('--test', '-t',
|
||||
dest = 'runTest',
|
||||
default = None,
|
||||
help = 'Run one particular test. Relative to test root.')
|
||||
dest='runTest',
|
||||
default=None,
|
||||
help='Run one particular test. Relative to test root.')
|
||||
parser.add_option('--question', '-q',
|
||||
dest = 'gradeQuestion',
|
||||
default = None,
|
||||
help = 'Grade one particular question.')
|
||||
dest='gradeQuestion',
|
||||
default=None,
|
||||
help='Grade one particular question.')
|
||||
parser.add_option('--no-graphics',
|
||||
dest = 'noGraphics',
|
||||
action = 'store_true',
|
||||
help = 'No graphics display for pacman games.')
|
||||
dest='noGraphics',
|
||||
action='store_true',
|
||||
help='No graphics display for pacman games.')
|
||||
(options, args) = parser.parse_args(argv)
|
||||
return options
|
||||
|
||||
@@ -107,14 +109,15 @@ def setModuleName(module, filename):
|
||||
|
||||
for i in dir(module):
|
||||
o = getattr(module, i)
|
||||
if hasattr(o, '__file__'): continue
|
||||
if hasattr(o, '__file__'):
|
||||
continue
|
||||
|
||||
if type(o) == functionType:
|
||||
setattr(o, '__file__', filename)
|
||||
elif type(o) == classType:
|
||||
setattr(o, '__file__', filename)
|
||||
# TODO: assign member __file__'s?
|
||||
#print(i, type(o))
|
||||
# print i, type(o)
|
||||
|
||||
|
||||
#from cStringIO import StringIO
|
||||
@@ -126,12 +129,14 @@ def loadModuleString(moduleSource):
|
||||
#f = StringIO(moduleCodeDict[k])
|
||||
#tmp = imp.load_module(k, f, k, (".py", "r", imp.PY_SOURCE))
|
||||
tmp = imp.new_module(k)
|
||||
exec(moduleCodeDict[k] in tmp.__dict__)
|
||||
exec(moduleCodeDict[k], tmp.__dict__)
|
||||
setModuleName(tmp, k)
|
||||
return tmp
|
||||
|
||||
|
||||
import py_compile
|
||||
|
||||
|
||||
def loadModuleFile(moduleName, filePath):
|
||||
with open(filePath, 'r') as f:
|
||||
return imp.load_module(moduleName, f, "%s.py" % moduleName, (".py", "r", imp.PY_SOURCE))
|
||||
@@ -149,8 +154,8 @@ def readFile(path, root=""):
|
||||
|
||||
# TODO: use these
|
||||
ERROR_HINT_MAP = {
|
||||
'q1': {
|
||||
"<type 'exceptions.IndexError'>": """
|
||||
'q1': {
|
||||
"<type 'exceptions.IndexError'>": """
|
||||
We noticed that your project threw an IndexError on q1.
|
||||
While many things may cause this, it may have been from
|
||||
assuming a certain number of successors from a state space
|
||||
@@ -158,9 +163,9 @@ ERROR_HINT_MAP = {
|
||||
state. Try making your code more general (no hardcoded indices)
|
||||
and submit again!
|
||||
"""
|
||||
},
|
||||
'q3': {
|
||||
"<type 'exceptions.AttributeError'>": """
|
||||
},
|
||||
'q3': {
|
||||
"<type 'exceptions.AttributeError'>": """
|
||||
We noticed that your project threw an AttributeError on q3.
|
||||
While many things may cause this, it may have been from assuming
|
||||
a certain size or structure to the state space. For example, if you have
|
||||
@@ -169,11 +174,12 @@ ERROR_HINT_MAP = {
|
||||
making your code more general and submit again!
|
||||
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
import pprint
|
||||
|
||||
|
||||
def splitStrings(d):
|
||||
d2 = dict(d)
|
||||
for k in d:
|
||||
@@ -215,14 +221,15 @@ def runTest(testName, moduleDict, printTestCase=False, display=None):
|
||||
printTest(testDict, solutionDict)
|
||||
|
||||
# This is a fragile hack to create a stub grades object
|
||||
grades = grading.Grades(projectParams.PROJECT_NAME, [(None,0)])
|
||||
grades = grading.Grades(projectParams.PROJECT_NAME, [(None, 0)])
|
||||
testCase.execute(grades, moduleDict, solutionDict)
|
||||
|
||||
|
||||
# returns all the tests you need to run in order to run question
|
||||
def getDepends(testParser, testRoot, question):
|
||||
allDeps = [question]
|
||||
questionDict = testParser.TestParser(os.path.join(testRoot, question, 'CONFIG')).parse()
|
||||
questionDict = testParser.TestParser(
|
||||
os.path.join(testRoot, question, 'CONFIG')).parse()
|
||||
if 'depends' in questionDict:
|
||||
depends = questionDict['depends'].split()
|
||||
for d in depends:
|
||||
@@ -232,11 +239,13 @@ def getDepends(testParser, testRoot, question):
|
||||
|
||||
# get list of questions to grade
|
||||
def getTestSubdirs(testParser, testRoot, questionToGrade):
|
||||
problemDict = testParser.TestParser(os.path.join(testRoot, 'CONFIG')).parse()
|
||||
problemDict = testParser.TestParser(
|
||||
os.path.join(testRoot, 'CONFIG')).parse()
|
||||
if questionToGrade != None:
|
||||
questions = getDepends(testParser, testRoot, questionToGrade)
|
||||
if len(questions) > 1:
|
||||
print('Note: due to dependencies, the following tests will be run: %s' % ' '.join(questions))
|
||||
print('Note: due to dependencies, the following tests will be run: %s' %
|
||||
' '.join(questions))
|
||||
return questions
|
||||
if 'order' in problemDict:
|
||||
return problemDict['order'].split()
|
||||
@@ -246,7 +255,7 @@ def getTestSubdirs(testParser, testRoot, questionToGrade):
|
||||
# evaluate student code
|
||||
def evaluate(generateSolutions, testRoot, moduleDict, exceptionMap=ERROR_HINT_MAP,
|
||||
edxOutput=False, muteOutput=False, gsOutput=False,
|
||||
printTestCase=False, questionToGrade=None, display=None):
|
||||
printTestCase=False, questionToGrade=None, display=None):
|
||||
# imports of testbench code. note that the testClasses import must follow
|
||||
# the import of student code due to dependencies
|
||||
import testParser
|
||||
@@ -263,14 +272,16 @@ def evaluate(generateSolutions, testRoot, moduleDict, exceptionMap=ERROR_HINT_MA
|
||||
continue
|
||||
|
||||
# create a question object
|
||||
questionDict = testParser.TestParser(os.path.join(subdir_path, 'CONFIG')).parse()
|
||||
questionDict = testParser.TestParser(
|
||||
os.path.join(subdir_path, 'CONFIG')).parse()
|
||||
questionClass = getattr(testClasses, questionDict['class'])
|
||||
question = questionClass(questionDict, display)
|
||||
questionDicts[q] = questionDict
|
||||
|
||||
# load test cases into question
|
||||
tests = filter(lambda t: re.match('[^#~.].*\.test\Z', t), os.listdir(subdir_path))
|
||||
tests = map(lambda t: re.match('(.*)\.test\Z', t).group(1), tests)
|
||||
tests = [t for t in os.listdir(
|
||||
subdir_path) if re.match('[^#~.].*\.test\Z', t)]
|
||||
tests = [re.match('(.*)\.test\Z', t).group(1) for t in tests]
|
||||
for t in sorted(tests):
|
||||
test_file = os.path.join(subdir_path, '%s.test' % t)
|
||||
solution_file = os.path.join(subdir_path, '%s.solution' % t)
|
||||
@@ -281,6 +292,7 @@ def evaluate(generateSolutions, testRoot, moduleDict, exceptionMap=ERROR_HINT_MA
|
||||
testDict['test_out_file'] = test_out_file
|
||||
testClass = getattr(projectTestClasses, testDict['class'])
|
||||
testCase = testClass(question, testDict)
|
||||
|
||||
def makefun(testCase, solution_file):
|
||||
if generateSolutions:
|
||||
# write solution file to disk
|
||||
@@ -308,11 +320,10 @@ def evaluate(generateSolutions, testRoot, moduleDict, exceptionMap=ERROR_HINT_MA
|
||||
for prereq in questionDicts[q].get('depends', '').split():
|
||||
grades.addPrereq(q, prereq)
|
||||
|
||||
grades.grade(sys.modules[__name__], bonusPic = projectParams.BONUS_PIC)
|
||||
grades.grade(sys.modules[__name__], bonusPic=projectParams.BONUS_PIC)
|
||||
return grades.points
|
||||
|
||||
|
||||
|
||||
def getDisplay(graphicsByDefault, options=None):
|
||||
graphics = graphicsByDefault
|
||||
if options is not None and options.noGraphics:
|
||||
@@ -327,8 +338,6 @@ def getDisplay(graphicsByDefault, options=None):
|
||||
return textDisplay.NullGraphics()
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
options = readCommand(sys.argv)
|
||||
if options.generateSolutions:
|
||||
@@ -344,15 +353,17 @@ if __name__ == '__main__':
|
||||
moduleDict = {}
|
||||
for cp in codePaths:
|
||||
moduleName = re.match('.*?([^/]*)\.py', cp).group(1)
|
||||
moduleDict[moduleName] = loadModuleFile(moduleName, os.path.join(options.codeRoot, cp))
|
||||
moduleDict[moduleName] = loadModuleFile(
|
||||
moduleName, os.path.join(options.codeRoot, cp))
|
||||
moduleName = re.match('.*?([^/]*)\.py', options.testCaseCode).group(1)
|
||||
moduleDict['projectTestClasses'] = loadModuleFile(moduleName, os.path.join(options.codeRoot, options.testCaseCode))
|
||||
|
||||
moduleDict['projectTestClasses'] = loadModuleFile(
|
||||
moduleName, os.path.join(options.codeRoot, options.testCaseCode))
|
||||
|
||||
if options.runTest != None:
|
||||
runTest(options.runTest, moduleDict, printTestCase=options.printTestCase, display=getDisplay(True, options))
|
||||
runTest(options.runTest, moduleDict, printTestCase=options.printTestCase,
|
||||
display=getDisplay(True, options))
|
||||
else:
|
||||
evaluate(options.generateSolutions, options.testRoot, moduleDict,
|
||||
gsOutput=options.gsOutput,
|
||||
edxOutput=options.edxOutput, muteOutput=options.muteOutput, printTestCase=options.printTestCase,
|
||||
questionToGrade=options.gradeQuestion, display=getDisplay(options.gradeQuestion!=None, options))
|
||||
gsOutput=options.gsOutput,
|
||||
edxOutput=options.edxOutput, muteOutput=options.muteOutput, printTestCase=options.printTestCase,
|
||||
questionToGrade=options.gradeQuestion, display=getDisplay(options.gradeQuestion != None, options))
|
||||
|
||||
Reference in New Issue
Block a user