fix: Resolves broken tokenization and codejail limits setting

Fixes incompatible tokenization of user submitted code when running under Python 3.x. This is used to add checks to ensure that students are using specific keywords or language features.

Fix bug when using codejail limits in xqueue-watcher

When the `limits` object is set in the codejail configuration block it causes the command used for launching a watcher process to be invalid. This is because there was a variable name used in a for loop that shadowed a variable from an outer scope that is then returned from the `enable_codejail` method that is used as an argument for the execution.

Fixed invalid conversion of conditional dictionary iterator

The PR that removed the usage of `six` failed to wrap the conditional `environment or {}` in parentheses before calling `.items()` on the resulting value. This would lead to trying to iterate over _either_ `environment` or `{}.items()` which is not the correct behavior.

Author: shaidar

grader_support/gradelib.py

@@ -3,7 +3,8 @@
 import random
 import re
 import sys
-import tokenize
+from tokenize import tokenize, COMMENT, STRING
+from io import BytesIO
 
 import six
 # the run library should overwrite this with a particular random seed for the test.
@@ -238,11 +239,7 @@ def _tokens(code):
     A wrapper around tokenize.generate_tokens.
     """
     # Protect against pathological inputs: http://bugs.python.org/issue16152
-    code = code.rstrip() + "\n"
-    if isinstance(code, str):
-        code = code.encode('utf8')
-    code = "# coding: utf8\n" + code
-    toks = tokenize.generate_tokens(six.BytesIO(code).readline)
+    toks = tokenize(BytesIO(code.encode('utf-8')).readline)
     return toks
 
 def _count_tokens(code, string):
@@ -253,7 +250,7 @@ def _count_tokens(code, string):
 
     try:
         for ttyp, ttok, __, __, __ in _tokens(code):
-            if ttyp in (tokenize.COMMENT, tokenize.STRING):
+            if ttyp in (COMMENT, STRING):
                 continue
             if ttok == string:
                 count += 1
@@ -353,7 +350,7 @@ def count_non_comment_lines(at_least=None, at_most=None, exactly=None, error_msg
     def check(code):
         linenums = set()
         for ttyp, ttok, (srow, __), __, __ in _tokens(code):
-            if ttyp in (tokenize.COMMENT, tokenize.STRING):
+            if ttyp in (COMMENT, STRING):
                 # Comments and strings don't count toward line count. If a string
                 # is the only thing on a line, then it's probably a docstring, so
                 # don't count it.
@@ -530,7 +527,7 @@ def invoke_student_function(fn_name, args, environment=None, output_writer=None)
     """
     output_writer = output_writer or repr
     def doit(submission_module):
-        for name, value in environment or {}.items():
+        for name, value in (environment or {}).items():
             setattr(submission_module, name, value)
         fn = getattr(submission_module, fn_name)
         print(output_writer(fn(*args)))

tests/test_manager.py

@@ -67,7 +67,8 @@ def test_codejail_config(self):
                 "VMEM": 1024
             }
         }
-        self.m.enable_codejail(config)
+        codejail_return = self.m.enable_codejail(config)
+        self.assertEqual(codejail_return, config["name"])
         self.assertTrue(codejail.jail_code.is_configured("python"))
         self.m.enable_codejail({
             "name": "other-python",

xqueue_watcher/manager.py

@@ -119,8 +119,8 @@ def enable_codejail(self, codejail_config):
         user = codejail_config.get('user', getpass.getuser())
         jail_code.configure(name, bin_path, user=user)
         limits = codejail_config.get("limits", {})
-        for name, value in limits.items():
-            jail_code.set_limit(name, value)
+        for limit_name, value in limits.items():
+            jail_code.set_limit(limit_name, value)
         self.log.info("configured codejail -> %s %s %s", name, bin_path, user)
         return name