From 56c1331ff64d7256f290f29ad829e048d0a54a83 Mon Sep 17 00:00:00 2001
From: rnichi1 <71671446+rnichi1@users.noreply.github.com>
Date: Sun, 2 Feb 2025 15:43:41 +0100
Subject: [PATCH 1/8] llm example task

---
 01_intro/hello_world/config.toml              |  2 +
 .../carpark_multiple_inheritance/config.toml  |  1 -
 04_llm/config.toml                            | 25 +++++++++++
 04_llm/string_manipulation/config.toml        | 45 +++++++++++++++++++
 .../string_manipulation/grading/examples.toml |  7 +++
 04_llm/string_manipulation/grading/post.md    |  1 +
 04_llm/string_manipulation/instructions_en.md | 11 +++++
 .../string_manipulation/rubrics/rubrics.toml  |  9 ++++
 .../solution/explanation.md                   | 16 +++++++
 04_llm/string_manipulation/solution/script.py |  3 ++
 .../string_manipulation/task/explanation.md   |  0
 04_llm/string_manipulation/task/script.py     |  7 +++
 12 files changed, 126 insertions(+), 1 deletion(-)
 create mode 100644 04_llm/config.toml
 create mode 100644 04_llm/string_manipulation/config.toml
 create mode 100644 04_llm/string_manipulation/grading/examples.toml
 create mode 100644 04_llm/string_manipulation/grading/post.md
 create mode 100644 04_llm/string_manipulation/instructions_en.md
 create mode 100644 04_llm/string_manipulation/rubrics/rubrics.toml
 create mode 100644 04_llm/string_manipulation/solution/explanation.md
 create mode 100644 04_llm/string_manipulation/solution/script.py
 create mode 100644 04_llm/string_manipulation/task/explanation.md
 create mode 100644 04_llm/string_manipulation/task/script.py

diff --git a/01_intro/hello_world/config.toml b/01_intro/hello_world/config.toml
index 56026cc..f171b51 100644
--- a/01_intro/hello_world/config.toml
+++ b/01_intro/hello_world/config.toml
@@ -30,3 +30,5 @@ solution = [
   "solution/script.py",
 ]
 
+
+
diff --git a/03_classes/carpark_multiple_inheritance/config.toml b/03_classes/carpark_multiple_inheritance/config.toml
index 0decb9e..e098b71 100644
--- a/03_classes/carpark_multiple_inheritance/config.toml
+++ b/03_classes/carpark_multiple_inheritance/config.toml
@@ -45,4 +45,3 @@ solution = [
   "solution/electric_car.py",
   "solution/hybrid_car.py",
 ]
-
diff --git a/04_llm/config.toml b/04_llm/config.toml
new file mode 100644
index 0000000..24ae09c
--- /dev/null
+++ b/04_llm/config.toml
@@ -0,0 +1,25 @@
+# Mandatory. This slug will be used for the URL of the assignment.
+# ACCESS will refuse to import or update the course if the course/assignment slug is already taken by another assignment.
+slug = "llm"
+
+# Mandatory.
+# Assignments will be invisible to regular users until the start date.
+# Users will be able to submit solutions between the start and end date.
+# After the end date, users will be able to see the assignment and run code, but they may not submit solutions.
+# They will also see files marked as "solution" in individual tasks.
+start = 2023-01-01T13:00:00
+end = 2028-01-01T13:00:00
+
+# Mandatory. List of directory paths containing the tasks.
+# ACCESS will show tasks in the order in which they are listed here.
+"tasks" = [
+  "string_manipulation",
+]
+
+# Information for at least one language must be specified.
+[information.en]
+title = "String Manipulation in Python"
+
+[information.de]
+title = "Stringmanipulation in Python"
+
diff --git a/04_llm/string_manipulation/config.toml b/04_llm/string_manipulation/config.toml
new file mode 100644
index 0000000..e0f57bf
--- /dev/null
+++ b/04_llm/string_manipulation/config.toml
@@ -0,0 +1,45 @@
+slug = "string_manipulation"
+
+
+max_attempts = 6
+refill = 43200 # 12 hours
+max_points = 2
+
+[information.en]
+title = "String Manipulation in Python"
+instructions_file = "instructions_en.md"
+
+[evaluator]
+docker_image = "python:latest"
+run_command = "python -m task.script"
+test_command = "python -m unittest discover -v task"
+grade_command = "python -m grading.tests"
+
+[llm]
+submission = "task/explanation.md"
+rubrics = 'rubrics/rubrics.toml'
+examples = 'grading/examples.toml'
+solution = 'solution/explanation.md'
+cot = true
+voting = 3
+post = "grading/post.md"
+temperature = 0.2 # Decides the randomness of the gpt model
+model = "gpt" # gpt or claude
+
+[files]
+visible = [
+  "task/script.py",
+  "task/explanation.md",
+  "task/tests.py",
+]
+editable = [
+  "task/script.py",
+  "task/explanation.md",
+]
+grading = [
+  "grading/tests.py",
+]
+solution = [
+  "solution/script.py",
+  "solution/explanation.md",
+]
\ No newline at end of file
diff --git a/04_llm/string_manipulation/grading/examples.toml b/04_llm/string_manipulation/grading/examples.toml
new file mode 100644
index 0000000..1e35797
--- /dev/null
+++ b/04_llm/string_manipulation/grading/examples.toml
@@ -0,0 +1,7 @@
+[[examples]]
+answer = "Reversing word order and reversing characters both have O(n) complexity, but character reversal requires more operations per word, making it slightly less efficient in practice."
+points = "{ \"R1\": 1, \"R2\": 1 }"
+
+[[examples]]
+answer = "Both operations have O(n) complexity because they process each character in the string."
+points = "{ \"R1\": 1, \"R2\": 0 }"
diff --git a/04_llm/string_manipulation/grading/post.md b/04_llm/string_manipulation/grading/post.md
new file mode 100644
index 0000000..8a624ec
--- /dev/null
+++ b/04_llm/string_manipulation/grading/post.md
@@ -0,0 +1 @@
+The student answer will not contain any code. This is expected, since you only need to grade the explanation according to the rubrics!
\ No newline at end of file
diff --git a/04_llm/string_manipulation/instructions_en.md b/04_llm/string_manipulation/instructions_en.md
new file mode 100644
index 0000000..dfbb0fb
--- /dev/null
+++ b/04_llm/string_manipulation/instructions_en.md
@@ -0,0 +1,11 @@
+# String Manipulation in Python
+
+## Task Description
+
+Your task is to implement a Python function called `reverse_words`. This function should take a single string input and return a string with the words in reverse order. For example:
+
+- Input: `"Hello World"`
+- Output: `"World Hello"`
+
+Additionally, reflect on the differences in complexity between reversing the order of words in a sentence and reversing the characters within each word in the `explanation.md` file provided. 
+Which operation do you think is more computationally efficient, and why? Consider factors such as string manipulation methods and time complexity in your explanation. Provide examples to support your reasoning.
diff --git a/04_llm/string_manipulation/rubrics/rubrics.toml b/04_llm/string_manipulation/rubrics/rubrics.toml
new file mode 100644
index 0000000..3b93112
--- /dev/null
+++ b/04_llm/string_manipulation/rubrics/rubrics.toml
@@ -0,0 +1,9 @@
+[[rubrics]]
+id = "R1"
+title = "Mentioned the time complexity of both operations is O(n)"
+points = 1
+
+[[rubrics]]
+id = "R2"
+title = "Explained that both are asymptotically equivalent but in practice, reversing characters is slower"
+points = 1
diff --git a/04_llm/string_manipulation/solution/explanation.md b/04_llm/string_manipulation/solution/explanation.md
new file mode 100644
index 0000000..61da112
--- /dev/null
+++ b/04_llm/string_manipulation/solution/explanation.md
@@ -0,0 +1,16 @@
+Reversing the order of words in a sentence and reversing the characters within each word have different complexities.
+
+ Time Complexity of Both Operations:
+     Reversing the word order involves splitting the string into a list of words O(n), reversing the list(O(n), and joining it back into a string O(n). This results in an overall complexity of O(n).
+     Reversing characters within each word requires iterating through each word and reversing it (O(m) per word, where m is the word length). Since this must be done for all words, the total complexity remains O(n).
+
+ Which is More Efficient and Why:
+     Both operations have an O(n) complexity, but reversing words is generally more efficient in practice because it operates at a higher level (list reversal), whereas reversing characters requires more fine-grained string manipulation.
+     If implemented using in-place reversal, reversing characters within each word can introduce additional overhead compared to simple list manipulation.
+
+Example:
+
+    "Hello World" → "World Hello" (word order reversal)
+    "Hello World" → "olleH dlroW" (character reversal)
+
+While both methods scale similarly, reversing characters involves additional operations per word, making it slightly more complex in practical execution.
\ No newline at end of file
diff --git a/04_llm/string_manipulation/solution/script.py b/04_llm/string_manipulation/solution/script.py
new file mode 100644
index 0000000..f4e4528
--- /dev/null
+++ b/04_llm/string_manipulation/solution/script.py
@@ -0,0 +1,3 @@
+def reverse_words(sentence):
+    # Split the sentence into words, reverse the list, and join it back into a string
+    return ' '.join(sentence.split()[::-1])
diff --git a/04_llm/string_manipulation/task/explanation.md b/04_llm/string_manipulation/task/explanation.md
new file mode 100644
index 0000000..e69de29
diff --git a/04_llm/string_manipulation/task/script.py b/04_llm/string_manipulation/task/script.py
new file mode 100644
index 0000000..7b3c71b
--- /dev/null
+++ b/04_llm/string_manipulation/task/script.py
@@ -0,0 +1,7 @@
+# Task: Implement a function `reverse_words` that takes a string
+# and returns the string with the order of words reversed.
+# Example: "Hello World" -> "World Hello"
+
+def reverse_words(sentence):
+    # TODO: Implement this function
+    pass

From 42d1dbbfebd1ca76e1a8babd7695809e61ecc7eb Mon Sep 17 00:00:00 2001
From: rnichi1 <71671446+rnichi1@users.noreply.github.com>
Date: Sun, 2 Feb 2025 15:53:39 +0100
Subject: [PATCH 2/8] Add llm to assignments list

---
 config.toml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config.toml b/config.toml
index c44ea59..caba144 100644
--- a/config.toml
+++ b/config.toml
@@ -12,7 +12,8 @@ logo = "python.svg"
 "assignments" = [
   "01_intro",
   "02_basics",
-  "03_classes"
+  "03_classes",
+  "04_llm",
 ]
 
 # Mandatory. Determines who can see the course in ACCESS at any given time.

From c2088ff7026ab6a5c191e5c30002ea878f3b6891 Mon Sep 17 00:00:00 2001
From: rnichi1 <71671446+rnichi1@users.noreply.github.com>
Date: Sun, 2 Feb 2025 16:02:14 +0100
Subject: [PATCH 3/8] Fix missing test file

---
 04_llm/string_manipulation/config.toml      |  1 -
 04_llm/string_manipulation/grading/tests.py | 37 +++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)
 create mode 100644 04_llm/string_manipulation/grading/tests.py

diff --git a/04_llm/string_manipulation/config.toml b/04_llm/string_manipulation/config.toml
index e0f57bf..8527e6f 100644
--- a/04_llm/string_manipulation/config.toml
+++ b/04_llm/string_manipulation/config.toml
@@ -30,7 +30,6 @@ model = "gpt" # gpt or claude
 visible = [
   "task/script.py",
   "task/explanation.md",
-  "task/tests.py",
 ]
 editable = [
   "task/script.py",
diff --git a/04_llm/string_manipulation/grading/tests.py b/04_llm/string_manipulation/grading/tests.py
new file mode 100644
index 0000000..b54e61e
--- /dev/null
+++ b/04_llm/string_manipulation/grading/tests.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+# Scaffolding necessary to set up ACCESS test
+import sys
+try: from universal.harness import *
+except: sys.path.append("../../universal/"); from harness import *
+
+# Grading test suite starts here
+
+script = grading_import("task", "script")
+
+class GradingTests(AccessTestCase):
+
+    def _test(self, sentence, expected):
+        actual = script.reverse_words(sentence)
+        self.hint(f"Reversal not correct for sentence='{sentence}'... expected result is '{expected}'!")
+        self.assertEqual(expected, actual)
+
+    def test_case1(self):
+        self._test("Hello World", "World Hello")
+
+    def test_case2(self):
+        self._test("  This   is  a  test  ", "test a is This")
+
+    def test_case3(self):
+        self._test("Python", "Python")
+
+    def test_case4(self):
+        self._test("", "")
+
+    def test_case5(self):
+        self._test("Hello, World!", "World! Hello,")
+
+    def test_case6(self):
+        self._test("123 456 789", "789 456 123")
+
+TestRunner().run(AccessTestSuite(1, [GradingTests]))

From d66cbdaf2dee93c6be4556357621623f71a330e2 Mon Sep 17 00:00:00 2001
From: rnichi1 <71671446+rnichi1@users.noreply.github.com>
Date: Sun, 2 Feb 2025 16:06:03 +0100
Subject: [PATCH 4/8] Add comment to explanation md

---
 04_llm/string_manipulation/task/explanation.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/04_llm/string_manipulation/task/explanation.md b/04_llm/string_manipulation/task/explanation.md
index e69de29..66f3e2d 100644
--- a/04_llm/string_manipulation/task/explanation.md
+++ b/04_llm/string_manipulation/task/explanation.md
@@ -0,0 +1 @@
+[comment]: <> (Add your solution here:)

From e18a0a07a8f6b77246b967296454f980495db79d Mon Sep 17 00:00:00 2001
From: rnichi1 <71671446+rnichi1@users.noreply.github.com>
Date: Sun, 2 Feb 2025 16:52:20 +0100
Subject: [PATCH 5/8] Add llm max points config

---
 04_llm/string_manipulation/config.toml          | 1 +
 04_llm/string_manipulation/rubrics/rubrics.toml | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/04_llm/string_manipulation/config.toml b/04_llm/string_manipulation/config.toml
index 8527e6f..b06b610 100644
--- a/04_llm/string_manipulation/config.toml
+++ b/04_llm/string_manipulation/config.toml
@@ -25,6 +25,7 @@ voting = 3
 post = "grading/post.md"
 temperature = 0.2 # Decides the randomness of the gpt model
 model = "gpt" # gpt or claude
+max_points = 1 # Max points for the sub-task that is passed to the model
 
 [files]
 visible = [
diff --git a/04_llm/string_manipulation/rubrics/rubrics.toml b/04_llm/string_manipulation/rubrics/rubrics.toml
index 3b93112..d2e6fd8 100644
--- a/04_llm/string_manipulation/rubrics/rubrics.toml
+++ b/04_llm/string_manipulation/rubrics/rubrics.toml
@@ -1,9 +1,9 @@
 [[rubrics]]
 id = "R1"
 title = "Mentioned the time complexity of both operations is O(n)"
-points = 1
+points = 0.5
 
 [[rubrics]]
 id = "R2"
 title = "Explained that both are asymptotically equivalent but in practice, reversing characters is slower"
-points = 1
+points = 0.5

From 09b275dda16b57fd07313f0d4c002728e73c418f Mon Sep 17 00:00:00 2001
From: rnichi1 <71671446+rnichi1@users.noreply.github.com>
Date: Sun, 2 Feb 2025 17:08:04 +0100
Subject: [PATCH 6/8] Set claude as default model for example due to better
 performance overall

---
 04_llm/string_manipulation/config.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/04_llm/string_manipulation/config.toml b/04_llm/string_manipulation/config.toml
index b06b610..1708cef 100644
--- a/04_llm/string_manipulation/config.toml
+++ b/04_llm/string_manipulation/config.toml
@@ -24,7 +24,7 @@ cot = true
 voting = 3
 post = "grading/post.md"
 temperature = 0.2 # Decides the randomness of the gpt model
-model = "gpt" # gpt or claude
+model = "claude" # gpt or claude
 max_points = 1 # Max points for the sub-task that is passed to the model
 
 [files]

From c964fab403a2b3408360af2f5342e539a6d53ecf Mon Sep 17 00:00:00 2001
From: rnichi1 <71671446+rnichi1@users.noreply.github.com>
Date: Fri, 14 Feb 2025 18:00:01 +0100
Subject: [PATCH 7/8] Comments for optional fields and toml structure of
 example points

---
 04_llm/string_manipulation/config.toml          | 17 +++++++++--------
 .../string_manipulation/grading/examples.toml   |  4 ++--
 04_llm/string_manipulation/rubrics/rubrics.toml |  4 ++--
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/04_llm/string_manipulation/config.toml b/04_llm/string_manipulation/config.toml
index 1708cef..bccf1a2 100644
--- a/04_llm/string_manipulation/config.toml
+++ b/04_llm/string_manipulation/config.toml
@@ -17,14 +17,15 @@ grade_command = "python -m grading.tests"
 
 [llm]
 submission = "task/explanation.md"
-rubrics = 'rubrics/rubrics.toml'
-examples = 'grading/examples.toml'
-solution = 'solution/explanation.md'
-cot = true
-voting = 3
-post = "grading/post.md"
+rubrics = 'rubrics/rubrics.toml' # This file contains the rubrics for the task to guide the model
+examples = 'grading/examples.toml' # This file contains examples of the task for the model to learn from
+solution = 'solution/explanation.md' # This is the solution file used to guide the model
+cot = true # This will add "think step by step" to the context prompt in order to encourage the model to think step by step
+voting = 3 # This allows for the results to be evaluated 3 times, and the most common result is chosen. This setting will increase the time it takes to grade the task!
+post = "grading/post.md" # Adds further instruction to context prompt
 temperature = 0.2 # Decides the randomness of the gpt model
-model = "claude" # gpt or claude
+model_family = "claude" # gpt or claude
+model = "claude-3-5-sonnet-latest" # gpt-4o, gpt-4o-mini, claude-3-5-sonnet-latest, etc.
 max_points = 1 # Max points for the sub-task that is passed to the model
 
 [files]
@@ -42,4 +43,4 @@ grading = [
 solution = [
   "solution/script.py",
   "solution/explanation.md",
-]
\ No newline at end of file
+]
diff --git a/04_llm/string_manipulation/grading/examples.toml b/04_llm/string_manipulation/grading/examples.toml
index 1e35797..ba40170 100644
--- a/04_llm/string_manipulation/grading/examples.toml
+++ b/04_llm/string_manipulation/grading/examples.toml
@@ -1,7 +1,7 @@
 [[examples]]
 answer = "Reversing word order and reversing characters both have O(n) complexity, but character reversal requires more operations per word, making it slightly less efficient in practice."
-points = "{ \"R1\": 1, \"R2\": 1 }"
+points = { time_complexity_mentioned = 0.5, asymptotically_equivalent = 0.5 }
 
 [[examples]]
 answer = "Both operations have O(n) complexity because they process each character in the string."
-points = "{ \"R1\": 1, \"R2\": 0 }"
+points = { time_complexity_mentioned = 0.5, asymptotically_equivalent = 0 }
diff --git a/04_llm/string_manipulation/rubrics/rubrics.toml b/04_llm/string_manipulation/rubrics/rubrics.toml
index d2e6fd8..ed067b5 100644
--- a/04_llm/string_manipulation/rubrics/rubrics.toml
+++ b/04_llm/string_manipulation/rubrics/rubrics.toml
@@ -1,9 +1,9 @@
 [[rubrics]]
-id = "R1"
+id = "time_complexity_mentioned"
 title = "Mentioned the time complexity of both operations is O(n)"
 points = 0.5
 
 [[rubrics]]
-id = "R2"
+id = "asymptotically_equivalent"
 title = "Explained that both are asymptotically equivalent but in practice, reversing characters is slower"
 points = 0.5

From c8df6a96eb31ec17e52eff3776c527c98f0a687b Mon Sep 17 00:00:00 2001
From: rnichi1 <71671446+rnichi1@users.noreply.github.com>
Date: Fri, 14 Feb 2025 18:04:36 +0100
Subject: [PATCH 8/8] Add pre and prompt comments

---
 04_llm/string_manipulation/config.toml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/04_llm/string_manipulation/config.toml b/04_llm/string_manipulation/config.toml
index bccf1a2..110670e 100644
--- a/04_llm/string_manipulation/config.toml
+++ b/04_llm/string_manipulation/config.toml
@@ -22,7 +22,9 @@ examples = 'grading/examples.toml' # This file contains examples of the task for
 solution = 'solution/explanation.md' # This is the solution file used to guide the model
 cot = true # This will add "think step by step" to the context prompt in order to encourage the model to think step by step
 voting = 3 # This allows for the results to be evaluated 3 times, and the most common result is chosen. This setting will increase the time it takes to grade the task!
-post = "grading/post.md" # Adds further instruction to context prompt
+post = "grading/post.md" # Adds further instruction at the end of the context prompt
+# pre = "grading/pre.md" # Adds further instruction in front of the context prompt
+# prompt = "grading/prompt.md" # replaces the context prompt with the content of the file. This is only for advanced usages!
 temperature = 0.2 # Decides the randomness of the gpt model
 model_family = "claude" # gpt or claude
 model = "claude-3-5-sonnet-latest" # gpt-4o, gpt-4o-mini, claude-3-5-sonnet-latest, etc.