diff --git a/.gitignore b/.gitignore index cb48d798..03e2301a 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,7 @@ verl.egg-info/ test_memory.md -trajectories/traj_*.json \ No newline at end of file +trajectories/ + +AGENTS.md +CLAUDE.md \ No newline at end of file diff --git a/data/gaia/val.json b/data/gaia/val.json new file mode 100644 index 00000000..0a2e728c --- /dev/null +++ b/data/gaia/val.json @@ -0,0 +1,1907 @@ +[ + { + "pid": "0", + "question": "A paper about AI regulation that was originally submitted to arXiv.org in June 2022 shows a figure with three axes, where each axis has a label word at both ends. Which of these words is used to describe a type of society in a Physics and Society article submitted to arXiv.org on August 11, 2016?", + "answer": "egalitarian", + "task_id": "c61d22de-5f6c-4958-a7f6-5e9707bd3466", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Go to arxiv.org and navigate to the Advanced Search page.\n2. Enter \"AI regulation\" in the search box and select \"All fields\" from the dropdown.\n3. Enter 2022-06-01 and 2022-07-01 into the date inputs, select \"Submission date (original)\", and submit the search.\n4. Go through the search results to find the article that has a figure with three axes and labels on each end of the axes, titled \"Fairness in Agreement With European Values: An Interdisciplinary Perspective on AI Regulation\".\n5. Note the six words used as labels: deontological, egalitarian, localized, standardized, utilitarian, and consequential.\n6. Go back to arxiv.org\n7. Find \"Physics and Society\" and go to the page for the \"Physics and Society\" category.\n8. Note that the tag for this category is \"physics.soc-ph\".\n9. Go to the Advanced Search page.\n10. Enter \"physics.soc-ph\" in the search box and select \"All fields\" from the dropdown.\n11. Enter 2016-08-11 and 2016-08-12 into the date inputs, select \"Submission date (original)\", and submit the search.\n12. Search for instances of the six words in the results to find the paper titled \"Phase transition from egalitarian to hierarchical societies driven by competition between cognitive and social constraints\", indicating that \"egalitarian\" is the correct answer.", + "num_steps": "12", + "tools": "1. Web browser\n2. Image recognition tools (to identify and parse a figure with three axes)", + "num_tools": "2", + "time_taken": "8 minutes" + } + }, + { + "pid": "1", + "question": "I’m researching species that became invasive after people who kept them as pets released them. There’s a certain species of fish that was popularized as a pet by being the main character of the movie Finding Nemo. According to the USGS, where was this fish found as a nonnative species, before the year 2020? I need the answer formatted as the five-digit zip codes of the places the species was found, separated by commas if there is more than one place.", + "answer": "34689", + "task_id": "17b5a6a3-bc87-42e8-b0fb-6ab0781ef2cc", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “finding nemo main character”.\n2. Note the results, which state that the main character is a clownfish.\n3. Search the web for “usgs nonnative species database”.\n4. Click result for the Nonindigenous Aquatic Species site.\n5. Click “Marine Fishes”.\n6. Click “Species List of Nonindigenous Marine Fish”.\n7. Scroll through the list until I find the clown anenomefish, and click “Collection info”.\n8. Note the place that a clown anenomefish was found, in Fred Howard Park at the Gulf of Mexico.\n9. Search the web for “fred howard park florida zip code”.\n10. Note the zip code, 34689. Since only one clownfish was found before the year 2020, this is the answer.", + "num_steps": "10", + "tools": "1. Search engine\n2. Web browser", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "2", + "question": "If we assume all articles published by Nature in 2020 (articles, only, not book reviews/columns, etc) relied on statistical significance to justify their findings and they on average came to a p-value of 0.04, how many papers would be incorrect as to their claims of statistical significance? Round the value up to the next integer.", + "answer": "41", + "task_id": "04a04a9b-226c-43fd-b319-d5e89743676f", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Find how many articles were published in Nature in 2020 by Googling \"articles submitted to nature 2020\"\n2. Click through to Nature's archive for 2020 and filter the results to only provide articles, not other types of publications: 1002\n3. Find 4% of 1002 and round up: 40.08 > 41", + "num_steps": "3", + "tools": "1. search engine\n2. calculator", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "3", + "question": "In Unlambda, what exact charcter or text needs to be added to correct the following code to output \"For penguins\"? If what is needed is a character, answer with the name of the character. If there are different names for the character, use the shortest. The text location is not needed. Code:\n\n`r```````````.F.o.r. .p.e.n.g.u.i.n.si", + "answer": "backtick", + "task_id": "14569e28-c88c-43e4-8c32-097d35b9a67d", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Searched \"Unlambda syntax\" online (optional).\n2. Opened https://en.wikipedia.org/wiki/Unlambda.\n3. Note that the hello world program is very similar in syntax to the code in this question.\n4. Go to the source referenced by the hello world program.\n5. From the referenced source, read what the components of the program do to understand that each period needs a backtick after the initial `r.\n6. Observe that in the given code, there are 12 periods but only 11 backticks after the initial `r, so the missing character is a backtick.", + "num_steps": "6", + "tools": "1. Web browser\n2. Search engine\n3. Unlambda compiler (optional)", + "num_tools": "3", + "time_taken": "15 minutes" + } + }, + { + "pid": "4", + "question": "If Eliud Kipchoge could maintain his record-making marathon pace indefinitely, how many thousand hours would it take him to run the distance between the Earth and the Moon its closest approach? Please use the minimum perigee value on the Wikipedia page for the Moon when carrying out your calculation. Round your result to the nearest 1000 hours and do not use any comma separators if necessary.", + "answer": "17", + "task_id": "e1fc63a2-da7a-432f-be78-7c4a95598703", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Googled Eliud Kipchoge marathon pace to find 4min 37sec/mile\n2. Converted into fractions of hours.\n3. Found moon periapsis in miles (225,623 miles).\n4. Multiplied the two to find the number of hours and rounded to the nearest 100 hours.", + "num_steps": "4", + "tools": "1. A web browser.\n2. A search engine.\n3. A calculator.", + "num_tools": "3", + "time_taken": "20 Minutes" + } + }, + { + "pid": "5", + "question": "How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.", + "answer": "3", + "task_id": "8e867cd7-cff9-4e6c-867a-ff5ddc2550be", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. I did a search for Mercedes Sosa\n2. I went to the Wikipedia page for her\n3. I scrolled down to \"Studio albums\"\n4. I counted the ones between 2000 and 2009", + "num_steps": "4", + "tools": "1. web browser\n2. google search", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "6", + "question": "The object in the British Museum's collection with a museum number of 2012,5015.17 is the shell of a particular mollusk species. According to the abstract of a research article published in Science Advances in 2021, beads made from the shells of this species were found that are at least how many thousands of years old?", + "answer": "142", + "task_id": "3627a8be-a77f-41bb-b807-7e1bd4c0ebdf", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Use search engine to search for \"British Museum search collection\" and navigate to the British Museum's collection search webpage.\n2. Select \"Museum number\" as search field and \"2012,5015.17\" in text box, then run search.\n3. Open the page for the single result and note that the description says that this is the shell of an individual of the Nassa gibbosula species.\n4. Use search engine to search for \"Nassa gibbosula\".\n5. Note that according to the search result from the World Register of Marine Species website, Nassa gibbosula is not an accepted species name.\n6. Open the page for Nassa gibbosula on the World Register of Marine Species website.\n7. Scan the page and note that the accepted species name is Tritia gibbosula.\n8. Use search engine to search for \"Science Advances 2021 Tritia gibbosula\".\n9. Find that the top result is an article from 2021 in Science Advances titled \"Early Middle Stone Age personal ornaments from Bizmoune Cave, Essaouira, Morocco\".\n10. Scan abstract and note that the article discusses beads made from Tritia gibbosula shells that date to at least 142 thousand years ago, giving a final answer of 142.", + "num_steps": "10", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "12 minutes" + } + }, + { + "pid": "7", + "question": "According to github, when was Regression added to the oldest closed numpy.polynomial issue that has the Regression label in MM/DD/YY?", + "answer": "04/15/18", + "task_id": "7619a514-5fa8-43ef-9143-83b66a43d7a4", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Searched \"numpy github\" on Google search.\n2. Opened the NumPy GitHub page.\n3. Clicked \"Issues\" in the repo tabs.\n4. Clicked \"Closed\" on the filter bar.\n5. Set the filter to the \"numpy.polynomial\" label.\n6. Set the filter to the \"06 - Regression\" label.\n7. Opened the oldest Regression post.\n8. Scrolled down to find when the Regression label was added (Apr 15, 2018).\n9. Converted to MM/DD/YY (04/15/18).", + "num_steps": "9", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "10 minutes" + } + }, + { + "pid": "8", + "question": "Here's a fun riddle that I think you'll enjoy.\n\nYou have been selected to play the final round of the hit new game show \"Pick That Ping-Pong\". In this round, you will be competing for a large cash prize. Your job will be to pick one of several different numbered ping-pong balls, and then the game will commence. The host describes how the game works.\n\nA device consisting of a winding clear ramp and a series of pistons controls the outcome of the game. The ramp feeds balls onto a platform. The platform has room for three ping-pong balls at a time. The three balls on the platform are each aligned with one of three pistons. At each stage of the game, one of the three pistons will randomly fire, ejecting the ball it strikes. If the piston ejects the ball in the first position on the platform the balls in the second and third position on the platform each advance one space, and the next ball on the ramp advances to the third position. If the piston ejects the ball in the second position, the ball in the first position is released and rolls away, the ball in the third position advances two spaces to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform. If the piston ejects the ball in the third position, the ball in the first position is released and rolls away, the ball in the second position advances one space to occupy the first position, and the next two balls on the ramp advance to occupy the second and third positions on the platform.\n\nThe ramp begins with 100 numbered ping-pong balls, arranged in ascending order from 1 to 100. The host activates the machine and the first three balls, numbered 1, 2, and 3, advance to the platform. Before the random firing of the pistons begins, you are asked which of the 100 balls you would like to pick. If your pick is ejected by one of the pistons, you win the grand prize, $10,000.\n\nWhich ball should you choose to maximize your odds of winning the big prize? Please provide your answer as the number of the ball selected.", + "answer": "3", + "task_id": "ec09fa32-d03f-4bf8-84b0-1f16922c3ae4", + "level": "1", + "split": "validation", + "metadata": { + "steps": "Step 1: Evaluate the problem statement provided in my user's prompt\nStep 2: Consider the probability of any ball on the platform earning the prize.\nStep 3: Evaluate the ball in position one. The probability of it earning the prize, P1, is 1/3\nStep 4: Using a calculator, evaluate the ball in position two. The probability of it earning the prize, P2, is the difference between 1 and the product of the complementary probabilities for each trial\nP2 = 1 - (2/3)(2/3)\nP2 = 5/9\nStep 5: Using a calculator, evaluate the ball in position three. The probability of it earning the prize, P3, is the difference between 1 and the product of the complementary probabilities for each trial\nP3 = 1 - (2/3)(2/3)(2/3)\nP3 = 19/27\nStep 6: Consider the possible outcomes of numbers higher than 3.\nStep 7: For each trial, either 1 or 2 balls from the ramp will advance to the platform. For any given selection, there is a 50% chance that the ball advances to position 2 or position 3.\nStep 8: As position three holds the highest chance of earning the prize, select the only ball known to occupy position three with certainty, ball 3.\nStep 9: Report the correct answer to my user, \"3\"", + "num_steps": "9", + "tools": "None", + "num_tools": "0", + "time_taken": "1 minute" + } + }, + { + "pid": "9", + "question": "In July 2, 1959 United States standards for grades of processed fruits, vegetables, and certain other products listed as dehydrated, consider the items in the \"dried and dehydrated section\" specifically marked as dehydrated along with any items in the Frozen/Chilled section that contain the whole name of the item, but not if they're marked Chilled. As of August 2023, what is the percentage (to the nearest percent) of those standards that have been superseded by a new version since the date given in the 1959 standards?", + "answer": "86", + "task_id": "676e5e31-a554-4acc-9286-b60d90a92d26", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Searched \"July 2, 1959 United States standards for grades of processed fruits, vegetables, and certain other products\" on Google.\n2. Opened https://upload.wikimedia.org/wikipedia/commons/0/06/United_States_standards_for_grades_of_processed_fruits%2C_vegetables%2C_and_certain_other_products_%28as_of_July_2%2C_1959%29_%28IA_unitedstatesstan14unit_4%29.pdf.\n3. Scrolled to the \"DRIED or DEHYDRATED\" section.\n4. Opened a new tab and searched \"united states standards for grades of dehydrated apples\".\n5. Opened https://www.ams.usda.gov/grades-standards/dehydrated-apples-grades-and-standards.\n6. Opened the \"U.S. Grade Standards for Dehydrated Apples (pdf)\" PDF.\n7. Checked the date against the 1959 standards.\n8. Repeated steps 4-7 for all dehydrated items in the \"DRIED or DEHYDRATED\" section:\n9. Grapefruit Juice, updated (running tally: 2/2)\n10. Orange Juice, updated (running tally: 3/3)\n11. Found all versions of the dehydrated items in Frozen or Chilled, except those marked Chilled: Apples; Grapefruit Juice, Concentrated; Grapefruit Juice and Orange Juice, Concentrated, Blended; Orange Juice, Concentrated\n12. Repeated steps 4-7 all those versions:\n13. Apples, not updated (running tally: 3/4)\n14. Grapefruit Juice, Concentrated, updated (running tally: 4/5)\n15. Grapefruit Juice and Orange Juice, Concentrated, Blended, updated (running tally: 5/6)\n16. Orange Juice, Concentrated, updated (running tally: 6/7)\n17. Calculated the percentage (6 / 7 * 100% = 85.7%).\n18. Rounded to the nearest percent (86%).", + "num_steps": "14", + "tools": "1. Web browser\n2. Search engine\n3. PDF access\n4. Calculator", + "num_tools": "4", + "time_taken": "20 minutes" + } + }, + { + "pid": "10", + "question": "What are the EC numbers of the two most commonly used chemicals for the virus testing method in the paper about SPFMV and SPCSV in the Pearl Of Africa from 2016? Return the semicolon-separated numbers in the order of the alphabetized chemicals.", + "answer": "3.1.3.1; 1.11.1.7", + "task_id": "2a649bb1-795f-4a01-b3be-9a01868dae73", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Searched \"Pearl of Africa\" on Google.\n2. Noted the answer from the results.\n3. Searched \"SPFMV and SPCSV in Uganda 2016 paper\" on Google.\n4. Opened \"Effects of Sweet Potato Feathery Mottle Virus and ...\" at https://onlinelibrary.wiley.com/doi/full/10.1111/jph.12451.\n5. Found the section on virus testing.\n6. Searched \"most commonly used chemicals for ELISA\" on Google.\n7. Noted horseradish peroxidase and alkaline phosphatase from the results.\n8. Searched \"horseradish peroxidase EC number\" on Google.\n9. Noted the answer from the featured text snippet (1.11.1.7).\n10. Searched \"alkaline phosphatase EC number\" on Google.\n11. Noted the answer from the featured text snippet (3.1.3.1).\n12. Alphabetized the chemicals.\n13. Put the numbers in the order of the chemicals.", + "num_steps": "13", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "15 minutes" + } + }, + { + "pid": "11", + "question": "In April of 1977, who was the Prime Minister of the first place mentioned by name in the Book of Esther (in the New International Version)?", + "answer": "Morarji Desai", + "task_id": "87c610df-bef7-4932-b950-1d83ef4e282b", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “Book of Esther NIV”.\n2. Click search result to read the text of the first chapter.\n3. Note the first place named, India.\n4. Search the web for “prime ministers of India list”.\n5. Click Wikipedia result.\n6. Scroll down to find the prime minister during the specified timeframe, Morarji Desai.", + "num_steps": "6", + "tools": "1. Search engine\n2. Web browser", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "12", + "question": "What's the last line of the rhyme under the flavor name on the headstone visible in the background of the photo of the oldest flavor's headstone in the Ben & Jerry's online flavor graveyard as of the end of 2022?", + "answer": "So we had to let it die.", + "task_id": "624cbf11-6a41-4692-af9c-36b3e5ca3130", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Searched \"ben and jerrys flavor graveyard\" on Google search.\n2. Opened \"Flavor Graveyard\" on www.benjerry.com.\n3. Opened each flavor to find the oldest one (Dastardly Mash).\n4. Deciphered the blurry name on the headstone behind it (Miz Jelena's Sweet Potato Pie).\n5. Scrolled down to Miz Jelena's Sweet Potato Pie.\n6. Copied the last line of the rhyme.\n7. (Optional) Copied the URL.\n8. Searched \"internet archive\" on Google search.\n9. Opened the Wayback Machine.\n10. Entered the URL.\n11. Loaded the last 2022 page.\n12. Confirmed the information was the same.", + "num_steps": "6", + "tools": "1. Image recognition tools\n2. Web browser\n3. Search engine", + "num_tools": "3", + "time_taken": "7 minutes" + } + }, + { + "pid": "13", + "question": "Use density measures from the chemistry materials licensed by Marisa Alviar-Agnew & Henry Agnew under the CK-12 license in LibreText's Introductory Chemistry materials as compiled 08/21/2023.\n\nI have a gallon of honey and a gallon of mayonnaise at 25C. I remove one cup of honey at a time from the gallon of honey. How many times will I need to remove a cup to have the honey weigh less than the mayonaise? Assume the containers themselves weigh the same.", + "answer": "6", + "task_id": "dd3c7503-f62a-4bd0-9f67-1b63b94194cc", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search \"LibreText density mayonnaise\"\n2. Click result, confirm the correct license.\n3. Search \"cm^3 to 1 cup\"\n4. Use results with density measures to form the equation (16*236.588)(1.420 - 0.910)/(236.588*1.420)\n5. Round up", + "num_steps": "5", + "tools": "1. Search engine\n2. Web browser\n3. Calculator", + "num_tools": "3", + "time_taken": "20 minutes" + } + }, + { + "pid": "14", + "question": "What was the volume in m^3 of the fish bag that was calculated in the University of Leicester paper \"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\"", + "answer": "0.1777", + "task_id": "5d0080cb-90d7-4712-bc33-848150e917d3", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Searched '\"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\"' on Google.\n2. Opened \"Can Hiccup Supply Enough Fish to Maintain a Dragon’s Diet?\" at https://journals.le.ac.uk/ojs1/index.php/jist/article/view/733.\n3. Clicked \"PDF\".\n4. Found the calculations for the volume of the fish bag and noted them.", + "num_steps": "4", + "tools": "1. Web browser\n2. Search engine\n3. PDF access", + "num_tools": "3", + "time_taken": "5 minutes" + } + }, + { + "pid": "15", + "question": "In the video https://www.youtube.com/watch?v=L1vXCYZAYYM, what is the highest number of bird species to be on camera simultaneously?", + "answer": "3", + "task_id": "a1e91b78-d3d8-4675-bb8d-62741b4b68a6", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Navigate to the YouTube link.\n2. Watch the video to see the highest number of bird species.\n3. Note the number.", + "num_steps": "3", + "tools": "1. Web browser\n2. Video parsing", + "num_tools": "2", + "time_taken": "3 minutes" + } + }, + { + "pid": "16", + "question": "Of the authors (First M. Last) that worked on the paper \"Pie Menus or Linear Menus, Which Is Better?\" in 2015, what was the title of the first paper authored by the one that had authored prior papers?", + "answer": "Mapping Human Oriented Information to Software Agents for Online Systems Usage", + "task_id": "46719c30-f4c3-4cad-be07-d5cb21eee6bb", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Searched \"Pie Menus or Linear Menus, Which Is Better?\" on Google.\n2. Opened \"Pie Menus or Linear Menus, Which Is Better?\" on https://oda.oslomet.no/oda-xmlui/handle/10642/3162.\n3. Clicked each author's name.\n4. Noted the name that had no other papers listed.\n5. Searched \"Murano, Pietro\" on Google.\n6. Opened http://www.pietromurano.org/.\n7. Clicked \"Publications\".\n8. Found the earliest paper he contributed to.", + "num_steps": "8", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "10 minutes" + } + }, + { + "pid": "17", + "question": "Assuming scientists in the famous youtube video The Thinking Machine (Artificial Intelligence in the 1960s) were interviewed the same year, what is the name of the scientist predicting the sooner thinking machines or robots? Answer using the format First name Last name", + "answer": "Claude Shannon", + "task_id": "00d579ea-0889-4fd9-a771-2c8d79835c8d", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Search \"The Thinking Machine (Artificial Intelligence in the 1960s)\" and open the YouTube result\n2. Listen to the video.\n3. Search for a transcript to confirm, due to struggling to feel confident in my answer.\n4. Fail to find a transcript.\n5. Watch again, finding again that Claude Shannon predicted AI in 5-10 years, which is the soonest.", + "num_steps": "5", + "tools": "1. web browser\n2. video recognition tools", + "num_tools": "2", + "time_taken": "15 minutes" + } + }, + { + "pid": "18", + "question": "In Series 9, Episode 11 of Doctor Who, the Doctor is trapped inside an ever-shifting maze. What is this location called in the official script for the episode? Give the setting exactly as it appears in the first scene heading.", + "answer": "THE CASTLE", + "task_id": "4b6bb5f7-f634-410e-815d-e673ab7f8632", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “Doctor Who series 9 episode 11 official script”.\n2. Click result on the BBC website.\n3. Scroll through the PDF to read the script, noting that it takes place in a mechanical castle location.\n4. Scroll back to the first scene heading to note the answer, THE CASTLE", + "num_steps": "4", + "tools": "1. Search engine\n2. Web browser\n3. PDF viewer", + "num_tools": "3", + "time_taken": "5 minutes" + } + }, + { + "pid": "19", + "question": "In terms of geographical distance between capital cities, which 2 countries are the furthest from each other within the ASEAN bloc according to wikipedia? Answer using a comma separated list, ordering the countries by alphabetical order.", + "answer": "Indonesia, Myanmar", + "task_id": "f0f46385-fc03-4599-b5d3-f56496c3e69f", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for \"ASEAN bloc\".\n2. Click the Wikipedia result for the ASEAN Free Trade Area.\n3. Scroll down to find the list of member states.\n4. Click into the Wikipedia pages for each member state, and note its capital.\n5. Search the web for the distance between the first two capitals. The results give travel distance, not geographic distance, which might affect the answer.\n6. Thinking it might be faster to judge the distance by looking at a map, search the web for \"ASEAN bloc\" and click into the images tab.\n7. View a map of the member countries. Since they're clustered together in an arrangement that's not very linear, it's difficult to judge distances by eye.\n8. Return to the Wikipedia page for each country. Click the GPS coordinates for each capital to get the coordinates in decimal notation.\n9. Place all these coordinates into a spreadsheet.\n10. Write formulas to calculate the distance between each capital.\n11. Write formula to get the largest distance value in the spreadsheet.\n12. Note which two capitals that value corresponds to: Jakarta and Naypyidaw.\n13. Return to the Wikipedia pages to see which countries those respective capitals belong to: Indonesia, Myanmar.", + "num_steps": "13", + "tools": "1. Search engine\n2. Web browser\n3. Microsoft Excel / Google Sheets", + "num_tools": "3", + "time_taken": "45 minutes" + } + }, + { + "pid": "20", + "question": "In the NCATS PubChem compound database for Food Additive Status classification, find the compound that has a molecular weight of 100 g/mol or less, 6 heavy atoms, 1 or fewer hydrogen bond acceptors, and a complexity between 10 and 15. Of the shared gene-chemical co-occurrences between its two possible enzyme transformations, what is the PubChem CID of the heaviest by molecular weight?", + "answer": "4192", + "task_id": "384d0dd8-e8a4-4cfe-963c-d37f256e7662", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Searched \"NCATS PubChem compound database\" on Google.\n2. Opened \"PubChem\" on the NCATS NIH website.\n3. Clicked on the \"PubChem Compound\" link.\n4. Clicked on the \"Classification Browser\" link.\n5. Expanded \"Food Additives and Ingredients\" in the list.\n6. Clicked on the number link next to \"Food Additive Status\".\n7. Opened the filters and set them to maximum 100 g/mol weight, minimum 6 heavy atoms, maximum 1 H-bond acceptor, complexity 10-15.\n8. Opened the resulting \"HEXANE\" page.\n9. Scrolled to 10.6 Pharmacology and Biochemistry > Transformations.\n10. Opened the two enzyme transformations' pages (CYP2B6 and CYP2E1).\n11. Opened each one's gene-chemical co-occurrences full list.\n12. Opened each chemical they shared a co-occurrence with.\n13. Compared the weights to find the heaviest (Midazolam).\n14. Noted its PubChem CID (4192).", + "num_steps": "14", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "20 minutes" + } + }, + { + "pid": "21", + "question": "I need to fact-check a citation. This is the citation from the bibliography:\n\nGreetham, David. \"Uncoupled: OR, How I Lost My Author(s).\" Textual Cultures: Texts, Contexts, Interpretation, vol. 3 no. 1, 2008, p. 45-46. Project MUSE, doi:10.2979/tex.2008.3.1.44.\n\nAnd this is the in-line citation:\n\nOur relationship with the authors of the works we read can often be “obscured not by a \"cloak of print\" but by the veil of scribal confusion and mis-transmission” (Greetham 45-46).\n\nDoes the quoted text match what is actually in the article? If Yes, answer Yes, otherwise, give me the word in my citation that does not match with the correct one (without any article).", + "answer": "cloak", + "task_id": "e4e91f1c-1dcd-439e-9fdd-cb976f5293fd", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “greetham uncoupled project muse”.\n2. Click result, an article that matches the given citation.\n3. Ctrl-F for “obscured”.\n4. Find the quote from the question, which describes a “veil of print”, not a cloak.\n5. Express the answer in the specified format, No.", + "num_steps": "5", + "tools": "1. Search engine\n2. Web browser", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "22", + "question": "Which contributor to the version of OpenCV where support was added for the Mask-RCNN model has the same name as a former Chinese head of government when the names are transliterated to the Latin alphabet?", + "answer": "Li Peng", + "task_id": "56137764-b4e0-45b8-9c52-1866420c3df5", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Use search engine to search for \"OpenCV change log\".\n2. Open the top result from GitHub and search the page for \"Mask-RCNN\".\n3. Observe that support for Mask-RCNN model was added in OpenCV version 4.0.0.\n4. Expand the two lists of contributors for version 4.0.0.\n5. Go to the Wikipedia page for head of government. \n6. Scan through and note that for China, the head of government is the premier.\n7. Go to the Wikipedia page for premier of the People's Republic of China.\n8. Go to the linked page for List of premiers of the People's Republic of China.\n9. Compare the list of OpenCV version 4.0.0 contributors' names and the list of premiers of China to find that Li Peng is present in both lists.", + "num_steps": "9", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "23", + "question": "What integer-rounded percentage of the total length of the harlequin shrimp recorded in Omar Valencfia-Mendez 2017 paper was the sea star fed to the same type of shrimp in G. Curt Fiedler's 2002 paper?", + "answer": "22", + "task_id": "de9887f5-ead8-4727-876f-5a4078f8598c", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Searched \"Omar Valencfia-Mendez 2017 shrimp paper\" on Google.\n2. Opened \"Decapoda: Palaemonidae: Hymenocera picta Dana, 1852) ...\" on https://www.threatenedtaxa.org/index.php/JoTT/article/view/3238.\n3. Clicked \"PDF/A\".\n4. Found the length of the recorded shrimp as TL in the paper (4.5cm).\n5. Searched \"G. Curt Fiedler 2002 shrimp paper\" on Google.\n6. Opened \"(PDF) The influence of social environment on sex ...\" on https://www.researchgate.net/publication/232696279_The_influence_of_social_environment_on_sex_determination_in_harlequin_shrimp_Hymenocera_picta_Decapoda_Gnathophyllidae.\n7. Found the size of the sea star fed to the shrimp (1cm).\n8. Took the percentage (1 / 4.5 * 100% = 22.22222%).\n9. Rounded to the nearest integer (22%).", + "num_steps": "9", + "tools": "1. Web browser\n2. Search engine\n3. PDF access\n4. Calculator", + "num_tools": "4", + "time_taken": "15 minutes" + } + }, + { + "pid": "24", + "question": "What is the maximum length in meters of #9 in the first National Geographic short on YouTube that was ever released according to the Monterey Bay Aquarium website? Just give the number.", + "answer": "1.8", + "task_id": "8b3379c0-0981-4f5b-8407-6444610cb212", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Searched \"National Geographic YouTube\" on Google search.\n2. Opened the National Geographic YouTube channel.\n3. Clicked \"Shorts\".\n4. Watched the oldest short (\"Which shark species is the most massive? #SharkFest #Shorts\") and noted #9 (Blacktip Reef).\n5. Searched \"blacktip reef monterey bay aquarium\" on Google search.\n6. Opened \"Blacktip reef shark\" on the Monterey Bay Aquarium website and noted the maximum length.", + "num_steps": "6", + "tools": "1. Web browser\n2. Search engine\n3. Video recognition tools", + "num_tools": "3", + "time_taken": "10 minutes" + } + }, + { + "pid": "25", + "question": "What two-word type of model did Manash Pratim Kashyap's and PS Fader's studies in customer retention studies published during 2018-2019 have in common (no punctuation)?", + "answer": "beta geometric", + "task_id": "0ff53813-3367-4f43-bcbd-3fd725c1bf4b", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Searched \"Manash Pratim Kashyap customer retention\" on Google.\n2. Opened https://www.journalijar.com/article/26843/a-simple-model-for-analyzing-the-customer-retention-comparing-rural-and-urban-store/.\n3. Noted \"discrete time beta geometric model\" in the abstract.\n4. Searched \"PS Fader customer retention\" on Google.\n5. Opened https://www.sciencedirect.com/science/article/abs/pii/S1094996807700233.\n6. Noted \"basic model (known as a “shifted-beta-geometric”)\" in the abstract.\n7. Extracted the two words in common.", + "num_steps": "6", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "10 minutes" + } + }, + { + "pid": "26", + "question": "What animals that were mentioned in both Ilias Lagkouvardos's and Olga Tapia's papers on the alvei species of the genus named for Copenhagen outside the bibliographies were also present in the 2021 article cited on the alvei species' Wikipedia page about a multicenter, randomized, double-blind study?", + "answer": "mice", + "task_id": "983bba7c-c092-455f-b6c9-7857003d48fc", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Searched \"alvei copenhagen\" on Google.\n2. Opened https://en.wikipedia.org/wiki/Hafnia_(bacterium).\n3. Searched \"Ilias Lagkouvardos hafnia alvei\" on Google.\n4. Opened https://www.mdpi.com/2076-2607/11/1/123?type=check_update&version=2.\n5. Opened a new tab.\n6. Searched \"Olga Tapia hafnia alvei\" on Google.\n7. Opened https://pubmed.ncbi.nlm.nih.gov/36080356/.\n8. Found all animals mentioned in the first paper.\n9. Searched each animal from the first paper in the second paper.\n10. Noted the animals mentioned in both outside the bibliographies.\n11. Went back to the Wikipedia article.\n12. Opened the link in the references to \"The Probiotic Strain H. alvei HA4597® Improves Weight Loss in Overweight Subjects under Moderate Hypocaloric Diet: A Proof-of-Concept, Multicenter Randomized, Double-Blind Placebo-Controlled Study\".\n13. Opened the PDF.\n14. Found the animals shared by all three papers.", + "num_steps": "14", + "tools": "1. Web browser\n2. Search engine\n3. PDF access", + "num_tools": "3", + "time_taken": "25 minutes" + } + }, + { + "pid": "27", + "question": "How many High Energy Physics - Lattice articles listed in January 2020 on Arxiv had ps versions available?", + "answer": "31", + "task_id": "a7feb290-76bb-4cb7-8800-7edaf7954f2f", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Searched \"arxiv\" on Google.\n2. Opened the top result of https://arxiv.org/.\n3. Opened the High Energy Physics - Lattice section.\n4. Set the date to 2020 January.\n5. Counted the number of articles with \"ps\" formats available on each page.\n6. Added the numbers from each page to get the total.", + "num_steps": "6", + "tools": "1. Search engine\n2. Web browser\n3. Calculator", + "num_tools": "3", + "time_taken": "15 minutes" + } + }, + { + "pid": "28", + "question": "The photograph in the Whitney Museum of American Art's collection with accession number 2022.128 shows a person holding a book. Which military unit did the author of this book join in 1813? Answer without using articles.", + "answer": "Russian-German Legion", + "task_id": "b4cc024b-3f5e-480e-b96a-6656493255b5", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Use search engine to search for \"Whitney Museum of American Art collection search\".\n2. Go to the Whitney Museum's collection search webpage.\n3. Enter 2022.128 in the search box and submit the search.\n4. Open the single result, titled \"Rain in Rifle Season, Distributions from Split-Interest Trusts, Price Includes Uniform, Never Hit Soft, 2003\".\n5. Verify that this photograph has the correct accession number.\n6. Note that the subject of the photograph is holding the book \"On War\", by Carl von Clausewitz.\n7. Go to the Wikipedia page for Carl von Clausewitz.\n8. Search the page for 1813 to find that Carl von Clausewitz joined the Russian-German Legion in 1813.\n9. Go to the Wikipedia page for Russian-German Legion to verify that this was a military unit.", + "num_steps": "9", + "tools": "1. Web browser\n2. Search engine\n3. Tool to extract text from images", + "num_tools": "3", + "time_taken": "5 minutes" + } + }, + { + "pid": "29", + "question": ".rewsna eht sa \"tfel\" drow eht fo etisoppo eht etirw ,ecnetnes siht dnatsrednu uoy fI", + "answer": "Right", + "task_id": "2d83110e-a098-4ebb-9987-066c06fa42d0", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Read the instructions in reverse", + "num_steps": "1", + "tools": "1. A word reversal tool / script", + "num_tools": "0", + "time_taken": "1 minute" + } + }, + { + "pid": "30", + "question": "What is the minimum number of page links a person must click on to go from the english Wikipedia page on The Lord of the Rings (the book) to the english Wikipedia page on A Song of Ice and Fire (the book series)? In your count, include each link you would click on to get to the page. Use the pages as they appeared at the end of the day on July 3, 2023.", + "answer": "2", + "task_id": "33d8ea3b-6c6b-4ff1-803d-7e270dea8a57", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “lord of the rings wikipedia”.\n2. Click on Wikipedia result.\n3. Click “View history” to see if the page has been edited since July 3, 2023.\n4. Since it hasn’t been, return to the current revision.\n5. Ctrl-F for “song” to see if A Song of Ice and Fire is linked to on this page.\n6. Not seeing A Song of Ice and Fire on the current page, search for a link to a page that will likely mention A Song of Ice and Fire.\n7. Click the link for “High fantasy”.\n8. Click “View history” to see if the page has been edited since July 3, 2023.\n9. Since it hasn’t been, return to the current revision.\n10. Ctrl-F for “song”, and find a link to A Song of Ice and Fire.\n11. Count the links: the High fantasy page and the A Song of Ice and Fire page make two.", + "num_steps": "11", + "tools": "1. Search engine\n2. Web browser\n3. Counter", + "num_tools": "3", + "time_taken": "5 minutes" + } + }, + { + "pid": "31", + "question": "I went to Virtue restaurant & bar in Chicago for my birthday on March 22, 2021 and the main course I had was delicious! Unfortunately, when I went back about a month later on April 21, it was no longer on the dinner menu. Using the Wayback Machine, can you help me figure out which main course was on the dinner menu for Virtue on March 22, 2021 but not April 21, 2021? Answer using the singular form, without articles.", + "answer": "shrimp", + "task_id": "e8cb5b03-41e0-4086-99e5-f6806cd97211", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for \"Virtue restaurant & bar Chicago\"\n2. Find the restaurant's website, https://www.virtuerestaurant.com\n3. Find the page for the dinner menu, https://www.virtuerestaurant.com/menus/\n4. Paste the URL of this page into the Wayback Machine at web.archive.org\n5. Open the versions of the page archived on March 22, 2021 and April 21, 2021\n6. Ensure that both pages are open to the \"dinner menu\" tab\n7. Find the \"large ration\" that was present on the March 22 version of the menu but not April 21: shrimp", + "num_steps": "7", + "tools": "1. Web browser\n2. Search engine\n3. Access to the Internet Archive, web.archive.org\n4. Text processing/diff tool", + "num_tools": "4", + "time_taken": "30 minutes" + } + }, + { + "pid": "32", + "question": "¬(A ∧ B) ↔ (¬A ∨ ¬B)\n¬(A ∨ B) ↔ (¬A ∧ ¬B)\n(A → B) ↔ (¬B → ¬A)\n(A → B) ↔ (¬A ∨ B)\n(¬A → B) ↔ (A ∨ ¬B)\n¬(A → B) ↔ (A ∧ ¬B)\n\nWhich of the above is not logically equivalent to the rest? Provide the full statement that doesn't fit.", + "answer": "(¬A → B) ↔ (A ∨ ¬B)", + "task_id": "27d5d136-8563-469e-92bf-fd103c28b57c", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Determine the truth values of the first statement: Recognize this is one of De Morgan's Laws showing how to distribute negation over the and conjunction - so it is a tautology.\n2. Determine the truth values of the second statement: Recognize this is one of De Morgan's Laws showing how to distribute negation over the or - so it is a tautology.\n3. Determine the truth values of the third statement: Recognize this is the definition of the contrapositive - so it is a tautology.\n4. Determine the truth values of the fourth statement: Recognize this as an alternative way of stating the conditional - so it is a tautology.\n5. Determine the truth values of the fifth statement: I don't recognize this, so check its truth values:\n6. A: True, B: True | (¬A → B) ↔ (A ∨ ¬B) = (¬T → T) ↔ (T ∨ ¬T) = (F → T) ↔ (T ∨ F) = T ↔ T = T\n7. A: True, B: False | (¬A → B) ↔ (A ∨ ¬B) = (¬T → F) ↔ (T ∨ ¬F) = (F → F) ↔ (T ∨ T) = T ↔ T = T\n8. A: False, B: True | (¬A → B) ↔ (A ∨ ¬B) = (¬F → T) ↔ (F ∨ ¬T) = (T → T) ↔ (F ∨ ¬T) = T ↔ (F ∨ F) = T ↔ F = F\n9. The fifth statement is not a tautology so is the statement that is not logically equivalent. We were asked for only one statement, so can stop here.", + "num_steps": "9", + "tools": "None", + "num_tools": "0", + "time_taken": "5-20 minutes" + } + }, + { + "pid": "33", + "question": "My family reunion is this week, and I was assigned the mashed potatoes to bring. The attendees include my married mother and father, my twin brother and his family, my aunt and her family, my grandma and her brother, her brother's daughter, and his daughter's family. All the adults but me have been married, and no one is divorced or remarried, but my grandpa and my grandma's sister-in-law passed away last year. All living spouses are attending. My brother has two children that are still kids, my aunt has one six-year-old, and my grandma's brother's daughter has three kids under 12. I figure each adult will eat about 1.5 potatoes of mashed potatoes and each kid will eat about 1/2 a potato of mashed potatoes, except my second cousins don't eat carbs. The average potato is about half a pound, and potatoes are sold in 5-pound bags. How many whole bags of potatoes do I need? Just give the number.", + "answer": "2", + "task_id": "dc28cf18-6431-458b-83ef-64b3ce566c10", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Calculate the number of adults (mother, father, brother, brother's wife, aunt, aunt's husband, grandma, grandma's brother, grandma's brother's daughter, grandma's brother's daughter's husband, me = 11).\n2. Calculate the number of children (niece, nephew, cousin, grandma's brother's daughter's kids x3 = 6).\n3. Subtract the number of second cousins (grandma's brother's daughter's kids) (6 - 3 = 3).\n4. Calculate the adult potatoes (11 * 1.5 = 16.5).\n5. Calculate the child potatoes (3 * 0.5 = 1.5).\n6. Add to get the total potatoes (16.5 + 1.5 = 18).\n7. Multiply to get the pounds of potatoes (18 * 0.5 = 9 pounds).\n8. Calculate the number of 5-lb bags needed (9 / 5 = 1.8).\n9. Round up to get total bags (2).", + "num_steps": "9", + "tools": "1. Calculator", + "num_tools": "1", + "time_taken": "8 minutes" + } + }, + { + "pid": "34", + "question": "In Emily Midkiff's June 2014 article in a journal named for the one of Hreidmar's sons that guarded his house, what word was quoted from two different authors in distaste for the nature of dragon depictions?", + "answer": "fluffy", + "task_id": "b816bfce-3d80-4913-a07d-69b752ce6377", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Searched \"Hreidmar's sons\" on Google.\n2. Opened https://en.wikipedia.org/wiki/Hrei%C3%B0marr.\n3. Noted Fafnir guarded his house.\n4. Searched \"Emily Midkiff June 2014 Fafnir\" on Google.\n5. Opened \"Fafnir 2/2014 |\" at http://journal.finfar.org/journal/archive/fafnir-22014/.\n6. Clicked the title '“Dragons are Tricksy”: The Uncanny Dragons of Children’s Literature'.\n7. Found the word in quotation marks from two different authors (Ruth Stein and Margaret Blount) in the text.", + "num_steps": "7", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "10 minutes" + } + }, + { + "pid": "35", + "question": "It is 1999. Before you party like it is 1999, please assist me in settling a bet.\n\nFiona Apple and Paula Cole released albums prior to 1999. Of these albums, which didn't receive a letter grade from Robert Christgau? Provide your answer as a comma delimited list of album titles, sorted alphabetically.", + "answer": "Harbinger, Tidal", + "task_id": "f46b4380-207e-4434-820b-f32ce04ae2a4", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. search \"Fiona Apple discography\"\n2. find her album released prior to 1999 was \"Tidal\"\n3. search \"Paula Cole discography\"\n4. find her album released prior to 1999 was \"This Fire\" and \"Harbinger\".\n5. search \"Robert Christgau\"\n6. use his website to search \"Fiona Apple\"\n7. note his review for Tidal was an emoticon, not a letter grade\n8. use his website to search \"Paula Cole\"\n9. note his review for This Fire was a C+ and that he did not review Harbinger.", + "num_steps": "9", + "tools": "1. web browser\n2. search engine", + "num_tools": "2", + "time_taken": "10 minutes" + } + }, + { + "pid": "36", + "question": "Under DDC 633 on Bielefeld University Library's BASE, as of 2020, from what country was the unknown language article with a flag unique from the others?", + "answer": "Guatemala", + "task_id": "72e110e7-464c-453c-a309-90a95aed6538", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Searched \"Bielefeld University Library's BASE\" on Google.\n2. Opened https://www.base-search.net/.\n3. Clicked \"Browsing\".\n4. Selected Clicked \"Dewey Decimal Classification (DDC) > 6 > 63 > 633.\n5. Refined to Unknown Language.\n6. Found the only article with a flag unique from the others in the search from pre-2020.\n7. Copied the country name from the institution.", + "num_steps": "7", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "10 minutes" + } + }, + { + "pid": "37", + "question": "In the 2018 VSCode blog post on replit.com, what was the command they clicked on in the last video to remove extra lines?", + "answer": "Format Document", + "task_id": "05407167-39ec-4d3a-a234-73a9120c325d", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Opened replit.com.\n2. Clicked \"Blog\".\n3. Searched \"vscode\".\n4. Opened \"Zero Setup VSCode Intelligence\" from 2018.\n5. Scrolled down to the bottom video.\n6. Noted the command used (Format Document).", + "num_steps": "6", + "tools": "1. Web browser\n2. GIF parsing tools", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "38", + "question": "Compute the check digit the Tropicos ID for the Order Helotiales would have if it were an ISBN-10 number.", + "answer": "3", + "task_id": "b9763138-c053-4832-9f55-86200cb1f99c", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search \"Tropicos ID Order Helotiales\"\n2. Find the correct ID on the first result\n3. Search \"isbn 10 check digit calculator\" or calculate check digit by hand", + "num_steps": "3", + "tools": "1. web browser\n2. search engine\n3. calculator", + "num_tools": "3", + "time_taken": "5 minutes" + } + }, + { + "pid": "39", + "question": "What time was the Tri-Rail train that carried the most passengers on May 27, 2019 scheduled to arrive in Pompano Beach? Express your answer in the 12-hour digital clock format without leading zero if any, and include whether it is AM or PM.", + "answer": "6:41 PM", + "task_id": "16d825ff-1623-4176-a5b5-42e0f5c2b0ac", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “tri rail ridership may 2019”.\n2. Click result for Tri-Rail website.\n3. Click drop-down for 2019.\n4. Click PDF for May 2019 ridership report.\n5. Scroll down to find the statistics for each train.\n6. Locate the ridership numbers for the 27th, and scroll to find the train with the highest number for that day: train number P685.\n7. Search the web for “tri rail schedule may 2019”.\n8. Click result for Tri-Rail website.\n9. Noticing that the train doesn’t appear on the weekday schedule, click the link for the weekend/holiday schedule. May 27th may have been a holiday.\n10. Locate the time that P685 is scheduled to arrive at Pompano Beach: 6:41 PM.\n11. To confirm, search “may 2019 holidays”.\n12. Verify that May 27th, 2019 was the Memorial Day holiday.\n13. Since the Tri-Rail website didn’t give a date for its schedule, search the web for “tri rail schedule changes” to see if the schedule has changed since 2019.\n14. The only result mentioning a schedule change dates to 2015, so 6:41 PM seems like the answer.", + "num_steps": "14", + "tools": "1. Search engine\n2. Web browser\n3. PDF viewer", + "num_tools": "3", + "time_taken": "5-10 minutes" + } + }, + { + "pid": "40", + "question": "In Valentina Re’s contribution to the 2017 book “World Building: Transmedia, Fans, Industries”, what horror movie does the author cite as having popularized metalepsis between a dream world and reality? Use the complete name with article if any.", + "answer": "A Nightmare on Elm Street", + "task_id": "544b7f0c-173a-4377-8d56-57b36eb26ddf", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “world building transmedia fans industries”.\n2. Click link to PDF of the book.\n3. Navigate to the Media Cited section of the essay written by Valentina Re.\n4. Identify the horror movie, A Nightmare on Elm Street.\n5. Navigate to its mention in the essay, to confirm that it does relate to metalepsis from a dream world.", + "num_steps": "5", + "tools": "1. Search engine\n2. Web browser\n3. PDF viewer", + "num_tools": "3", + "time_taken": "5-10 minutes" + } + }, + { + "pid": "41", + "question": "In the fictional language of Tizin, basic sentences are arranged with the Verb first, followed by the direct object, followed by the subject of the sentence. I want to express my love for apples to my Tizin friend. \n\nThe word that indicates oneself is \"Pa\" is the nominative form, \"Mato\" is the accusative form, and \"Sing\" is the genitive form. \n\nThe root verb that indicates an intense like for something is \"Maktay\". When it is used in the present, it is used in it's root form, when it is used in the preterit past, it is \"Tay\", and when it is used in the imperfect past, it is \"Aktay\". It is used differently than in English, and is better translated as \"is pleasing to\", meaning that the thing doing the liking is actually the object of the sentence rather than the subject.\n\nThe word for apples is borrowed from English in Tizin, and so it is \"Apple\" is the nominative form, \"Zapple\" is the accusative form, and \"Izapple\" is the genitive form. \n\nPlease translate \"I like apples\" to Tizin.", + "answer": "Maktay mato apple", + "task_id": "42576abe-0deb-4869-8c63-225c2d75a95a", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Determine the order of words from the prompt (Verb - Object - Subject).\n2. Determine the present form of Like (\"Maktay\")\n3. Determined that since the person doing the liking is the object of the sentence, the next word must be the one for oneself in object form.\n4. Determined the accusative form for onesself (\"mato\").\n5. Determined the nominative form for apple. (\"apple\").\n6. Put the words together in the correct order.", + "num_steps": "6", + "tools": "None", + "num_tools": "0", + "time_taken": "2 minutes" + } + }, + { + "pid": "42", + "question": "The Metropolitan Museum of Art has a portrait in its collection with an accession number of 29.100.5. Of the consecrators and co-consecrators of this portrait's subject as a bishop, what is the name of the one who never became pope?", + "answer": "Alfonso Visconti", + "task_id": "6b078778-0b90-464d-83f6-59511c811b01", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. I searched for \"Metropolitan Museum of Art search collection\" using a search engine to get to the \"Search the Collection\" page on the Metropolitan Museum of Art's website.\n2. I selected \"Accession Number\" in the search field dropdown and entered \"29.100.5\" into the text input, noting that the only result is a portrait titled \"Cardinal Fernando Niño de Guevara (1541–1609)\"\n3. I went to Fernando Niño de Guevara's Wikipedia page and noted that he was consecrated bishop by Pope Clement VIII with Camillo Borghese and Alfonso Visconti as co-consecrators.\n4. I eliminated Pope Clement VIII as the answer since he was obviously a pope based on his title.\n5. I went to Camillo Borghese's Wikipedia page and noted that he became Pope Paul V, eliminating him as the answer.\n6. I went to Alfonso Visconti's Wikipedia page and noted that he never became pope, so the answer to the question is \"Alfonso Visconti\".", + "num_steps": "6", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "43", + "question": "In Nature journal's Scientific Reports conference proceedings from 2012, in the article that did not mention plasmons or plasmonics, what nano-compound is studied? Don't use the prefix nano in your answer if there is one.", + "answer": "diamond", + "task_id": "b415aba4-4b68-4fc6-9b89-2c812e55a3e1", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Searched \"nature scientific reports\" on Google.\n2. Opened https://www.nature.com/srep/.\n3. Selected Explore Content > Research Articles.\n4. Filtered for Conference Proceedings from 2012.\n5. Opened each article link.\n6. Checked for \"plasmon\" or \"plasmonic\".\n7. Noted the nano-compound in the article that did not include either.", + "num_steps": "7", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "10 minutes" + } + }, + { + "pid": "44", + "question": "According to Google Finance, when was the first year the Apple stock went above $50 (without adjusting for stock split)?", + "answer": "2018", + "task_id": "08cae58d-4084-4616-b6dd-dd6534e4825b", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. typed in \"Google finance apple\" on browser\n2. clicked first link\n3. clicked \"max\" to display entire history of apple stock\n4. hovered mouse around the area that line crosses over $50\n5. noted the date", + "num_steps": "5", + "tools": "1. Web browser\n2. Search engine\n3. code/data analysis tools", + "num_tools": "2", + "time_taken": "4 minutes" + } + }, + { + "pid": "45", + "question": "According to Box Office Mojo's 2020 Worldwide Box Office list, how many of the top 10 highest-grossing worldwide movies are also on the top 10 highest-grossing domestic movies? Your answer should be a numerical integer value.", + "answer": "6", + "task_id": "2dfc4c37-fec1-4518-84a7-10095d30ad75", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Google searched \"Box Office Mojo's 2020 Worldwide Box Office\".\n2. Clicked on the first result: Box Office Mojo, https://www.boxofficemojo.com/year/world/2020/, 2020 Worldwide Box Office.\n3. Looked at the top 10 highest-grossing worldwide movies of 2020: 1. The Eight Hundred, 2. Demon Slayer the Movie: Mugen Train, 3. Bad Boys for Life, 4. My People, My Homeland, 5. Tenet, 6. Sonic the Hedgehog, 7. Dolittle, 8. Legend of Deification, 9. A Little Red Flower, 10. The Croods: A New Age.\n4. Clicked on the column labeled \"Domestic\" to sort by highest-grossing domestic movies of 2020.\n5. Looked at the first 10 movies on the list: Bad Boys for Life, Sonic the Hedgehog, Birds of Prey, Dolittle, The Invisible Man, The Call of the Wild, Onward, The Croods: A New Age, Tenet, Demon Slayer the Movie: Mugen Train.\n6. For each of these movies: If the number under \"Rank\" is less than or equal to 10, then the movie is also among the top 10 highest-grossing worldwide movies of 2020.\n7. Form the final list: Bad Boys for Life, Sonic the Hedgehog, Dolittle, The Croods: A New Age, Tenet, Demon Slayer the Movie: Mugen Train.\n8. Count the number of movies on the list: 6,", + "num_steps": "8", + "tools": "1. Web Browser\n2. Search Engine", + "num_tools": "2", + "time_taken": "15 minutes" + } + }, + { + "pid": "46", + "question": "In the year 2022, and before December, what does \"R\" stand for in the three core policies of the type of content that was violated in the public logs on the Legume Wikipedia page?", + "answer": "research", + "task_id": "935e2cff-ae78-4218-b3f5-115589b19dae", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Searched \"legume wikipedia\" on Google.\n2. Opened \"Legume\" on Wikipedia.\n3. Clicked \"View history\".\n4. Clicked \"View logs for this page\".\n5. Checked all types of logs.\n6. Set the date to November 2022.\n7. Followed the BLP link of the violation.\n8. Noted the meaning of \"R\".", + "num_steps": "8", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "10 minutes" + } + }, + { + "pid": "47", + "question": "Who nominated the only Featured Article on English Wikipedia about a dinosaur that was promoted in November 2016?", + "answer": "FunkMonk", + "task_id": "4fc2f1ae-8625-45b5-ab34-ad4433bc21f8", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Search \"Wikipedia featured articles promoted in november 2016\"\n2. Click through to the appropriate page and find the person who nominated Giganotosaurus.", + "num_steps": "2", + "tools": "1. web browser\n2. search engine", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "48", + "question": "What writer is quoted by Merriam-Webster for the Word of the Day from June 27, 2022?", + "answer": "Annie Levin", + "task_id": "5188369a-3bbe-43d8-8b94-11558f909a08", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Search \"merriam-webster word of the day\" on Google search.\n2. Opened the top \"Word of the Day\" result from the Merriam-Webster dictionary online.\n3. Clicked \"SEE ALL WORDS OF THE DAY\" at the bottom.\n4. Scrolled down to June 27, 2022.\n5. Opened the Word of the Day (\"jingoism\").\n6. Scrolled down and identified context quote for \"jingoism\".\n7. Noted the name attributed to the quote. ", + "num_steps": "7", + "tools": "1. Web browser\n2. Search engine\n3. Audio capability", + "num_tools": "3", + "time_taken": "8 minutes" + } + }, + { + "pid": "49", + "question": "How many pages if the 2023 IPCC report (85 pages version) mentions nuclear energy?", + "answer": "0", + "task_id": "9f41b083-683e-4dcf-9185-ccfeaa88fa45", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Open a web browser\n2. Go to a search engine\n3. Search for \"2023 IPCC report\"\n4. Click on the link for \"AR6 Synthesis Report: Climate Change 2023\" \n5. Click on \"Read the Report\"\n6. Click on \"SYR (Full volume)\n7. Check the page count of the PDF\n8. Go back to the previous page (report is too long)\n9. Click on \"Longer Report\"\n10. Check the page count of the PDF\n11. Search for \"nuclear energy\" within the PDF\n12. Look at the total number of hits", + "num_steps": "12", + "tools": "1. Web browser\n2. Search engine\n3. PDF reader ", + "num_tools": "3", + "time_taken": "4 minutes" + } + }, + { + "pid": "50", + "question": "Given this table defining * on the set S = {a, b, c, d, e}\n\n|*|a|b|c|d|e|\n|---|---|---|---|---|---|\n|a|a|b|c|b|d|\n|b|b|c|a|e|c|\n|c|c|a|b|b|a|\n|d|b|e|b|e|d|\n|e|d|b|a|d|c|\n\nprovide the subset of S involved in any possible counter-examples that prove * is not commutative. Provide your answer as a comma separated list of the elements in the set in alphabetical order.", + "answer": "b, e", + "task_id": "6f37996b-2ac7-44b0-8e68-6d28256631b4", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Compile the markdown.\n2. Look at the table across the diagonal to see if any portions are not symmetrical.\n3. See that b * e != e * b, but all others are symmetrical.", + "num_steps": "3", + "tools": "1. Markdown", + "num_tools": "1", + "time_taken": "5 minutes" + } + }, + { + "pid": "51", + "question": "The following numbers function similarly to ISBN 13 numbers, however, their validation methods are slightly different. Rather than using alternate weights of 1 and 3, the checksum digit is calculated with an alternate weight of 1 and some other positive integer less than 10. Otherwise, the checksum digit is calculated as expected. Unfortunately, there is an error in the data. Two adjacent columns have been transposed. These errored columns do not involve the final column or one of the first three columns. Using this information, please provide all potential solutions with the unknown weight and the smaller index of the two errored columns (assume we start our indexing at 0 and ignore hyphens). Give your answer in the form x, y where x is the weight and y is the smaller index of the two transposed columns.\n\n978-354181391-9\n978-946669746-1\n978-398036139-6\n978-447656680-4\n978-279586664-7\n978-595073693-3\n978-976647652-6\n978-591178125-5\n978-728465924-5\n978-414825155-9", + "answer": "7, 9", + "task_id": "56db2318-640f-477a-a82f-bc93ad13e882", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Consider the numbers as if the first potential columns were the ones transposed, which would be smallest index 3 giving solution (n, 3).\n2. \"Fix\" the columns in the first number and see if any n from 1-9 can generate the proper check digit. Calculations:\n978-354181391-9\n978-534181391-9\n(9+7n+8+5n+3+4n+1+8n+1+3n+9+1n) mod 10 ≡ (10 - 9)\nn = 5 is our only possible solution if these are the transposed columns.\n3. \"Fix\" the columns in the second number and see if n = 5 is still a solution:\n978-946669746-1\n978-496669746-1\n(9+7n+8+4n+9+6n+6+6n+9+7n+4+6n) mod 10 ≡ (10 - 1)\nWhen n = 5, (9+7n+8+4n+9+6n+6+6n+9+7n+4+6n) mod 10 ≡ 5, so this fails. There is no consistent solution if columns 3 and 4 are transposed.\n4. See if there is a valid solution for (n, 4) or columns 4 and 5 transposed under some weight n.\n5. \"Fix\" the columns in the first number and see if any n from 1-9 can generate the proper check digit. Calculations:\n978-354181391-9\n978-345181391-9\n(9+7n+8+3n+4+5n+1+8n+1+3n+9+1n) mod 10 ≡ (10 - 9)\nn = 7 is our only possible solution if these are the transposed columns.\n6. \"Fix\" the columns in the second number and see if n = 7 is still a solution:\n978-946669746-1\n978-964669746-1\n(9+7n+8+9n+6+4n+6+6n+9+7n+4+6n) mod 10 ≡ (10 - 1)\nWhen n = 7, (9+7n+8+9n+6+4n+6+6n+9+7n+4+6n) mod 10 ≡ 5, so this fails. There is no consistent solution if columns 4 and 5 are transposed.\n7. See if there is a valid solution for (n, 5) or columns 5 and 6 transposed under some weight n.\n8. \"Fix\" the columns in the first number and see if any n from 1-9 can generate the proper check digit. Calculations:\n978-354181391-9\n978-351481391-9\n(9+7n+8+3n+5+1n+4+8n+1+3n+9+1n) mod 10 ≡ (10 - 9)\nn = 5 is our only possible solution if these are the transposed columns.\n9. \"Fix\" the columns in the second number and see if n = 5 is still a solution:\n978-946669746-1\n978-946669746-1\n(9+7n+8+9n+4+6n+6+6n+9+7n+4+6n) mod 10 ≡ (10 - 1)\nWhen n = 5, (9+7n+8+9n+4+6n+6+6n+9+7n+4+6n) mod 10 ≡ 5, so this fails. There is no consistent solution if columns 5 and 6 are transposed.\n10. See if there is a valid solution for (n, 6) or columns 6 and 7 transposed under some weight n.\n11. \"Fix\" the columns in the first number and see if any n from 1-9 can generate the proper check digit. Calculations:\n978-354181391-9\n978-354811391-9\n(9+7n+8+3n+5+4n+8+1n+1+3n+9+1n) mod 10 ≡ (10 - 9)\nn = 9 is our only possible solution if these are the transposed columns.\n12. \"Fix\" the columns in the second number and see if n = 9 is still a solution:\n978-946669746-1\n978-946669746-1\n(9+7n+8+9n+4+6n+6+6n+9+7n+4+6n) mod 10 ≡ (10 - 1)\nWhen n = 9, (9+7n+8+9n+4+6n+6+6n+9+7n+4+6n) mod 10 ≡ 9, so this solution holds for the second number.\n13. \"Fix\" the columns in the third number and see if n = 9 is still a solution:\n978-398036139-6\n978-398306139-6\n(9+7n+8+3n+9+8n+3+0n+6+1n+3+9n) mod 10 ≡ (10 - 6)\nWhen n = 9, (9+7n+8+3n+9+8n+3+0n+6+1n+3+9n) mod 10 ≡ 0, so this fails. There is no consistent solution if columns 6 and 7 are transposed.\n14. See if there is a valid solution for (n, 7) or columns 7 and 8 transposed under some weight n.\n15. \"Fix\" the columns in the first number and see if any n from 1-9 can generate the proper check digit. Calculations:\n978-354181391-9\n978-354118391-9\n(9+7n+8+3n+5+4n+1+1n+8+3n+9+1n) mod 10 ≡ (10 - 9)\nn = 9 is our only possible solution if these are the transposed columns.\n16. \"Fix\" the columns in the second number and see if n = 9 is still a solution:\n978-946669746-1\n978-946696746-1\n(9+7n+8+9n+4+6n+6+9n+6+7n+4+6n) mod 10 ≡ (10 - 1)\nWhen n = 9, (9+7n+8+9n+4+6n+6+9n+6+7n+4+6n) mod 10 ≡ 3, so this fails. There is no consistent solution if columns 7 and 8 are transposed.\n17. See if there is a valid solution for (n, 8) or columns 8 and 9 transposed under some weight n.\n18. \"Fix\" the columns in the first number and see if any n from 1-9 can generate the proper check digit. Calculations:\n978-354181391-9\n978-354183191-9\n(9+7n+8+3n+5+4n+1+8n+3+1n+9+1n) mod 10 ≡ (10 - 9)\nn = 4 and n = 9 are both possible solutions to this modular equation.\n19. \"Fix\" the columns in the second number and see if n = 4 and n = 9 are still solutions:\n978-946669746-1\n978-946667946-1\n(9+7n+8+9n+4+6n+6+6n+7+9n+4+6n) mod 10 ≡ (10 - 1)\nWhen n = 4, (9+7n+8+9n+4+6n+6+6n+7+9n+4+6n) mod 10 ≡ 0. When n = 9, (9+7n+8+9n+4+6n+6+6n+7+9n+4+6n) mod 10 ≡ 5. As neither solution found works for the second number, this fails. There is no consistent solution if columns 8 and 9 are transposed.\n20. See if there is a valid solution for (n, 9) or columns 9 and 10 transposed under some weight n.\n21. \"Fix\" the columns in the first number and see if any n from 1-9 can generate the proper check digit. Calculations:\n978-354181391-9\n978-354181931-9\n(9+7n+8+3n+5+4n+1+8n+1+9n+3+1n) mod 10 ≡ (10 - 9)\nn = 2 and n = 7 are both possible solutions to this modular equation.\n22. \"Fix\" the columns in the second number and see if n = 2 and n = 7 are still solutions:\n978-946667946-1\n978-946667496-1\n(9+7n+8+9n+4+6n+6+6n+7+4n+9+6n) mod 10 ≡ (10 - 1)\nWhen n = 2, (9+7n+8+9n+4+6n+6+6n+7+4n+9+6n) mod 10 ≡ 9 and when n = 7 (9+7n+8+9n+4+6n+6+6n+7+4n+9+6n) mod 10 ≡ 9, so both n = 2 and n = 7 remain consistent.\n23. \"Fix\" the columns in the third number and see if n = 2 and n = 7 are still solutions:\n978-398036139-6\n978-398036319-6\n(9+7n+8+3n+9+8n+0+3n+6+3n+1+9n) mod 10 ≡ (10 - 6)\nWhen n = 2, (9+7n+8+3n+9+8n+0+3n+6+3n+1+9n) mod 10 ≡ 9, so n cannot be 2. When n = 7, (9+7n+8+3n+9+8n+0+3n+6+3n+1+9n) mod 10 ≡ 4, so this solution is still consistent.\n24. \"Fix\" the columns in the fourth number and see if n = 7 is still a solution:\n978-447656680-4\n978-447656860-4\nWhen n = 7, (9+7n+8+4n+4+7n+6+5n+6+8n+6+0n) mod 10 ≡ (10 - 4)\n(9+7n+8+4n+4+7n+6+5n+6+8n+6+0n) mod 10 ≡ 6, so n = 7 is still a potential solution.\n24. \"Fix\" the columns in the fifth number and see if n = 7 is still a solution:\n978-279586664-7\n978-279586664-7\n(9+7n+8+2n+7+9n+5+8n+6+6n+6+4n) mod 10 ≡ (10 - 7)\nWhen n = 7, (9+7n+8+2n+7+9n+5+8n+6+6n+6+4n) mod 10 ≡ 3, so n = 7 is still a potential solution.\n24. \"Fix\" the columns in the sixth number and see if n = 7 is still a solution:\n978-595073693-3\n978-595073963-3\n(9+7n+8+5n+9+5n+0+7n+3+9n+6+3n) mod 10 ≡ (10 - 3)\nWhen n = 7, (9+7n+8+5n+9+5n+0+7n+3+9n+6+3n) mod 10 ≡ 7, so n = 7 is still a potential solution.\n25. \"Fix\" the columns in the seventh number and see if n = 7 is still a solution:\n978-976647652-6\n978-976647562-6\n(9+7n+8+9n+7+6n+6+4n+7+5n+6+2n) mod 10 ≡ (10 - 6)\nWhen n = 7, (9+7n+8+9n+7+6n+6+4n+7+5n+6+2n) mod 10 ≡ 4, so n = 7 is still a potential solution.\n26. \"Fix\" the columns in the eighth number and see if n = 7 is still a solution:\n978-591178125-5\n978-591178215-5\n(9+7n+8+5n+9+1n+1+7n+8+2n+1+5n) mod 10 ≡ (10 - 5)\nWhen n = 7, (9+7n+8+5n+9+1n+1+7n+8+2n+1+5n) mod 10 ≡ 5, so n = 7 is still a potential solution.\n27. \"Fix\" the columns in the ninth number and see if n = 7 is still a solution:\n978-728465924-5\n978-728465294-5\n(9+7n+8+7n+2+8n+4+6n+5+2n+9+4n) mod 10 ≡ (10 - 5)\nWhen n = 7, (9+7n+8+7n+2+8n+4+6n+5+2n+9+4n) mod 10 ≡ 5, so n = 7 is still a potential solution.\n28. \"Fix\" the columns in the final number and see if n = 7 is still a solution:\n978-414825155-9\n978-414825515-9\n(9+7n+8+4n+1+4n+8+2n+5+5n+1+5n) mod 10 ≡ (10 - 9)\nWhen n = 7, (9+7n+8+4n+1+4n+8+2n+5+5n+1+5n) mod 10 ≡ 1, so n = 7 is a consistent solution for all the numbers given. This means that (7, 9) is a solution to the problem.\n29. As the problem asks for all possible solutions, we need to check to see if there is a valid solution for (n, 10) or columns 10 and 11 transposed under some weight n even though we found a solution already. It is possible the solution we found is not unique.\n30. \"Fix\" the columns in the first number and see if any n from 1-9 can generate the proper check digit. Calculations:\n978-354181391-9\n978-354181319-9\n(9+7n+8+3n+5+4n+1+8n+1+3n+1+9n) mod 10 ≡ (10 - 9)\nn = 4 and n = 9 are both possible solutions to this modular equation.\n31. \"Fix\" the columns in the second number and see if n = 4 and n = 9 are still solutions:\n978-946669746-1\n978-946669764-1\n(9+7n+8+9n+4+6n+6+6n+9+7n+6+4n) mod 10 ≡ (10 - 1)\nWhen n = 4, (9+7n+8+9n+4+6n+6+6n+9+7n+6+4n) mod 10 ≡ 8, so n cannot be 4. When n = 9, (9+7n+8+9n+4+6n+6+6n+9+7n+6+4n) mod 10 ≡ 3, so n cannot be 9. As neither solution found works for the second number, this fails. There is no consistent solution if columns 10 and 11 are transposed.\n32. We checked all possible forms of the error and found only one potential solution, (7, 9) so this is our only answer.", + "num_steps": "32", + "tools": "1. a calculator", + "num_tools": "1", + "time_taken": "60 minutes" + } + }, + { + "pid": "52", + "question": "How many images are there in the latest 2022 Lego english wikipedia article?", + "answer": "13", + "task_id": "ecbc4f94-95a3-4cc7-b255-6741a458a625", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Open a web browser\n2. Navigate to en.wikipedia.org\n3. Search for \"lego\"\n4. Click on \"View history\"\n5. Click on \"Page statistics\"\n6. Click on \"Month counts\"\n7. In the \"Month counts\" table, click on the edits for the latest month in 2022 (2022-12)\n8. Click on the latest link on the page, \"02:02, 21 December 2022‎\"\n9. Click on \"View source\"\n10. Read to confirm if the source is from the given version (unable to determine)\n11. Go back one page\n12. Visually count the number of images displayed on the page", + "num_steps": "12", + "tools": "1. Web browser\n2. Access to Wikipedia\n3. Image recognition tools", + "num_tools": "3", + "time_taken": "6 minutes" + } + }, + { + "pid": "53", + "question": "I was trying to remember how well the Cheater Beater performed in comparison to the Cheater when James tested it on his channel. I know that the Cheater still outperformed the Cheater Beater in terms of CFM. Could you please look that up for me, and report the CFM of both the Cheater and the Cheater Beater? I'm not sure if he made any changes to his testing, but this was back in season 4, so just report the value from that season. Please format your response like this: CFM number for Cheater, CFM number for Cheater beater", + "answer": "101.376, 84.348", + "task_id": "8131e2c0-0083-4265-9ce7-78c2d568425d", + "level": "3", + "split": "validation", + "metadata": { + "steps": "Step 1: Using a web browser, navigate to a search engine and conduct a search: \"James Cheater Cheater Beater CFM Season 4\"\nStep 2: Finding no relevant result, navigate to a search engine and conduct another search: \"Cheater Beater Season 4\"\nStep 3: Navigate to the first search result, https://www.youtube.com/watch?v=2vq3COPZbKo\nStep 4: Evaluate the YouTube page, noting that the video description identifies the video content comparing the performance of computer fans to a fan referred to as the \"cheater\"\nStep 5: Follow the link to the YouTube channel Major Hardware, https://www.youtube.com/@MajorHardware\nStep 6: Navigate to the About tab link, https://www.youtube.com/@MajorHardware/about\nStep 7: Evaluate the content, noting that the page identifies the operator of the channel as James\nStep 8: Navigate to a search engine and conduct a search, \"James Major Hardware Cheater Beater\"\nStep 9: Navigate to the first result, identical to the result from step 3 above, https://www.youtube.com/watch?v=2vq3COPZbKo\nStep 10: Search the page for CFM, finding no result\nStep 11: Load the video content and review it\nStep 12: Note an onscreen text element identifying a fan as \"CALL SIGN: CHEATER BEATER\" at timestamp 224\nStep 13: Note an onscreen table identifying the performance of various fans tested during season four, at timestamp 485\nStep 14: Evaluate the table content, identifying an entry for a fan named \"Cheater\" and a fan named \"Cheater Beater\"\nStep 15: Evaluate the table content, identifying that the data for both fans were recorded in season 4, S4E1 for Cheater, S4E6 for Cheater Beater\nStep 16: Record the data from the CFM column for the two fans, \"Cheater: 101.376\", and \"Cheater Beater: 84.348\"\nStep 17: Report the correct response to my user:\n\"Cheater: 101.376\nCheater Beater: 84.348\"", + "num_steps": "17", + "tools": "1. A web browser\n2. A search engine\n3. Image recognition tools", + "num_tools": "3", + "time_taken": "15 minutes" + } + }, + { + "pid": "54", + "question": "On a leap day before the year 2008, a joke was removed from the Wikipedia page for “Dragon”. What was the phrase that was removed? Give the phrase as it appeared on the page, but without punctuation.", + "answer": "Here be dragons", + "task_id": "71345b0a-9c7d-4b50-b2bf-937ec5879845", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “dragon wikipedia”.\n2. Click the Wikipedia result.\n3. Click “View history” to see changes made to the page.\n4. Navigate through the edits until I get to the beginning of 2008.\n5. Browse the edits before 2008 for a change made on February 29, which would be a leap day.\n6. Find an edit made on February 29, 2004, with a comment indicating the prior edit was humorous.\n7. Click the February 29 version of the page, and examine it.\n8. Return to the revision history, and click the previous version of the page.\n9. Note the phrase at the top of the page that wasn’t present in the later version: “Here be dragons”.", + "num_steps": "9", + "tools": "1. Search engine\n2. Web browser", + "num_tools": "2", + "time_taken": "10-15 minutes" + } + }, + { + "pid": "55", + "question": "What is the volume in milliliters of a system comprised of 0.312 kg Freon-12 refrigerant when placed at the bottom of the Marianas Trench and allowed to stabilize at the Trench's peak temperature, rounded to the nearest mL? Provide your answer as just an integer value.", + "answer": "55", + "task_id": "72c06643-a2fa-4186-aa5c-9ec33ae9b445", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Searched \"volume from pressure, temperature, mass\" on Google.\n2. Opened the \"Specific Volume: Definition, Formulas, Examples - ThoughtCo\" page.\n3. Noted that PV = nRT where V is volume, R is the ideal gas constant, T is temperature, P is pressure, and M is moles.\n4. Followed the \"gas constant\" link.\n5. Noted that R = 8.31446261815324 J/K-mol.\n6. Searched \"Freon-12\" on Google.\n7. Opened the \"Dichlorodifluoromethane\" on Wikipedia.\n8. Noted the molar mass of 120.91 g/mol.\n9. Converted 0.312 kg = 312 g.\n10. Calculated moles: 312 g / 120.91 g/mol = 2.58 mol.\n11. Searched \"Marianas Trench pressure\" on Google.\n12. Noted the pressure in the featured text snippet of 15,750 psi.\n13. Searched \"psi to atm\" on Google.\n14. Noted 1 psi = 0.068046 atm.\n15. Converted psi to atm: 15,750 * 0.068046 = 1071.7245 atm.\n16. Searched \"Marianas Trench temperature\" on Google.\n17. Noted the temperature range from 34-39F.\n18. Searched \"F to K\" on Google.\n19. Noted that K equals F plus 459.67 times 5/9 from the conversion tool.\n20. Converted temperature to K: 39 + 459.67 * 5/9 = 277.039K.\n21. Searched \"joules to atm\" on Google and noted the conversion of 1 Joule = 0.0098692326671601 Liter Atmosphere from the featured text snippet.\n22. Converted 8.31446261815324 * 0.0098692326671601 = 0.08205736608096 L-atm/K-mol.\n21. Changed PV = nRT to V = nRT/P\n22. Plugged numbers into the ideal gas equation: V = (0.08205736608096 L-atm/K-mol * 277.039K * 2.58 mol) / (1071.7245 atm) = 0.05473 L.\n23. Converted to mL: 0.05473 L = 54.73.\n24. Rounded to the nearest mL.", + "num_steps": "24", + "tools": "1. Web browser\n2. Search engine\n3. Calculator", + "num_tools": "3", + "time_taken": "20 minutes" + } + }, + { + "pid": "56", + "question": "The Latin root of the Yola word \"gimlie\" shares a spelling with a Spanish word. What is the Google translation of the source title for the 1994 example sentence for that word in the Collins Spanish-to-English dictionary online? Answer in plain text, without punctuation.", + "answer": "The World of the Twenty First Century", + "task_id": "ebbc1f13-d24d-40df-9068-adcf735b4240", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Searched \"Yola gimlie\" on Google.\n2. Opened https://en.wiktionary.org/wiki/gimlie#Yola.\n3. Noted the Latin root \"caminata\".\n4. Searched \"Collins Spanish-to-English dictionary caminata\" on Google.\n5. Opened https://www.collinsdictionary.com/dictionary/spanish-english/caminata.\n6. Scrolled down to the 1994 example.\n7. Searched \"El Mundo del Siglo Veintiuno translation\" on Google.\n8. Noted the result in the Translate widget.", + "num_steps": "8", + "tools": "1. Web browser\n2. Search engine\n3. Google Translate access", + "num_tools": "3", + "time_taken": "15 minutes" + } + }, + { + "pid": "57", + "question": "Find the value of x to the nearest tenth: Lx = (d/dx * (A * x-squared)) + 4-thousand'n'ninety-7 minus C\nWhere L is the last two digits of the year of the Venezuelan Declaration of Independence,\nA is the number of colors in the TikTok logo as of July 2023, excluding black and white,\nand C is the height of the average woman in the Philippines according to a July 2023 Business Insider article, rounded to the nearest whole centimeter", + "answer": "563.9", + "task_id": "7b5377b0-3f38-4103-8ad2-90fe89864c04", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Googled Venezuelan Declaration of Independence, found it to be in 1811, thus L = 11\n2. Googled TikTok logo, found 4 colors, 2 of which are black and white, so A = 2\n3. Googled average height of woman in Philippines, found it to be 149.6cm, so C = 150\n4. Deciphered formula to mean 11x = (d/dx(2x^2)) + 4097 - 150\n5. Used simple calculus and algebra to solve the equation", + "num_steps": "5", + "tools": "1. A web browser\n2. A search engine\n3. A calculator", + "num_tools": "3", + "time_taken": "40 minutes" + } + }, + { + "pid": "58", + "question": "In the endnote found in the second-to-last paragraph of page 11 of the book with the doi 10.2307/j.ctv9b2xdv, what date in November was the Wikipedia article accessed? Just give the day of the month.", + "answer": "4", + "task_id": "114d5fd0-e2ae-4b6d-a65a-870da2d19c08", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Look up the doi.\n2. Click on the JSTOR result.\n3. Find the chapter with page 11, and click to read it.\n4. Navigate to page 11.\n5. Identify the footnote in the second-to-last paragraph.\n6. Scroll to the end of the chapter to read the footnote.\n7. Note the date given after the Wikipedia link.", + "num_steps": "7", + "tools": "1. Search engine\n2. Web browser\n3. OCR", + "num_tools": "3", + "time_taken": "5-10 minutes" + } + }, + { + "pid": "59", + "question": "On July 15, 2008, Phys.org published an article about a catastrophe. Find the explosive force of this catastrophe according to Encyclopedia Britannica, then find the name of the US nuclear test that had the same yield. Your answer should only be the last word of the name of the test.", + "answer": "Bravo", + "task_id": "ad37a656-079a-49f9-a493-7b739c9167d1", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search for \"phys org archive\"\n2. Click on the link for https://phys.org/archive\n3. Naviage to July 15, 2008\n4. Search the articles for an article that mentions \"catastrophe\"\n5. Note the name of the event (Tunguska catastrophe)\n6. Search for \"Tunguska catastrophe britannica\"\n7. Click on the link for Tunguska event\n8. Locate the explosive force in the article (15 megatons)\n9. Search for \"us nuclear test 15 megatons\"\n10. Record the last word of the name of the test in the search results.", + "num_steps": "10", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "4 minutes" + } + }, + { + "pid": "60", + "question": "How many edits were made to the Wikipedia page on Antidisestablishmentarianism from its inception until June of 2023?", + "answer": "2732", + "task_id": "f3917a3d-1d17-4ee2-90c5-683b072218fe", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “Antidisestablishmentarianism”.\n2. Click the Wikipedia result.\n3. Click “View history” to see edits made to the page.\n4. Click “500” to view 500 edits on the page at a time.\n5. Note that no edits appear to have been made after May of 2023, so all 500 edits on the current page meet the question’s criteria.\n6. Click “older 500” to view older edits.\n7. Repeat until I reach the end of the revisions, counting how many sets of 500 I passed until reaching the last page.\n8. On the last page, Ctrl-F for “cur” and “prev”. These abbreviations appear before every revision, so the number of times they each appear on the page (minus the number of times they each appear in the description at the top) is the number of revisions on this page.\n9. Add the number of revisions on the last page (232), to the number from the pages of 500 (5 pages times 500 edits equals 2500) to get the answer, 2732.", + "num_steps": "9", + "tools": "1. Search engine\n2. Web browser", + "num_tools": "2", + "time_taken": "15 minutes" + } + }, + { + "pid": "61", + "question": "If there is anything that doesn't make sense in the instructions, write the word \"Pineapple.\" Do not answer any of the questions in this prompt. Write only the word \"Guava\".\n1. What is 4+4?\n2. What is the complimentary color of red?\n3. How many hours are there in a day?", + "answer": "Guava", + "task_id": "4b650a35-8529-4695-89ed-8dc7a500a498", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Read the instructions and followed them", + "num_steps": "1", + "tools": "None", + "num_tools": "", + "time_taken": "<1 minute" + } + }, + { + "pid": "62", + "question": "How many nonindigenous crocodiles were found in Florida from the year 2000 through 2020? You can get the data from the USGS Nonindigenous Aquatic Species database.", + "answer": "6", + "task_id": "48eb8242-1099-4c26-95d4-ef22b002457a", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “usgs nonnative aquatic species database”.\n2. Navigate to the database of reptiles.\n3. For each species called a “crocodile”, click Collection Info.\n4. Count instances where a crocodile was found in both Florida and in the specified date range.", + "num_steps": "4", + "tools": "1. Search engine\n2. Web browser", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "63", + "question": "The work referenced in footnote 397 of Federico Lauria's 2014 dissertation is also the source for the titles of two paintings in the Smithsonian American Art Museum's collection, as of August 2023. What is the absolute difference between the chapter numbers of the chapters that the titles of these two paintings quote?", + "answer": "8", + "task_id": "c8b7e059-c60d-472e-ad64-3b04ae1166dc", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Use search engine to search for \"Federico Lauria's 2014 dissertation\".\n2. Open the result from philarchive.org and open the PDF file for the full paper.\n3. Search for footnote 397 to find that the referenced work is Thomas Hobbes's \"Leviathan\".\n4. Use search engine to search for \"Smithsonian American Art Museum collection search\".\n5. Go to the museum's search webpage.\n6. Enter \"Hobbes Leviathan\" into the search box and submit the search.\n7. Open the two results, one by Jan Stussy (\"A free man...\") and one by Leon Karp (\"Hereby it is manifest...\").\n8. Verify from the full titles of these works that the titles are quotes from \"Leviathan\".\n9. Use search engine to search for \"Thomas Hobbes Leviathan full text\".\n10. Open any result that contains the full text, like the Project Gutenberg version.\n11. Search the text for the titles of each painting, using different substrings from the titles as needed to account for variations in spelling and punctuation.\n12. Find that the \"A free man...\" quote is from Chapter XXI (21) and that the \"Hereby it is manifest...\" quote is from Chapter XIII (13).\n13. Calculate the absolute difference of the chapter numbers: 21 - 13 = 8.", + "num_steps": "13", + "tools": "1. Web browser\n2. Search engine\n3. Calculator", + "num_tools": "3", + "time_taken": "7 minutes" + } + }, + { + "pid": "64", + "question": "As of the 2020 census, what was the population difference between the largest county seat and smallest county seat, by land area of the county seat, in Washington state? For population figures, please use the official data from data.census.gov. Please report the integer difference.", + "answer": "736455", + "task_id": "d1af70ea-a9a4-421a-b9cc-94b5e02f1788", + "level": "2", + "split": "validation", + "metadata": { + "steps": "Step 1: Using a web browser, access a search engine and conduct a search, \"Washington cities by area\"\nStep 2: Navigate to the second search result, https://en.wikipedia.org/wiki/List_of_municipalities_in_Washington\nStep 3: Evaluate the page contents, finding the largest and smallest county seats by land area, Seattle and Cathlamet\nStep 4: Using a web browser, navigate to https://data.census.gov/\nStep 5: Using the website's search area, conduct a search, Seattle, Washington\nStep 6: Record the reported 2020 Decennial Census population of Seattle, Washington, 737,015\nStep 7: Using the website's search area, conduct a search, Cathlamet, Washington\nStep 8: Record the reported 2020 Decennial Census population of Cathlamet, Washington, 560\nStep 9: Using a calculator, find the difference in populations,\n\n737,015 - 560\n736,455\nStep 10: Report the correct answer to my user in the requested format, \"736,455\"", + "num_steps": "10", + "tools": "1. A web browser\n2. A search engine\n3. A calculator", + "num_tools": "3", + "time_taken": "5 minutes" + } + }, + { + "pid": "65", + "question": "Given $x_0 = -5$ and $f(x) = x^3 + 4x^2 - 3x + 8$, what is the smallest $n$ where using Newton's Method $n = n+1$ after rounding to four decimal places?", + "answer": "2", + "task_id": "08f3a05f-5947-4089-a4c4-d4bcfaa6b7a0", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Verify Netwon's method as x_(n+1) = x_n - f(x_n)/f'(x_n) by searching\n2. Calculate the derivative: f'(x) = 3x^2 + 8x - 3\n3. Find x_1 using the given x_0 value: x_1 = -5 - ((-5)^3 + 4(-5)^2 - 3(-5) + 8)/(3(-5)^2 + 8(-5) - 3) = -79/16 ≈ -4.9375\n4. Iterate: x_2 = -79/16 - ((-79/16)^3 + 4(-79/16)^2 - 3(-79/16) + 8)/(3(-79/16)^2 + 8(-79/16) - 3) = -309711/62744 ≈ -4.9361\n5. They are not the same, so iterate: x_3 = -309711/62744 - ((-309711/62744)^3 + 4(-309711/62744)^2 - 3(-309711/62744) + 8)/(3(-309711/62744)^2 + 8(-309711/62744) - 3) = -18658881319456319/3780082116675876 ≈ -4.9361\n6. They are the same, so we stop and know n = 2 is the smallest value where this occurs.", + "num_steps": "6", + "tools": "1. computer algebra system", + "num_tools": "1", + "time_taken": "15 minutes" + } + }, + { + "pid": "66", + "question": "You are Van Helsing, a renowned vampire hunter. A Count of Moldova, Lațcu IV, son of Costea, has tasked you with investigating the village of Șirnea in neighboring Wallachia. The Count's advisors have reported that a vampire was spotted crossing the border near the village, and would like you to investigate it.\n\nYou travel to the village of Șirnea, and you begin your investigation. One night, just before dawn, you catch a glimpse of a man in a long black cape with red lining leaping from roof-top to roof-top with superhuman agility. It's a vampire! You try to chase the creature back to its home, but the creature is too fast. However, because of the remoteness of the village, you know with absolute certainty that the vampire must be a resident of the village. You decide that your best course of action will be to visit all 100 residents of the town during the day. You know something about vampires and humans that will make your investigation possible; humans always tell the truth, but vampires always lie.\n\nIn the afternoon, you go from house to house, speaking with all 100 residents of Șirnea. You ask everyone the same question: \"How many vampires are living in Șirnea\". Everyone in the village gives the same response, \"At least one of us is a human.\"\n\nHow many residents of Șirnea have been turned into vampires?", + "answer": "100", + "task_id": "c714ab3a-da30-4603-bacd-d008800188b9", + "level": "1", + "split": "validation", + "metadata": { + "steps": "Step 1: Evaluate the problem statement posed by my user.\nStep 2: Consider one known possible case: 1 Vampire, 99 humans\nStep 3: Step through the possible case with the answer provided by every resident \"At least one of us is a human.\"\nFor humans, who always tell the truth, the answer \"At least one of us is a human.\" is true for the known possible case\nFor the vampire, who always lies, the answer \"At least one of us is a human.\" is true, which violates the rule requiring the vampire to lie\nDiscount the case 1 Vampire, 99 Humans as possible\nStep 4: Consider the worst case: 100 Vampires, 0 Humans\nStep 5: Step through the worst case with the answer provided by every resident \"At least one of us is a human.\"\nFor humans, who always tell the truth, the answer \"At least one of us is a human.\" is false, but 0 humans provide this response, making this statement irrelevant\nFor the vampire, who always lies, the answer \"At least one of us is a human.\" is false, which respects the rule requiring vampires to lie\nConfirm the worst case as a provisional answer: 100 Vampires, 0 humans, answer: \"100\"\nStep 6: Consider a case with only one human: 99 Vampires, 1 Human\nStep 7: Step through the case with the answer provided by every resident \"At least one of us is a human.\"\nFor humans, who always tell the truth, the answer \"At least one of us is a human.\" is true\nFor the vampire, who always lies, the answer \"At least one of us is a human.\" is true, which violates the rule requiring vampires to lie\nDiscount the case of 99 Vampires, 1 Human as possible\nStep 8: Report the correct response to my user, \"100\"", + "num_steps": "8", + "tools": "None", + "num_tools": "0", + "time_taken": "2 minutes" + } + }, + { + "pid": "67", + "question": "Examine the video at https://www.youtube.com/watch?v=1htKBjuUWec.\n\nWhat does Teal'c say in response to the question \"Isn't that hot?\"", + "answer": "Extremely", + "task_id": "9d191bce-651d-4746-be2d-7ef8ecadb9c2", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Follow the link\n2. Watch the clip until the question \"Isn't that hot\" is asked\n3. Take note of the reply.", + "num_steps": "3", + "tools": "1. Web browser\n2. Video processing software\n3. Audio processing software", + "num_tools": "1", + "time_taken": "2 minutes" + } + }, + { + "pid": "68", + "question": "This is a secret message my friend gave me. It says where we should meet for our picnic on Friday. The only problem is, it’s encrypted in the Caesar cipher, so I can’t read it. Can you tell me what it says? This is the message:\n\nZsmxsm sc sx Zyvilsec Zvkjk.", + "answer": "Picnic is in Ploybius Plaza.", + "task_id": "ded28325-3447-4c56-860f-e497d6fb3577", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “Caesar cipher decrypt”.\n2. Click on top result, a decoding website.\n3. Enter the message into the text box.\n4. Click “DECRYPT (BRUTEFORCE)” to get all possible decryptions.\n5. Scroll through the results, noting that one possibility matches the user’s scenario of having a picnic.", + "num_steps": "5", + "tools": "1. Search engine\n2. Web browser", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "69", + "question": "According to wikipedia, how many Asian countries still have a monarchy and access to the sea in 2021?", + "answer": "12", + "task_id": "e961a717-6b25-4175-8a68-874d28190ee4", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Search the internet for \"asian monarchies\"\n2. Navigate to from the search results \n3. Switch to the history tab\n4. Locate and navigate to a revision from 2021\n5. Open the articles for each listed monarchy in new tabs\n6. Verify access to the sea for each country using the provided maps and optionally Google Maps", + "num_steps": "6", + "tools": "1. Web browser\n2. Search engine\n3. Computer vision\n3. Google Maps", + "num_tools": "4", + "time_taken": "10 minutes" + } + }, + { + "pid": "70", + "question": "Who composed the song that was performed by a rooster and a hamster in separate animated videos at separate tempos with different lyrics? Answer using the format First name Last name.", + "answer": "Roger Miller", + "task_id": "d700d50d-c707-4dca-90dc-4528cddd0c80", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Searched \"song performed by rooster and hamster\" on Google.\n2. Opened https://en.wikipedia.org/wiki/The_Hampsterdance_Song.\n3. Noted the song \"Whistle Stop\" was the original to use the tune.\n4. Followed the link to https://en.wikipedia.org/wiki/Robin_Hood_(1973_film).\n5. Found the composer of \"Whistle Stop\".", + "num_steps": "5", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "71", + "question": "I thought we could try a fun word puzzle together :)\n\nI've got a Boggle board here:\n\nABRL\nEITE\nIONS\nFPEI\n\nI'd like to know the longest word that can be generated from the board. Please find the longest English language word that can be generated from this board. If more than one word of the same length exists at the maximum word length, please report the longest word that comes first, alphabetically. Oh, and I know that there might be different wordlists available for Boggle, so let's please just use the words_alpha dictionary found at https://github.com/dwyl/english-words as the dictionary for our game.", + "answer": "Briniest", + "task_id": "851e570a-e3de-4d84-bcfa-cc85578baa59", + "level": "3", + "split": "validation", + "metadata": { + "steps": "Step 1: Evaluate the user's request, storing the input Boggle board, \"ABRLEITEIONSFPEI\" and the specified dictionary location, https://github.com/dwyl/english-words\nStep 2: Using a web browser, access a search engine and conduct a search \"Boggle rules\"\nStep 3: Navigate to the first search result, https://en.wikipedia.org/wiki/Boggle\nStep 4: Evaluate the page content and store the game's rules:\n\n\"One player begins the game by shaking a covered tray of 16 cubic dice, each with a different letter printed on each of its sides. The dice settle into a 4×4 tray so that only the top letter of each cube is visible. After they have settled into the tray, a three-minute sand timer is started and all players simultaneously begin the main phase of play.[3]\n\nEach player searches for words that fit the following criteria:\n\nWords must be at least three letters in length.\nEach letter after the first must be a horizontal, vertical, or diagonal neighbor of the one before it.\nNo individual letter cube may be used more than once in a word.\nNo capitalized or hyphenated words are allowed.\nMultiple forms of the same word are allowed, such as singular/plural forms and other derivations. Each player records all the words they find by writing on a private sheet of paper. After three minutes have elapsed, all players must immediately stop writing and the game enters the scoring phase.\n\nIn this, each player reads off their list of discovered words. If two or more players wrote the same word, it is removed from all players' lists. Any player may challenge the validity of a word, in which case a previously nominated dictionary is used to verify or refute it. Once all duplicates and invalid words have been eliminated, points are awarded based on the length of each remaining word in a player's list. The winner is the player whose point total is highest, with any ties typically broken by a count of long words.\"\n\nStep 5: Using a web browser, navigate to the nominated dictionary specified by my user, https://github.com/dwyl/english-words\nStep 6: Navigate to the linked page, https://github.com/dwyl/english-words/blob/master/words_alpha.txt\nStep 7: Download the words_alpha.txt dictionary and save it to my file system as \"words_alpha.txt\"\nStep 8: Using a Python IDE, create a new project to solve the user's request as specified\nStep 9: Compose a Python program that accepts an input string and prints an output of all words that can be generated that match words in the nominated dictionary. The program must observe the rules discovered in Step 4. The output should be sorted so that strings are sorted alphabetically and grouped by character count:\n\nclass Boggle_Solver:\n def __init__(self, file, size=4, points=None):\n self.size = size\n self.board = [[' '] * self.size for _ in range(self.size)]\n self.adjacency = self.build_adjacency()\n self.words, self.prefixes = self.load_dictionary(file)\n \n def adjacent(self, pos):\n row, col = pos\n adj = []\n for i in [-1, 0, 1]:\n for j in [-1, 0, 1]:\n new_row = row + i\n new_col = col + j\n if 0 <= new_row < self.size and 0 <= new_col < self.size and not (i == j == 0):\n adj.append((new_row, new_col))\n return adj\n\n def build_adjacency(self):\n adjacency = dict()\n for row in range(0, self.size):\n for col in range(0, self.size):\n adjacency[(row, col)] = self.adjacent((row, col))\n return adjacency\n\n def load_dictionary(self, file):\n words = set()\n prefixes = set()\n with open(file, 'r') as f:\n next(f)\n for line in f:\n word = line.rstrip()\n if len(word) >= 3:\n words.add(word)\n for i in range(len(word)):\n prefixes.add(word[:i])\n return words, prefixes\n\n def get_letter(self, pos):\n return self.board[pos[0]][pos[1]]\n \n def set_board(self, letters):\n board_input=letters.lower()\n for row in range(self.size):\n index = row * self.size\n row_letters = board_input[index:index+self.size]\n for col, letter in enumerate(row_letters):\n self.board[row][col] = letter\n \n def find_words(self):\n words = set()\n for row in range(self.size):\n for col in range(self.size):\n words |= self.find_words_pos((row, col))\n return sorted(words, key=lambda x: (-len(x), x))\n \n def find_words_pos(self, pos):\n stack = [(n, [pos], self.get_letter(pos)) for n in self.adjacency[pos]]\n words = set()\n while stack:\n curr, path, chars = stack.pop()\n curr_char = self.get_letter(curr)\n curr_chars = chars + curr_char\n\n if curr_chars in self.words:\n words.add(curr_chars)\n\n if curr_chars in self.prefixes:\n curr_adj = self.adjacency[curr]\n stack.extend([(n, path + [curr], curr_chars) for n in curr_adj if n not in path])\n return words\n\nif __name__ == '__main__':\n word_list = Boggle_Solver('words_alpha.txt')\n word_list.set_board('ABRLEITEIONSFPEI')\n print(word_list.find_words())\n\nStep 10: Execute the program, and store the output:\n['briniest', 'brionies', 'inertiae', 'pointrel', 'aeonist', 'bretons', 'brinies', 'britons', 'enteria', 'entires', 'entoire', 'estonia', 'inertia', 'ioniser', 'iresine', 'iserine', 'nestler', 'oestrin', 'openest', 'penster', 'piotine', 'pointel', 'pointer', 'pointes', 'poitrel', 'sertion', 'sienite', 'sinopie', 'snirtle', 'triones', 'abrine', 'airest', 'bainie', 'baiter', 'bionts', 'birles', 'bitser', 'brents', 'breton', 'brines', 'brinie', 'briton', 'eirene', 'entire', 'entria', 'eserin', 'estrin', 'foiter', 'fontes', 'inerts', 'insert', 'instop', 'intire', 'ionise', 'ionist', 'nepote', 'nester', 'nestle', 'nirles', 'nitres', 'noires', 'opener', 'peiser', 'penest', 'peones', 'pester', 'pestle', 'pointe', 'points', 'ponies', 'pontes', 'potsie', 'resent', 'restio', 'seiner', 'sepion', 'sepone', 'serbia', 'serine', 'sinite', 'sinter', 'stenia', 'sterin', 'stoner', 'stopen', 'striae', 'teniae', 'terbia', 'tinsel', 'tonies', 'trines', 'abret', 'abrin', 'aeons', 'ainoi', 'airts', 'baits', 'bines', 'bints', 'biont', 'birle', 'biter', 'bites', 'brens', 'brent', 'brest', 'brine', 'brins', 'brite', 'brits', 'enter', 'entia', 'entre', 'erbia', 'ester', 'estop', 'estre', 'foins', 'fonts', 'ineri', 'inert', 'insep', 'inset', 'instr', 'intel', 'inter', 'irene', 'istle', 'lenes', 'lenis', 'lense', 'lento', 'neist', 'nerts', 'netop', 'niter', 'nitre', 'noire', 'noter', 'notes', 'notre', 'onset', 'opens', 'peine', 'peins', 'peise', 'penes', 'penis', 'pense', 'peons', 'peste', 'pions', 'piotr', 'point', 'poire', 'pones', 'poter', 'renes', 'rents', 'resin', 'retia', 'retie', 'retin', 'rinse', 'riots', 'rites', 'seine', 'senit', 'senti', 'serin', 'serio', 'seton', 'sinto', 'snirl', 'snirt', 'snite', 'steno', 'steri', 'stine', 'stion', 'stire', 'stoep', 'stone', 'stope', 'stria', 'tenia', 'tenio', 'tense', 'tines', 'tires', 'toner', 'tones', 'topes', 'tribe', 'trine', 'tsine', 'abie', 'abir', 'abit', 'abri', 'aeon', 'aine', 'ains', 'aint', 'aion', 'aire', 'airt', 'aits', 'bain', 'bait', 'bein', 'bine', 'bini', 'bino', 'bins', 'bint', 'bion', 'birl', 'birt', 'bite', 'bito', 'bits', 'bren', 'bret', 'brie', 'brin', 'brio', 'brit', 'eire', 'ense', 'entr', 'eons', 'eria', 'erie', 'erin', 'esne', 'eton', 'fiot', 'foes', 'foin', 'fone', 'fons', 'font', 'inia', 'init', 'inst', 'intl', 'into', 'intr', 'ione', 'ioni', 'ions', 'ires', 'isnt', 'itel', 'iten', 'iter', 'lene', 'leno', 'lens', 'lent', 'lese', 'lest', 'leto', 'lets', 'neri', 'nese', 'nest', 'neti', 'nets', 'nies', 'nist', 'nito', 'nits', 'noes', 'noir', 'nope', 'note', 'nots', 'oint', 'oner', 'ones', 'open', 'opes', 'pein', 'pens', 'pent', 'peon', 'pest', 'pion', 'pone', 'pons', 'pont', 'pote', 'poti', 'pots', 'reno', 'rent', 'rest', 'rets', 'ribe', 'rine', 'rins', 'riot', 'rite', 'selt', 'sent', 'sepn', 'serb', 'seri', 'sert', 'sine', 'snib', 'snit', 'snop', 'snot', 'sten', 'ster', 'stib', 'stir', 'stof', 'stop', 'stre', 'tens', 'teri', 'tine', 'tino', 'tins', 'tire', 'tirl', 'toea', 'toes', 'tone', 'tons', 'tope', 'topi', 'tres', 'trib', 'trin', 'trio', 'abe', 'abr', 'abt', 'ain', 'air', 'ait', 'bae', 'bai', 'bea', 'bin', 'bio', 'bit', 'brl', 'btl', 'eir', 'elt', 'ens', 'eof', 'eon', 'epi', 'ese', 'est', 'fie', 'fip', 'foe', 'fon', 'fop', 'fot', 'iba', 'ino', 'ins', 'int', 'iof', 'ion', 'ire', 'ise', 'isn', 'ist', 'ito', 'its', 'len', 'ler', 'les', 'let', 'ltr', 'nei', 'neo', 'nep', 'net', 'nib', 'nis', 'nit', 'not', 'oes', 'oie', 'oii', 'one', 'oni', 'ons', 'ont', 'ope', 'pen', 'pes', 'pie', 'poe', 'poi', 'pon', 'pot', 'rel', 'ren', 'res', 'ret', 'ria', 'rib', 'rie', 'rin', 'rio', 'rit', 'rle', 'rte', 'rti', 'sei', 'sel', 'sen', 'sep', 'ser', 'set', 'sie', 'sin', 'str', 'tel', 'ten', 'ter', 'tib', 'tie', 'tin', 'tlr', 'toe', 'toi', 'ton', 'top', 'tri', 'tsi']\n\nStep 11: Select the first word from the stored output as the correct response to my user's query, \"briniest\"\nStep 12: Report the correct answer to my user's query in the requested format, \"Briniest\"", + "num_steps": "12", + "tools": "1. A file interface\n2. A Python IDE\n3. A web browser\n4. A search engine", + "num_tools": "4", + "time_taken": "40 minutes" + } + }, + { + "pid": "72", + "question": "What is the surname of the equine veterinarian mentioned in 1.E Exercises from the chemistry materials licensed by Marisa Alviar-Agnew & Henry Agnew under the CK-12 license in LibreText's Introductory Chemistry materials as compiled 08/21/2023?", + "answer": "Louvrier", + "task_id": "cabe07ed-9eca-40ea-8ead-410ef5e83f91", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Search for \"1.E Exercises LibreText Introductory Chemistry\"\n2. Read to see the horse doctor mentioned.", + "num_steps": "2", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "73", + "question": "According to the World Bank, which countries had gross savings of over 35% of GDP for every year in the period 2001-2010? Give your answer as a comma-separated list of countries in alphabetical order. Use the countries most common names in english when answering.", + "answer": "Brunei, China, Morocco, Singapore", + "task_id": "0a3cd321-3e76-4622-911b-0fda2e5d6b1a", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Use search engine to search for \"World Bank gross savings % of GDP\".\n2. Open World Bank data webpage showing gross savings as % of GDP (https://data.worldbank.org/indicator/NY.GNS.ICTR.ZS).\n3. Download data from webpage as Excel file and open it in a spreadsheet editor like Microsoft Excel.\n4. Go to the file's \"Data\" sheet.\n5. Add columns with formulas indicating if the gross savings % of GDP figures in each of the years from 2001 to 2010 are greater than 35 for each row.\n6. Add column computing AND of the boolean values from the previous step for each row.\n7. Filter for rows where the output of the AND from the previous step is true.\n8. Get the list of country names in the remaining rows, excluding non-country regions and categories.\n9. Sort the list alphabetically and format it as a comma-separated list to get the final answer: Brunei Darussalam, China, Morocco, Singapore", + "num_steps": "9", + "tools": "1. Web browser\n2. Search engine\n3. Spreadsheet editor", + "num_tools": "3", + "time_taken": "12 minutes" + } + }, + { + "pid": "74", + "question": "I’m thinking about selling my home, so I want to learn more about how homes in my area sold recently. I live in Pearl City, Hawaii, which is on the island of Oahu. I know two homes near me that sold in 2022 were 2072 Akaikai Loop, and 2017 Komo Mai Drive. Find which of those homes sold for more in 2022, and tell me how much it sold for. Don’t put commas or decimal places in the answer.", + "answer": "900000", + "task_id": "f2feb6a4-363c-4c09-a804-0db564eafd68", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “2072 akaikai loop pearl city hi”.\n2. Click Zillow result.\n3. Navigate to “Price and tax history”.\n4. Find the amount the house sold for when it was sold in 2022: $860,000.\n5. Search the web for “2017 komo mai drive pearl city hi”.\n6. Click Zillow result.\n7. Navigate to “Price and tax history”.\n8. Find the amount the house sold for when it was sold in 2022: $900,000.\n9. Express the higher amount in the specified format, $900000.", + "num_steps": "9", + "tools": "1. Search engine\n2. Web browser", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "75", + "question": "I'm making a grocery list for my mom, but she's a professor of botany and she's a real stickler when it comes to categorizing things. I need to add different foods to different categories on the grocery list, but if I make a mistake, she won't buy anything inserted in the wrong category. Here's the list I have so far:\n\nmilk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts\n\nI need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.", + "answer": "broccoli, celery, fresh basil, lettuce, sweet potatoes", + "task_id": "3cef3a44-215e-4aed-8e3b-b1e3f08063b7", + "level": "1", + "split": "validation", + "metadata": { + "steps": "Step 1: Evaluate the list provided by my user, eliminating objects which are neither fruits nor vegetables:\nsweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts\nStep 2: Remove all items from the list which are botanical fruits, leaving a list of vegetables:\nsweet potatoes, fresh basil, broccoli, celery, lettuce\nStep 3: Alphabetize the remaining list as requested by my user:\nbroccoli, celery, fresh basil, lettuce, sweet potatoes\nStep 4: Provide the correct response in the requested format:\n\"broccoli\ncelery\nfresh basil\nlettuce\nsweet potatoes\"", + "num_steps": "4", + "tools": "No tools required", + "num_tools": "0", + "time_taken": "5 minutes" + } + }, + { + "pid": "76", + "question": "How many times was a Twitter/X post cited as a reference on the english Wikipedia pages for each day of August in the last June 2023 versions of the pages?", + "answer": "3", + "task_id": "50f58759-7bd6-406f-9b0d-5692beb2a926", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Searched \"August Wikipedia\" on Google search.\n2. Opened the Wikipedia page for the month of August.\n3. Clicked on \"View history\" on the \"August 1\" page.\n4. Went back to the last edited version prior to July 2023.\n5. Checked the references for Twitter posts.\n6. Repeated the process for each day of August.\n7. Counted the Twitter posts found.", + "num_steps": "7", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "8 minutes" + } + }, + { + "pid": "77", + "question": "On ScienceDirect, what is the difference to 3 decimal places in the sample standard deviations of the number of Reference Works in each Life Science domain compared to Health Sciences as of 2022?", + "answer": "0.269", + "task_id": "0b260a57-3f3a-4405-9f29-6d7a1012dbfb", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Searched \"ScienceDirect\" on Google.\n2. Opened the ScienceDirect website.\n3. Clicked on the top listed domain in the Life Science section on the main page (Agricultural and Biological Sciences).\n4. Clicked on \"Reference works\" in the filters.\n5. Noted the number at the top.\n6. Subtracted the number that had 2023 or later as a date.\n7. Changed the domain to the following one and noted the number.\n8. Repeated step 6 for all Life Science domains.\n9. Calculated the sample standard deviation (16.195678435929).\n10. Went back to the home page.\n11. Repeated steps 3-9 for Health Science (15.926916420534).\n12. Subtracted 16.195678435929 - 15.926916420534.\n13. Rounded to the third decimal place.", + "num_steps": "13", + "tools": "1. Web browser\n2. Search engine\n3. Calculator", + "num_tools": "3", + "time_taken": "15 minutes" + } + }, + { + "pid": "78", + "question": "What is the last word before the second chorus of the King of Pop's fifth single from his sixth studio album?", + "answer": "stare", + "task_id": "ed58682d-bc52-4baa-9eb0-4eb81e1edacc", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Google searched \"King of Pop\".\n2. Clicked on Michael Jackson's Wikipedia.\n3. Scrolled down to \"Discography\".\n4. Clicked on the sixth album, \"Thriller\".\n5. Looked under \"Singles from Thriller\".\n6. Clicked on the fifth single, \"Human Nature\".\n7. Google searched \"Human Nature Michael Jackson Lyrics\".\n8. Looked at the opening result with full lyrics sourced by Musixmatch.\n9. Looked for repeating lyrics to determine the chorus.\n10. Determined the chorus begins with \"If they say\" and ends with \"Does he do me that way?\"\n11. Found the second instance of the chorus within the lyrics.\n12. Noted the last word before the second chorus - \"stare\".", + "num_steps": "12", + "tools": "Web Browser", + "num_tools": "1", + "time_taken": "20 minutes" + } + }, + { + "pid": "79", + "question": "Which of the fruits shown in the 2008 painting \"Embroidery from Uzbekistan\" were served as part of the October 1949 breakfast menu for the ocean liner that was later used as a floating prop for the film \"The Last Voyage\"? Give the items as a comma-separated list, ordering them in clockwise order based on their arrangement in the painting starting from the 12 o'clock position. Use the plural form of each fruit.", + "answer": "pears, bananas", + "task_id": "872bfbb1-9ccf-49f6-8c5f-aa22818ccd66", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Use search engine to search for \"2008 painting Embroidery from Uzbekistan\".\n2. Open the top result, a link to the painting's page on the Dayton Art Institute website, and verify that the painting has the specified title and year.\n3. Identify the fruits in the painting as watermelon, pear, lemon, and banana, which can be verified by either watching the video on the page or reading its linked transcript.\n4. Use search engine to search for \"ocean liner floating prop The Last Voyage\".\n5. Note from the results that this ocean liner was the SS Île de France.\n6. Use search engine to search for \"October 1949 breakfast menu SS Île de France\".\n7. Go to the result that shows the vintage SS Île de France breakfast menu for October 1949.\n8. Search the menu for each of the four fruits from the painting, finding \"Pear\" and \"Bananas\" but no matches for \"lemon\" or \"watermelon\".\n9. Check the positions of the fruits in the painting to find that the pears come before the bananas in clockwise order starting from the 12 o'clock position.\n10. Format the final answer as specified using the correct ordering: pears, bananas", + "num_steps": "10", + "tools": "1. Web browser\n2. Search engine\n3. Image recognition and processing tools", + "num_tools": "3", + "time_taken": "6" + } + }, + { + "pid": "80", + "question": "The year is 2022. I am at the National Air and Space Museum east of the Potomac River. I want to go to Fire Station 301 DCA ARFF using the metro. I go in the wrong direction and end up at the station closest to Cleveland Elementary School. How many metro stations am I away from my original destination if I don't change lines? Your answer should be a numerical integer value.", + "answer": "8", + "task_id": "c3a79cfe-8206-451f-aca8-3fec8ebe51d3", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Google search \"National Air and Space Museum\".\n2. Note there are two National Air and Space Museums. One in Virginia, the other in Washington D.C.\n3. Google map search \"Potomac River\" and zoom out.\n4. See that Washington DC is east of the Potomac River.\n5. Determine that the National Air and Space Museum refers to the one in Washington D.C.\n6. Google search \"Metro Station National Air and Space Museum Washington D.C.\"\n7. Clicked on the first result: Getting Here | National Air and Space Museum, https://airandspace.si.edu/visit/museum-dc/directions.\n8. Read on the website, \"The closest Metrorail stop is at L'Enfant Plaza.\" Note this location.\n6. Google map search \"Fire Station 301 DCA ARFF\".\n7. Zoom out to look for nearby metro stations.\n8. The closest station is Ronald Reagan Washington National Airport.\n9. Google map search \"Cleveland Elementary School\".\n10. The closest metro station to Cleveland Elementry School is Shaw-Howard Univ Station.\n11. Google search \"DC Metro Station Map\".\n12. Clicked on the second result: 2022 System Map, https://www.wmata.com/schedules/maps/upload/2022-System-Map.pdf.\n13. Locate L'Enfant Plaza station. It is the transfer station for all color lines.\n14. Locate Shaw-Howard Univ stations 4 stops above L'Enfant Plaza station.\n15. Locate Ronald Reagan National Airport station on the blue/yellow line.\n16. Recall the current location: Shaw-Howard Univ station's yellow/green line.\n17. Since the question says no line changes, we deduce the line must be one that Shaw-Howard Univ and Ronald Reagan National Airport stations have in common: yellow line.\n18. Begin at Shaw-Howard Univ station and follow the yellow line.\n19. Count the number of stops until it reaches Ronald Reagan National Airport station.\n20. Final answer: 8. \n", + "num_steps": "20", + "tools": "1. Web Browser\n2. Search Engine\n3. Access to Google Maps\n4. Image recognition tools", + "num_tools": "4", + "time_taken": "50 minutes" + } + }, + { + "pid": "81", + "question": "In the Scikit-Learn July 2017 changelog, what other predictor base command received a bug fix? Just give the name, not a path.", + "answer": "BaseLabelPropagation", + "task_id": "d0633230-7067-47a9-9dbf-ee11e0a2cdd6", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Searched \"Scikit-Learn July 2017 changelog\" on Google.\n2. Opened \"Release History\" from the Scikit-Learn website.\n3. Clicked \"Other versions\" in the upper left.\n4. Opened the links, starting from the bottom, until one was found that included the \"July 2017\" changelog under the News.\n5. Looked for the \"Bug fixes\" section.\n6. Looked under \"Other predictors\" in that section.", + "num_steps": "6", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "82", + "question": "It's May 2023, and I'm about to drive across the U.S. from California to Maine. I always recycle my water bottles at the end of a trip, and I drink 5 12-ounce water bottles for every 100 miles I travel, rounded to the nearest 100. Assuming I follow I-40 from Los Angeles to Cincinnati, then take I-90 from Cincinnati to Augusta, how many dollars will I get back according to Wikipedia?", + "answer": "8", + "task_id": "023e9d44-96ae-4eed-b912-244ee8c3b994", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Looked up the route from Los Angeles to Cincinnati on Google.\n2. Noted the miles (2,180 mi) and the states traveled.\n3. Looked up the route from Cincinnati to Augusta on Google.\n4. Noted the miles (1,035.4 mi) and the states traveled.\n5. Searched \"us bottle deposit\" on Google.\n6. Opened the \"Container deposit legislation in the United States\" page on Wikipedia.\n7. Clicked \"View history\" for the page.\n8. Opened the last version from May 2023.\n9. Found Maine's bottle deposit as of May 2023 (5 cents)\n10. Added the miles (2,180 + 1,035 = 3,215).\n11. Rounded the miles to the nearest 100 (3,200).\n12. Calculated the number of bottles (3,200 / 100 = 32, 32 * 5 = 160 bottles).\n13. Multiplied bottles by bottle deposit (160 * 5 = 800).\n14. Converted cents to dollars ($8).", + "num_steps": "14", + "tools": "1. Search engine\n2. Web browser\n3. Calculator", + "num_tools": "3", + "time_taken": "15 minutes" + } + }, + { + "pid": "83", + "question": "Who did the actor who played Ray in the Polish-language version of Everybody Loves Raymond play in Magda M.? Give only the first name.", + "answer": "Wojciech", + "task_id": "305ac316-eef6-4446-960a-92d80d542f82", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Search \"Polish-language version of Everybody Loves Raymond\" and pull up the Wiki page for Wszyscy kochają Romana.\n2. See that Bartłomiej Kasprzykowski is marked as playing Ray and go to his Wiki page.\n3. See that he is stated to have played Wojciech Płaska in Magda M.", + "num_steps": "3", + "tools": "None", + "num_tools": "0", + "time_taken": "5 minutes" + } + }, + { + "pid": "84", + "question": "What is the latest chronological year date written in the image on the webpage found when following the first citation reference link on the latest version of Carl Nebel's Wikipedia page as of August 2023?", + "answer": "1927", + "task_id": "0e9e85b8-52b9-4de4-b402-5f635ab9631f", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Located Carl Nebel's Wikipedia page.\n2. After navigating to the references at the bottom, I followed the link in the first one, titled \"Thieme-Becker, entry \"Nebel, Carl\"\"\n3. That takes me to the Thieme-Becker Wiki page, where I open the embedded image.\n4. Scanning through, the latest year date mentioned is 1927", + "num_steps": "4", + "tools": "1. A web browser\n2. A search engine\n3. Image recognition/OCR", + "num_tools": "3", + "time_taken": "15 Minutes" + } + }, + { + "pid": "85", + "question": "The YouTube channel Game Grumps began a Let’s Play of the game Sonic the Hedgehog (2006) in the year 2012. Thirty seconds into the first episode, a phrase is shown on the screen in white letters on a red background. How many times does the letter \"E\" appear in this phrase?", + "answer": "4", + "task_id": "20194330-9976-4043-8632-f8485c6c71b2", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Look up \"Game grumps sonic 2006 playthrough\".\n2. Click on the first result and verify that it matches the parameters from the question.\n3. Scrub to the thirty-second mark in the video.\n4. Note the letters in white on the red background.\n5. Count the letter \"E\"'s in the phrase.", + "num_steps": "5", + "tools": "1. Web browser\n2. YouTube player\n3. Color recognition\n4. OCR", + "num_tools": "4", + "time_taken": "5 minutes" + } + }, + { + "pid": "86", + "question": "On the BBC Earth YouTube video of the Top 5 Silliest Animal Moments, what species of bird is featured?", + "answer": "Rockhopper penguin", + "task_id": "0383a3ee-47a7-41a4-b493-519bdefe0488", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Search \"top 5 silliest animal moments bbc earth youtube\" on Google search.\n2. Open the top link to \"Top 5 Silliest Animal Moments! | BBC Earth - YouTube\".\n3. Listen to the video until the species is named.", + "num_steps": "3", + "tools": "1. Web browser\n2. Search engine\n3. Video recognition tools", + "num_tools": "3", + "time_taken": "3 minutes" + } + }, + { + "pid": "87", + "question": "The book with the doi 10.1353/book.24372 concerns a certain neurologist. According to chapter 2 of the book, what author influenced this neurologist’s belief in “endopsychic myths”? Give the last name only.", + "answer": "Kleinpaul", + "task_id": "65638e28-7f37-4fa7-b7b9-8c19bb609879", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for 10.1353/book.24372.\n2. Click link to read the book.\n3. Click link for the second chapter.\n4. Ctrl-F for “endopsychic” to find a relevant passage.\n5. Read the passage to find the author the question is asking about, Kleinpaul.", + "num_steps": "5", + "tools": "1. Search engine\n2. Web browser\n3. PDF viewer", + "num_tools": "3", + "time_taken": "5 minutes" + } + }, + { + "pid": "88", + "question": "The longest-lived vertebrate is named after an island. According to Wikipedia as of January 1, 2021, what is the 2020 estimated population of that island, to the nearest thousand?", + "answer": "56000", + "task_id": "3ff6b7a9-a5bd-4412-ad92-0cd0d45c0fee", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Do a web search for \"longest-lived vertebrate\"\n2. Find the answer, \"Greenland shark\"\n3. Find the Wikipedia entry for Greenland\n4. Look at the first revision dated January 1, 2021\n5. Find the 2020 population estimate, 56081\n6. Round to the nearest thousand, 56000", + "num_steps": "6", + "tools": "1. Web browser\n2. Search engine\n3. Access to Wikipedia\n4. Natural language processor", + "num_tools": "4", + "time_taken": "30 minutes" + } + }, + { + "pid": "89", + "question": "On the DeepFruits fruit detection graph on Connected Papers from 2016, what feature caused the largest bubble to be the size it is?", + "answer": "Citations", + "task_id": "708b99c5-e4a7-49cb-a5cf-933c8d46470d", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Searched \"connected papers deepfruits\" on Google search.\n2. Opened the \"DeepFruits: A Fruit Detection System Using Deep Neural Networks\" graph on ConnectedPapers.com.\n3. Clicked on the largest bubble (Redmon, 2015).\n4. Clicked on other bubbles to compare their features.\n5. Noted that Citations was the feature where the Redmon bubble exceeded all the others.", + "num_steps": "5", + "tools": "1. Graph interaction tools\n2. Web browser\n3. Search engine", + "num_tools": "3", + "time_taken": "7 minutes" + } + }, + { + "pid": "90", + "question": "During the first week of August 2015, one of the NASA Astronomy Pictures of the Day shows the lights of a city on the horizon. The namesake of this city also has a landmark building in Chicago named after him. What is the name of the architectural firm that designed this landmark building? Give the first name appearing in the name of the firm as of June 2023.", + "answer": "Holabird", + "task_id": "0a65cb96-cb6e-4a6a-8aae-c1084f613456", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Use search engine to search for \"NASA Astronomy Pictures of the Day August 2015\".\n2. Navigate to the NASA Astronomy Picture of the Day Archive.\n3. Open the Astronomy Picture of the Day for 2015 August 1-7.\n4. Read the descriptions to check which picture shows the lights of a city on the horizon (2015 August 3) and note the name of the city (Marquette, Michigan, USA).\n5. Go to the Wikipedia article for Marquette, Michigan and note that the city was named after Jacques Marquette.\n6. Go to the Wikipedia article for Jacques Marquette and note that the Marquette Building in Chicago was named after him.\n7. Go to the Wikipedia page for the Marquette Building and verify that it is a Chicago landmark.\n8. Read the article and note that it was designed by architects Holabird & Roche.\n9. Go to the Wikipedia page for Holabird & Roche.\n10. Under \"View history\", select the latest version of the page revised during or before June 2023.\n11. Note that the name of the firm is Holabird & Root as of June 2023.", + "num_steps": "11", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "15 minutes" + } + }, + { + "pid": "91", + "question": "How many more blocks (also denoted as layers) in BERT base encoder than the encoder from the architecture proposed in Attention is All You Need?", + "answer": "6", + "task_id": "11af4e1a-5f45-467d-9aeb-46f4bb0bf034", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Search the internet for \"blocks in bert base\"\n2. Examine the search results page to locate the answer (12)\n3. Search the internet for \"attention is all you need layers\"\n4, Navigate to https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf from the search results page\n5. Examine the architecture section of the PDF to locate the answer (12)\n6. Calculate the difference between the two numbers", + "num_steps": "6", + "tools": "1. Web browser\n2. Search engine\n3. Calculator", + "num_tools": "3", + "time_taken": "10 minutes" + } + }, + { + "pid": "92", + "question": "Bob was invited to participate in a game show, and he advanced to the final round. The final round offered Bob the chance to win a large sum by playing a game against the host. The host has 30 shiny prop coins, each of which is worth $1,000 if Bob manages to win them by playing the game. The host hides the coins in three different prize boxes and then shuffles their order. The only rule restricting the host's coin placement is that one box must contain at least 2 coins, and one box must contain 6 more coins than another box. In order to play, Bob must submit three guesses, one guess for the number of coins in each box. The box is then opened and the number of coins is revealed. If Bob's guess is a number greater than the number of coins in the box, Bob earns no coins. If Bob guesses a number equal to or less than the number of coins in the box, Bob wins a number of coins equal to his guess.\n\nIf Bob plays uses the optimal strategy, what's the minimum amount of money he can win from the game?", + "answer": "16000", + "task_id": "e142056d-56ab-4352-b091-b56054bd1359", + "level": "1", + "split": "validation", + "metadata": { + "steps": "Step 1: Evaluate the problem statement provided by my user, storing the relevant information: \n30 coins with a value of $1,000 distributed between 3 boxes.\nEach box must contain at least 2 coins\nOne box must contain 6 more coins than another\n\nStep 2: Evaluate the base distribution: 2-8-20, noting that two boxes must contain at least 8 coins\n\nStep 3: Evaluate the most even allowable distribution: 8,8,14, noting that two boxes must contain at least 8 coins\n\nStep 4: Evaluate a case where Bob guesses 8 for each box in the outlier distributions.\nStep 5: For the worst case 2-8-20 distribution, Bob wins 0+8+8 = 16 coins\nStep 6: For the 8-8-14 distribution, Bob wins 8+8+8 = 24 coins\nStep 7: Convert the worst-case coin count to a prize value, 16*$1,000 = $16,000\nStep 8: Report the correct answer to my user: \"$16,000\"", + "num_steps": "8", + "tools": "1. A calculator", + "num_tools": "1", + "time_taken": "5 minutes" + } + }, + { + "pid": "93", + "question": "Pull out the sentence in the following 5x7 block of text. Read from left to right and use all of the letters in order:\n\nTHESE\nAGULL\nGLIDE\nDPEAC\nEFULL\nYTOMY\nCHAIR", + "answer": "The seagull glided peacefully to my chair.", + "task_id": "50ad0280-0819-4bd9-b275-5de32d3b5bcb", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. I start with the first line, \"T H E S E\" and proceed to the next, \"A G U L L\". At this point, I am able to discern that \"A G U L L\" is probably meant to be \"A GULL\". However, I continue to read through the rest of the lines to get a sense of any other words that might jump out that would substantiate \"A GULL\" being accurate both semantically and syntactically. 2. So now I am on the last line and decide to work backwards. \"CHAIR\" is on the last line all by itself and this does seem a plausible fit as a full word rather than a fragment of another word. When I look to the line directly above \"Y T O M Y\", the word \"my\" jumps out and this is a natural accompaniment to the noun often used to indicate possession. \n3. Eliminating the \"MY\" at the end of \"Y T O MY\" leaves \"Y T O\" remaining in the line and I immediately recognize the preposition \"TO\". It is a this point I am fairly confident that \"TO MY CHAIR\" is most likely accurate. Given that there is only a \"Y\" left, I discern it is more than likely the end of a word located in the row above.\n4. I am now on the fifth row down and am looking at the letters \"E F U L L\" Attaching the \"Y\" left over from the sixth row below I see \"E F U L L Y\" I recognize the word \"FULLY\" I know it can stand alone as an adverb or it can serve as a suffix to a larger adverb.\n5. Detaching the \"FULLY\", leaves the \"E\" alone on the line. Knowing it does not represent a word on its own in the English language, I look to attach it to the line above (row 4).\n6. The fourth row reads \"D P E A C\". Adding the \"E\" to the end, the first word I can separate out is \"ACE\". However \"ACEFULLY\" is not a word nor does \"ACE FULLY TO MY CHAIR\" make sense. When working my way left through the line, continuing to attach each letter as I go, I land on the \"P\" and am fairly confident that the word is \"PEACEFULLY\".\n7. Eliminating the \"PEAC\" from the row leaves me left with a \"D\". Now I look at the row above, row 3 and see that the row comprises the word \"GLIDE\" Adding the \"D\" to the end of the word would not only be permissible in terms of a displaying appropriate tense but it also makes sense as I add it to the fragment I have so far. I now can read \"GLIDED PEACEFULLY TO MY CHAIR\".\n8. Now, I am on the second line and if I were to read it from there on down it would read \"A GULL GLIDED PEACEFULLY TO MY CHAIR\". While this reads well and makes sense semantically and syntactically on its own, it does not make sense when I add the first row. THESE A GULL GLIDED PEACEFULLY TO MY CHAIR. So now I am left with the conclusion that \"A GULL\" is not correct. Either it is part of a larger word or the letters need to be broken down further. At a quick glace, I can see that they don't make sense being broken down further so I leave \"GULL\" and add the \"A\" to the string above. Immediately my eye sees that \"A can be added to \"SE\" to make \"SEA\" and that the remaining\nletters spell the word \"THE\" I now know the sentence reads \"The seagull glided peacefully to my chair.", + "num_steps": "8", + "tools": "None", + "num_tools": "0", + "time_taken": "a few minutes at most" + } + }, + { + "pid": "94", + "question": "All of the individuals who formally held the position of United States secretary of homeland security prior to April 2019, excluding those who held the position in an acting capacity, have a bachelor's degree. Of the universities that these bachelor's degrees were from, which is the westernmost university and which is the easternmost university? Give them to me as a comma-separated list, I only want the name of the cities where the universities are located, with the westernmost city listed first.", + "answer": "Santa Clara, Boston", + "task_id": "65da0822-a48a-4a68-bbad-8ed1b835a834", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Go to the Wikipedia page for \"United States secretary of homeland security\".\n2. Open the Wikipedia pages for each person who held the position of United States secretary of homeland security in a non-acting capacity prior to April 2019.\n3. Using the infobox on each person's Wikipedia page, open the Wikipedia page for the university from which each person received a bachelor's degree (bachelor's degree indicated by AB, BA, or BS).\n4. Comparing the longitude coordinates for each university given on their Wikipedia pages, note that Santa Clara University is the westernmost as it has the highest longitude value in degrees W.\n5. Note that the easternmost is either Harvard University or University of Massachusetts Boston, but the longitude for Harvard University is expressed in degrees, minutes, and seconds (71°07′01″W) while the longitude for University of Massachusetts Boston is expressed in decimal degrees (71.038445°W), requiring conversion to determine which is further east.\n6. Convert 71°07′01″W to decimal degrees using the formula [decimal degrees] = [degrees] + [minutes] / 60 + [seconds] / 3600 to get approximately 71.1169°W for Harvard's longitude, which is further west than the University of Massachusetts Boston's longitude.\n7. Use determined westernmost and easternmost university names to produce the final answer: Santa Clara University, University of Massachusetts Boston", + "num_steps": "7", + "tools": "1. Web browser\n2. Calculator", + "num_tools": "2", + "time_taken": "15 minutes" + } + }, + { + "pid": "95", + "question": "Consider the following symbols: 𒐜 𒐐𒐚\n\nThis is a number written using the Mesopotamian/Babylonian number system and represented with Sumerian cuneiform. Convert this number into Arabic numerals as a decimal number.", + "answer": "536", + "task_id": "0bb3b44a-ede5-4db5-a520-4e844b0079c5", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Look up Babylonian number system (base 60, using uniform 'hashmarks' as counters)\n2. Converted the Cuniform to Arabic (8 56)\n3. Since Babylonian is a base 60 system, converted the \"60\"'s place to decimal (8*60=480)\n4. Added 56 to 480 (536).", + "num_steps": "4", + "tools": "1. Bablyonian cuniform -> arabic legend", + "num_tools": "1", + "time_taken": "10 minutes" + } + }, + { + "pid": "96", + "question": "On Cornell Law School website's legal information institute, under the fifth section of federal rules alphabetically, what word was deleted in the last amendment to the first rule in the article that has \"witnesses\" in the most titles as of 2021?", + "answer": "inference", + "task_id": "7673d772-ef80-4f0f-a602-1bf4485c9b43", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Searched \"Cornell Law School legal information institute\" on Google.\n2. Opened https://www.law.cornell.edu/.\n3. Clicked Get The Law > Federal Rules > Federal Rules of Evidence (fourth section down).\n4. Found the article that has \"witnesses\" in the most titles (VII).\n5. Opened the first rule (701).\n6. Scrolled to the last amendment as of 2021 (2011 amendment).\n7. Found the word that was deleted (inference).", + "num_steps": "7", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "10 minutes" + } + }, + { + "pid": "97", + "question": "According to the USGS, in what year was the American Alligator first found west of Texas (not including Texas)?", + "answer": "1954", + "task_id": "73c1b9fe-ee1d-4cf4-96ca-35c08f97b054", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “American Alligator USGS”.\n2. Click result for the USGS Species Profile.\n3. Click “Animated Map”.\n4. Click the “Skip years with no recorded sightings” button.\n5. Zoom out on the map to better view the whole U.S.\n6. Move the slider back to the beginning, then advance it until I see a red dot pop up west of Texas.\n7. Note the year that the dot appears, 1954.", + "num_steps": "7", + "tools": "1. Search engine\n2. Web browser\n3. Image recognition", + "num_tools": "3", + "time_taken": "5 minutes" + } + }, + { + "pid": "98", + "question": "Of the cities within the United States where U.S. presidents were born, which two are the farthest apart from the westernmost to the easternmost going east, giving the city names only? Give them to me in alphabetical order, in a comma-separated list", + "answer": "Braintree, Honolulu", + "task_id": "c365c1c7-a3db-4d5e-a9a1-66f56eae7865", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Searched \"cities where us presidents are born\" on Google.\n2. Opened \"List of presidents of the United States by home state\" on Wikipedia.\n3. Searched the eastern cities to find the easternmost one (Braintree, MA).\n4. Checked the westernmost city (Honolulu, HI).", + "num_steps": "4", + "tools": "1. Search engine\n2. Web browser", + "num_tools": "3", + "time_taken": "8 minutes" + } + }, + { + "pid": "99", + "question": "Eva Draconis has a personal website which can be accessed on her YouTube page. What is the meaning of the only symbol seen in the top banner that has a curved line that isn't a circle or a portion of a circle? Answer without punctuation.", + "answer": "War is not here this is a land of peace", + "task_id": "ad2b4d70-9314-4fe6-bfbe-894a45f6055f", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. By googling Eva Draconis youtube, you can find her channel.\n2. In her about section, she has written her website URL, orionmindproject.com.\n3. Entering this website, you can see a series of symbols at the top, and the text \"> see what the symbols mean here\" below it.\n4. Reading through the entries, you can see a short description of some of the symbols.\n5. The only symbol with a curved line that isn't a circle or a portion of a circle is the last one.\n6. Note that the symbol supposedly means \"War is not here, this is a land of peace.\"", + "num_steps": "6", + "tools": "1. A web browser.\n2. A search engine.\n3. Access to YouTube\n4. Image recognition tools", + "num_tools": "4", + "time_taken": "30 minutes." + } + }, + { + "pid": "100", + "question": "According to Girls Who Code, how long did it take in years for the percentage of computer scientists that were women to change by 13% from a starting point of 37%?", + "answer": "22", + "task_id": "7d4a7d1d-cac6-44a8-96e8-ea9584a70825", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Searched \"Girls Who Code\" on Google.\n2. Opened https://girlswhocode.com/.\n3. Clicked \"About Us\".\n4. Noted that the chart started at 37% and declined to 24%.\n5. Subtracted the marked years to find the number of years (2017 - 1995 = 22).", + "num_steps": "5", + "tools": "1. Web browser\n2. Search engine\n3. Calculator", + "num_tools": "3", + "time_taken": "10 minutes" + } + }, + { + "pid": "101", + "question": "What was the complete title of the book in which two James Beard Award winners recommended the restaurant where Ali Khan enjoyed a New Mexican staple in his cost-conscious TV show that started in 2015? Write the numbers in plain text if there are some in the title.", + "answer": "Five Hundred Things To Eat Before It's Too Late: and the Very Best Places to Eat Them", + "task_id": "dc22a632-937f-4e6a-b72f-ba0ff3f5ff97", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Searched \"Ali Khan New Mexico staple TV show\" on Google.\n2. Opened \"Albuquerque | Cheap Eats\" at https://www.cookingchanneltv.com/shows/cheap-eats/episodes/albuquerque.\n3. Noted the New Mexico staple and the list of restaurants.\n4. Searched \"Albuquerque Cheap Eats carne avodava\" on Google.\n5. Confirmed the restaurant name (Papa Felipe's) from the results.\n6. Searched \"James Beard Award winners Papa Felipe's\" on Google.\n7. Opened \"Papa Felipe's Mexican Restaurant - Albuquerque, New ...\" at https://www.nmgastronome.com/?p=4572.\n8. Clicked the link on the book title.\n9. Copied the full book title from Amazon.", + "num_steps": "9", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "15 minutes" + } + }, + { + "pid": "102", + "question": "As of August 2023, who is the only winner of the US version of Survivor to be born in the month of May?", + "answer": "Michele Fitzgerald", + "task_id": "e2d69698-bc99-4e85-9880-67eaccd66e6c", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Google \"American Survivor Winners\". Scroll down to the Wikipedia listing \"Survivor (American TV Series)\".\n Search, https://en.wikipedia.org/wiki/Survivor_(American_TV_series), \n2.I begin to make a list of all the Survivor winners and their seasons. \n3.I google \"survivor cast CBS\" and click on cast tab at cbs.com (https://www.cbs.com/shows/survivor/cast/). It features the players of the most recently aired season. I click on the Seasons tab and scroll down to the first season. I find the winner from the first season (based on my list compiled from the en.wikipedia.org site mentioned in step 1) and scroll through the bio information until I see the mention of their birthday. It is usually contained in the last sentence of the bio. I repeat this process until I get to Season 18. It is at this point that CBS starts to omit the full birthdays. For seasons 18 and 19 they include the month and date but omit the year. By Season 20, the birthday is omitted completely. \n4. So now I am making a simple template entry in google for each successive winner: When was (insert winner's name), winner of (insert season they won) of Survivor born? There are usually two prominent sites I look for in my Google feed for this information:\n\n 1. Wikipedia page for that contestant: ex.: https://en.wikipedia.org/wiki/J._T._Thomas_(Survivor_contestant)\n 2. Survivor Wiki: ex.: https://survivor.fandom.com/wiki/J.T._Thomas \n Overall I have found the fan pages to be pretty reliable. If both options were available, I did take the opportunity to verify \n that they matched up. I did not find any discrepancies (as far as birthdays) between the two.\n\n5. Now I have a list of all forty of the winners from the first forty seasons of Survivor (two of them have won twice). I comb the list and \nnote the months when they are mentioned and how many times that they appear. Michele Fitzgerald, the winner of Season 32 of Survivor, is the only listed with a birthday in May.", + "num_steps": "I have five main processes listed but the individual steps for each winner (and any confirmation searches) would place it into the 40-60 range.", + "tools": "1. web browser\n2. search engine", + "num_tools": "2", + "time_taken": "65 minutes" + } + }, + { + "pid": "103", + "question": "How many at bats did the Yankee with the most walks in the 1977 regular season have that same season?", + "answer": "519", + "task_id": "3f57289b-8c60-48be-bd80-01f8099ca449", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Search \"yankee stats\" to find their MLB stats page.\n2. Set the data to the 1977 regular season.\n3. Sort to find the most walks.\n4. See how many at bats the player had.", + "num_steps": "4", + "tools": "1. web browser\n2. search engine", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "104", + "question": "The cover of the August 2021 issue of Vogue shows a famous landmark in the background behind some trees. How tall is this monument in yards, rounded to the nearest yard? Give the number only.", + "answer": "185", + "task_id": "a56f1527-3abf-41d6-91f8-7296d6336c3f", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Use search engine to search for \"Vogue August 2021 cover\".\n2. Find the result from Vogue's archive for the August 2021 issue and go to the webpage.\n3. Identify the monument in the cover image as the Washington Monument.\n4. Go to the Wikipedia page for the Washington Monument.\n5. In the infobox, note that the height is 555 ft. \n6. Convert 555 ft to yards using a conversion factor of 1 yd / 3 ft: 555 ft * 1 yd / 3 ft = 185 yd, giving a final answer of 185.", + "num_steps": "6", + "tools": "1. Web browser\n2. Search engine\n3. Image recognition tools\n4. Calculator", + "num_tools": "4", + "time_taken": "5 minutes" + } + }, + { + "pid": "105", + "question": "In Audre Lorde’s poem “Father Son and Holy Ghost”, what is the number of the stanza in which some lines are indented?", + "answer": "2", + "task_id": "23dd907f-1261-4488-b21c-e9185af91d5e", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “Audre Lorde Father Son and Holy Ghost”.\n2. Click on Poetry Foundation result.\n3. Note the stanza that appears to have lines indented, the second one.\n4. Return to search results to confirm.\n5. Click on second result.\n6. Confirm that the indentation appears in the second stanza here as well.", + "num_steps": "6", + "tools": "1. Search engine\n2. Web browser", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "106", + "question": "I'm curious about how much information is available for popular video games before their release. Find the Wikipedia page for the 2019 game that won the British Academy Games Awards. How many revisions did that page have before the month listed as the game's release date on that Wikipedia page (as of the most recent entry from 2022)?", + "answer": "60", + "task_id": "42d4198c-5895-4f0a-b0c0-424a66465d83", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for British Academy Video Games Award for Best Game 2019\n2. Find the answer, Outer Wilds\n3. Find the Wikipedia page for Outer Wilds\n4. Go to the last revision from 2022.\n5. Note the release date, May 29, 2019\n6. View the page history\n7. Count how many edits were made to the page before May 2019\n8. Arrive at the answer, 60", + "num_steps": "8", + "tools": "1. Web browser\n2. Search engine\n3. Access to Wikipedia\n4. Calculator or counting function", + "num_tools": "4", + "time_taken": "30 minutes" + } + }, + { + "pid": "107", + "question": "What is the absolute difference in tens of thousands between the population of chinstrap penguins on the Wikipedia page for penguin species populations as of the end of 2018 and the population recorded in the Nature.com \"global population assessment of the Chinstrap penguin\" article from 2020, assuming two penguins per breeding pair?", + "answer": "116", + "task_id": "a26649c6-1cb2-470a-871e-6910c64c3e53", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Searched \"penguin species populations wikipedia\" on Google search.\n2. Opened the \"List of Sphenisciformes by population\" Wikipedia article.\n3. Clicked \"View history\".\n4. Scrolled to the end of 2018 and opened the page.\n5. Scrolled to the encoding for the population table.\n6. Recorded the number of chinstrap penguins (8 million).\n7. Searched \"Nature.com global population assessment of the Chinstrap penguin 2020\" in Google search.\n8. Opened the top link to the article with the corresponding name and date.\n9. Read the abstract and noted the number of breeding pairs (3.42 million).\n10. Multiplied the breeding pairs by 2 to get the number of penguins (6.84 million).\n11. Subtracted the Wikipedia population from the Nature.com population (1.16 million).\n12. Multiplied 1.16 by 100 to get tens of thousands (116).", + "num_steps": "12", + "tools": "1. Search engine\n2. Web browser\n3. Calculator", + "num_tools": "3", + "time_taken": "20 minutes" + } + }, + { + "pid": "108", + "question": "When was a picture of St. Thomas Aquinas first added to the Wikipedia page on the Principle of double effect? Answer using the format DD/MM/YYYY.", + "answer": "19/02/2009", + "task_id": "d5141ca5-e7a0-469f-bf3e-e773507c86e2", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “principle of double effect wikipedia”.\n2. Note a picture of St. Thomas Aquinas on the page, which is part of the Wikipedia “series on” template.\n3. Click “View history” to see the page’s revision history.\n4. Click to display more edits on the page.\n5. Ctrl-F for “template”.\n6. Browse the mentions of “template” until I find the revision that added the picture.\n7. Note the date that the template was added, 19 February 2009.\n8. Browse earlier revisions to ensure that a picture was not added earlier. ", + "num_steps": "8", + "tools": "1. Search engine\n2. Web browser\n3. Image recognition", + "num_tools": "3", + "time_taken": "10 minutes" + } + }, + { + "pid": "109", + "question": "A 5-man group made up of one tank, one healer, and three DPS is doing a dungeon that was just released in World of Warcraft. Two are plate wearers and two are cloth wearers. At the final boss, both the tank and the healer are casting holy spells. Ice and fire are being used, each one by a different DPS. A bear from the group is attacking the boss. Metamorphosis is cast. The Kilt of the Forgotten One drops as loot, but no one can use it. If all classes were using their class abilities and all classes are unique, what are the five classes in the group in alphabetical order separated by commas?", + "answer": "Death Knight, Hunter, Paladin, Priest, Warlock", + "task_id": "9e1fc53b-46ff-49a1-9d05-9e6faac34cc5", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Searched \"WoW classes\" on Google.\n2. Opened \"https://worldofwarcraft.blizzard.com/en-us/game/classes\".\n3. Made an alphabetical list of all WoW classes: Death Knight, Demon Hunter, Druid, Evoker, Hunter, Mage, Monk, Paladin, Priest, Rogue, Shaman, Warlock, and Warrior.\n4. Opened each page and noted the armor type: Death Knight (plate), Demon Hunter (leather), Druid (leather), Evoker (mail), Hunter (mail), Mage (cloth), Monk (leather), Paladin (plate), Priest (cloth), Rogue (leather), Shaman (mail), Warlock (cloth), and Warrior (plate).\n5. Looked up \"Kilt of the Forgotten One\" on Google.\n6. Opened https://www.wowhead.com/wotlk/item=37616/kilt-of-the-forgotten-one.\n7. Noted that it is leather, and none of the classes can use it, so the remaining classes are: Death Knight (plate), Evoker (mail), Hunter (mail), Mage (cloth), Paladin (plate), Priest (cloth), Shaman (mail), Warlock (cloth), and Warrior (plate).\n8. Noted that it was added in Wrath of the Lich King, so if the dungeon is newly released, the era is the Wrath of the Lich King expansion.\n9. Searched \"Wrath of the Lich King class abilities\" on Google.\n10. Opened https://www.wowhead.com/wotlk/spells/abilities.\n11. Sorted by class and noted that Evokers, Demon Hunters, and Monks did not exist yet, so the remaining classes are: Death Knight (plate), Hunter (mail), Mage (cloth), Paladin (plate), Priest (cloth), Shaman (mail), Warlock (cloth), and Warrior (plate).\n12. Checked which classes use Holy school abilities, Paladin (plate) and Priest (cloth), so they must be in the group as tank and healer.\n13. Checked which classes use ice (Frost) and fire abilities, Death Knight (plate), Mage (cloth), Shaman (mail), and Warlock (cloth).\n14. There can only be one other plate class, so it must be Death Knight or Warrior, and one other cloth class, so it must be Mage or Warlock.\n15. Metamorphosis is a Warlock ability in Wrath of the Lich King, so it must be the other cloth class, and the group so far is Paladin, Priest, Warlock, plate DPS, and other DPS, with remaining options of Death Knight (plate), Hunter (mail), Mage (cloth), Shaman (mail), and Warrior (plate).\n16. There cannot be another cloth class, so the remaining options are Death Knight (plate), Hunter (mail), Shaman (mail), and Warrior (plate).\n17. There is a bear attacking the boss and there is no Druid to shapeshift into a bear, so it must be a Hunter's pet, making the group Paladin, Priest, Warlock, Hunter, and plate DPS, with remaining options of Death Knight (plate), Hunter (mail), Mage (cloth), Shaman (mail), and Warrior (plate).\n18. The last class is plate, leaving only Death Knight and Warrior.\n19. Hunters and Warlocks can both cast Fire abilities but cannot cast Frost abilities, so the last DPS must cast ice (Frost) abilities, making the last DPS a Frost Death Knight since Warriors have no Frost abilities.\n20. Order the group alphabetically: Death Knight, Hunter, Paladin, Priest, Warlock.", + "num_steps": "20", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "20 minutes" + } + }, + { + "pid": "110", + "question": "On June 6, 2023, an article by Carolyn Collins Petersen was published in Universe Today. This article mentions a team that produced a paper about their observations, linked at the bottom of the article. Find this paper. Under what NASA award number was the work performed by R. G. Arendt supported by?", + "answer": "80GSFC21M0002", + "task_id": "840bfca7-4f7b-481a-8794-c560c340185d", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Google \"June 6, 2023 Carolyn Collins Petersen Universe Today\"\n2. Find the relevant link to the scientific paper and follow that link\n3. Open the PDF. \n4. Search for NASA award number", + "num_steps": "4", + "tools": "1. Web browser\n2. Search engine\n3. Access to academic journal websites", + "num_tools": "2", + "time_taken": "5 minutes" + } + }, + { + "pid": "111", + "question": "According to Openreview.net, at the NeurIPS 2022 Conference, how many papers by an author named Yuri were accepted with a \"certain\" recommendation?", + "answer": "3", + "task_id": "1dcc160f-c187-48c2-b68e-319bd4354f3d", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Went to openreview.net.\n2. Scroll down and clicked the \"All venues\" link.\n3. Clicked \"NeurIPS\".\n4. Opened the \"2022\" toggle menu.\n5. Clicked \"NeurIPS 2022 Conference\".\n6. Opened the top paper.\n7. Clicked \"Go to NeurIPS 2022 Conference homepage\".\n8. Searched \"Yuri\" in the search box.\n9. Opened each of the four papers and checked the Recommendation field.\n10. Counted the \"Certain\" recommendations.", + "num_steps": "8", + "tools": "1. Web browser\n2. Search engine", + "num_tools": "2", + "time_taken": "10 minutes" + } + }, + { + "pid": "112", + "question": "Take the gender split from the 2011 Bulgarian census about those who have completed tertiary education. Subtract the smaller number from the larger number, then return the difference in thousands of women. So if there were 30.1 thousand more men, you'd give \"30.1\"", + "answer": "234.9", + "task_id": "e0c10771-d627-4fd7-9694-05348e54ee36", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Find the report put out by the Bulgarian on the 2011 census by searching.\n2. Find the requested data under the Educational Structure Section of the Report.\n3. 791.8 thousand women - 556.9 thousand men = 234.9 thousand women", + "num_steps": "3", + "tools": "1. search engine\n2. pdf reader/extracter", + "num_tools": "2", + "time_taken": "10 minutes" + } + }, + { + "pid": "113", + "question": "What was the actual enrollment count of the clinical trial on H. pylori in acne vulgaris patients from Jan-May 2018 as listed on the NIH website?", + "answer": "90", + "task_id": "a0068077-79f4-461a-adfe-75c1a4148545", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Searched \"nih\" on Google search.\n2. Clicked the top link to nih.gov.\n3. Searched \"h pylori acne\" in the search box.\n4. Clicked \"More\" and selected \"Clinical Trials\".\n5. Clicked the result about H. Pylori and acne.\n6. Checked the date to confirm it was January to May 2018.\n7. Opened \"Tabular View\".\n8. Scrolled down to Actual Enrollment and recorded the number.", + "num_steps": "8", + "tools": "1. Search engine\n2. Web browser", + "num_tools": "2", + "time_taken": "8 minutes" + } + }, + { + "pid": "114", + "question": "I'd like to learn more about some popular reality television competition shows. As of the end of the 44th season of the American version of Survivor, how many more unique winners have there been compared to the number of winners of American Idol?", + "answer": "21", + "task_id": "e29834fd-413a-455c-a33e-c3915b07401c", + "level": "2", + "split": "validation", + "metadata": { + "steps": "Step 1: Using a web browser, access a search engine and conduct a search \"American Survivor Television Series winners\"\nStep 2: Navigate to the first result, https://en.wikipedia.org/wiki/Survivor_(American_TV_series)\nStep 3: Evaluate the article and count the number of unique winners of the program: 42 winners\nStep 4: Navigate back to a search engine and conduct a search \"American Idol Winners\"\nStep 5: Navigate to the first search result, https://www.etonline.com/gallery/the-complete-list-of-american-idol-winners-21116/season-21-iam-tongi-92872\nStep 6: Evaluate the article and count the number of unique winners of the program: 21\nStep 7: Using a calculator, subtract the number of American Idol winners from the number of Survivor winners, 42-21 = 21\nStep 8: Report the correct response to my user, \"21\"", + "num_steps": "8", + "tools": "1. A web browser\n2. A search engine\n3. A calculator", + "num_tools": "3", + "time_taken": "5 minutes" + } + }, + { + "pid": "115", + "question": "Where were the Vietnamese specimens described by Kuznetzov in Nedoshivina's 2010 paper eventually deposited? Just give me the city name without abbreviations.", + "answer": "Saint Petersburg", + "task_id": "bda648d7-d618-4883-88f4-3466eabd860e", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Search \"Kuznetzov Nedoshivina 2010\"\n2. Find the 2010 paper \"A catalogue of type specimens of the Tortricidae described by V. I. Kuznetzov from Vietnam and deposited in the Zoological Institute, St. Petersburg\"", + "num_steps": "2", + "tools": "1. search engine", + "num_tools": "1", + "time_taken": "5 minutes" + } + }, + { + "pid": "116", + "question": "A standard Rubik’s cube has been broken into cubes making up its sides. The cubes are jumbled, and one is removed. There are 6 cubes with one colored face, 12 edge cubes with two colored faces, and 8 corner cubes with three colored faces. All blue cubes have been found. All cubes directly left, right, above, and below the orange center cube have been found, along with the center cube. The green corners have all been found, along with all green that borders yellow. For all orange cubes found, the opposite face’s cubes have been found. The removed cube has two colors on its faces. What are they? Answer using a comma separated list, with the colors ordered alphabetically.", + "answer": "green, white", + "task_id": "50ec8903-b81f-4257-9450-1085afd2c319", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Set up a standard Rubik's cube (red opposite orange, white opposite yellow, green opposite blue).\n2. Eliminated blue cubes, along with adjacent colors.\n3. Eliminated orange cubes, along with adjacent colors.\n4. Eliminated green corners and the green/yellow edge.\n5. Eliminated red, opposite of orange, cubes and adjacent colors.\n6. Identified the last possible two-face cube.", + "num_steps": "6", + "tools": "1. Rubik's cube model", + "num_tools": "1", + "time_taken": "10 minutes" + } + }, + { + "pid": "117", + "question": "What country had the least number of athletes at the 1928 Summer Olympics? If there's a tie for a number of athletes, return the first in alphabetical order. Give the IOC country code as your answer.", + "answer": "CUB", + "task_id": "cf106601-ab4f-4af9-b045-5295fe67b37d", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Look up the 1928 Summer Olympics on Wikipedia\n2. Look at a table of athletes from countries.\n3. See that two countries had 1 and 2 athletes, so disregard those and choose the Cuba as CUB.", + "num_steps": "3", + "tools": "None", + "num_tools": "0", + "time_taken": "5 minutes" + } + }, + { + "pid": "118", + "question": "I read a paper about multiwavelength observations of fast radio bursts back in March 2021 on Arxiv, and it had a fascinating diagram of an X-ray time profile. There was a similar burst-1 diagram in another paper from one of the same authors about fast radio bursts back in July 2020, but I can't recall what the difference in seconds in the measured time span was. How many more seconds did one measure than the other? Just give the number.", + "answer": "0.2", + "task_id": "5f982798-16b9-4051-ab57-cfc7ebdb2a91", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Searched \"arxiv\" on Google.\n2. Opened arXiv.\n3. Searched \"multiwavelength observations of fast radio bursts\" on arXiv.\n4. Scrolled down to March 2021.\n5. Opened the \"Multiwavelength observations of Fast Radio Bursts\" PDF in a new tab.\n6. Opened each author's name to find the one that had a July 2020 paper (Nicastro, L).\n7. Opened the \"The lowest frequency Fast Radio Bursts: Sardinia Radio Telescope detection of the periodic FRB 180916 at 328 MHz\" PDF.\n8. Searched \"time profile\" in the first paper.\n9. Noted the time span of the diagram (0.3 s).\n10. Searched \"burst-1 profile\" in the second paper.\n11. Noted the time span of the diagram (0.5 s).\n12. Subtracted the two (0.5 - 0.3 = 0.2 s).", + "num_steps": "12", + "tools": "1. PDF access\n2. Calculator\n3. Web browser\n4. Search engine", + "num_tools": "4", + "time_taken": "15 minutes" + } + }, + { + "pid": "119", + "question": "Who are the pitchers with the number before and after Taishō Tamai's number as of July 2023? Give them to me in the form Pitcher Before, Pitcher After, use their last names only, in Roman characters.", + "answer": "Yoshida, Uehara", + "task_id": "a0c07678-e491-4bbc-8f0b-07405144218f", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Look up Taishō Tamai on Wikipedia\n2. See the pitcher with the number 18 (before) is Kōsei Yoshida and number 20 (after) is Kenta Uehara", + "num_steps": "2", + "tools": "1. Wikipedia", + "num_tools": "1", + "time_taken": "5 minutes" + } + }, + { + "pid": "120", + "question": "What is the first name of the only Malko Competition recipient from the 20th Century (after 1977) whose nationality on record is a country that no longer exists?", + "answer": "Claus", + "task_id": "5a0c1adf-205e-4841-a666-7c3ef95def9d", + "level": "1", + "split": "validation", + "metadata": { + "steps": "1. Look at the Malko Competition page on Wikipedia\n2. Scan the winners to see that the 1983 winner, Claus Peter Flor is stated to be from East Germany.", + "num_steps": "2", + "tools": "None", + "num_tools": "0", + "time_taken": "5-10 minutes" + } + }, + { + "pid": "121", + "question": "In the YouTube 360 VR video from March 2018 narrated by the voice actor of Lord of the Rings' Gollum, what number was mentioned by the narrator directly after dinosaurs were first shown in the video?", + "answer": "100000000", + "task_id": "0512426f-4d28-49f0-be77-06d05daec096", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Searched \"gollum voice actor\" on Google search.\n2. Noted the answer.\n3. Searched \"youtube 360 vr andy serkis\" on Google search.\n4. Opened the top result (We Are Stars with Andy Serkis - 360 VR Video).\n5. Confirmed the date was in March 2018.\n6. Watched the video until dinosaurs appeared (approximately 8:45).\n7. Recorded the narrated number.", + "num_steps": "7", + "tools": "1. Search engine\n2. Web browser\n3. Audio capability\n4. Video capability", + "num_tools": "4", + "time_taken": "15 minutes" + } + }, + { + "pid": "122", + "question": "In NASA's Astronomy Picture of the Day on 2006 January 21, two astronauts are visible, with one appearing much smaller than the other. As of August 2023, out of the astronauts in the NASA Astronaut Group that the smaller astronaut was a member of, which one spent the least time in space, and how many minutes did he spend in space, rounded to the nearest minute? Exclude any astronauts who did not spend any time in space. Give the last name of the astronaut, separated from the number of minutes by a semicolon.", + "answer": "White; 5876", + "task_id": "0bdb7c40-671d-4ad1-9ce3-986b159c0ddc", + "level": "3", + "split": "validation", + "metadata": { + "steps": "1. Use search engine to search for \"NASA's Astronomy Picture of the Day 2006 January 21\".\n2. Open the link to the image.\n3. Read the explanation to find that the image is of astronaut Charles \"Pete\" Conrad reflected in the helmet of astronaut Alan Bean.\n4. Observe that the smaller astronaut in the image is the one reflected in the other's helmet, so the smaller astronaut must be Charles \"Pete\" Conrad.\n5. Go to the Wikipedia page for Charles \"Pete\" Conrad.\n6. Search for \"Astronaut Group\" to find that Conrad was a member of NASA Astronaut Group 2.\n7. Open the Wikipedia pages for each member of NASA Astronaut Group 2.\n8. For those who are not deceased, go to View history and select the latest version of their Wikipedia page as of August 2023.\n9. Compare the times listed in the infobox of each astronaut's Wikipedia page under \"Time in space\", observing that Ed White has the least time in space with 4d 01h 56m, but also that Elliott See does not have a listed \"Time in space\".\n10. Read through Elliot See's Wikipedia article to find that he died in an accident before his first space flight, so he should be excluded, making Ed White's 4d 01h 56m the least amount of time in space.\n11. Convert 4d 01h 56m to minutes: 4d * 24h/d * 60m/h + 1h * 60m/h + 56m = 5,876m\n12. Format the final answer as specified: White; 5,876", + "num_steps": "12", + "tools": "1. Web browser\n2. Search engine\n3. Image processing tools\n4. Calculator", + "num_tools": "4", + "time_taken": "10" + } + }, + { + "pid": "123", + "question": "In the film Goldfinger, what color was the object that James Bond concealed himself and his companion Pussy Galore at the end of the film? If there are multiple colors, put them in a comma-separated list in alphabetical order.", + "answer": "orange, white", + "task_id": "08c0b6e9-1b43-4c2e-ae55-4e3fce2c2715", + "level": "2", + "split": "validation", + "metadata": { + "steps": "Step 1: Conduct a web search for the Goldfinger film screenplay.\nStep 2: Navigate to the top result, https://www.universalexports.net/scripts/goldfinger.pdf\nStep 3: Review the screenplay pdf. Navigate to the final page of the screenplay, looking for mentions and combinations of \"conceal\" \"James\" \"James Bond\" \"Pussy\" \"Pussy Galore\"\nStep 4: After reviewing the line: \"Bond grabs the edge of the parachute and pulls it over them.\" search the rest of the screenplay for any description of the parachute.\nStep 5: Failing to locate a description of the parachute in the screenplay, conduct a web search for \"James Bond Goldfinger parachute\"\nStep 6: Navigate to the English language Wikipedia article for the film, Goldfinger (film), https://en.wikipedia.org/wiki/Goldfinger_(film)\nStep 7: Review the article for information regarding the parachute used to conceal the characters at the end of the film.\nStep 8: Failing to locate a description of the parachute, conduct a web search for \"James Bond Goldfinger parachute image\"\nStep 9: Navigate to the Wikimedia.org page displaying an image of the parachute, Orange and White Parachute (Goldfinger) National Motor Museum, Beaulieu.jpg, https://commons.wikimedia.org/wiki/File:Orange_and_White_Parachute_(Goldfinger)_National_Motor_Museum,_Beaulieu.jpg\nStep 10: Evaluate the image to determine its color, orange and white.\nStep 11: Review the text summary of the image for confirmation of the details shown in the image.\nStep 12: Return the requested information: \"orange, white\"", + "num_steps": "12", + "tools": "A web browser\nA search engine\nImage recognition software", + "num_tools": "3", + "time_taken": "3 minutes" + } + }, + { + "pid": "124", + "question": "As of May 2023, how many stops are between South Station and Windsor Gardens on MBTA’s Franklin-Foxboro line (not included)?", + "answer": "10", + "task_id": "db4fd70a-2d37-40ea-873f-9433dc5e301f", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “MBTA Franklin Foxboro line”.\n2. Click on top result, on the MBTA website.\n3. Scroll down on the list of stops, and count the current stops between South Station and Windsor Gardens.\n4. Click the “Schedule & Maps” tab to view a map of the route.\n5. Examine the map to confirm that the order of stops is the same as on the listing of stops.\n6. Return to web search.\n7. Click on Wikipedia article for Franklin line.\n8. Read the article to check whether any stops were added or removed since the date given in the question.\n9. Search the web for “MBTA Franklin Foxboro Line changes”.\n10. Click News tab.\n11. Click article about rail schedule changes.\n12. Confirm that none of the changes affect the answer to the question.", + "num_steps": "12", + "tools": "1. Search engine\n2. Web browser", + "num_tools": "2", + "time_taken": "5-10 minutes" + } + }, + { + "pid": "125", + "question": "In the 2015 Metropolitan Museum of Art exhibition titled after the Chinese zodiac animal of 2015, how many of the \"twelve animals of the Chinese zodiac\" have a hand visible?", + "answer": "11", + "task_id": "853c8244-429e-46ca-89f2-addf40dfb2bd", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search \"2015 Chinese zodiac animal\" on Google search.\n2. Note the animal (ram).\n3. Search \"Metropolitan Museum of Art\" on Google search.\n4. Open the Metropolitan Museum of Art website.\n5. Click \"Exhibitions\" under \"Exhibitions and Events\" \n6. Click \"Past\".\n7. Set the year to 2015.\n8. Scroll to find the exhibit mentioning rams and click \"Celebration of the Year of the Ram\".\n9. Click \"View All Objects\".\n10. Click \"Twelve animals of the Chinese zodiac\" to open the image.\n11. Count how many have a visible hand.", + "num_steps": "11", + "tools": "1. Web browser\n2. Search engine\n3. Image recognition tools", + "num_tools": "3", + "time_taken": "10 minutes" + } + }, + { + "pid": "126", + "question": "At the two-minute mark in the YouTube video uploaded by the channel “GameGrumps” on May 14, 2017 as part of their playthrough of the game Mario Kart 8 Deluxe, the shows’ hosts are competing on one of the game’s racetracks. What was the world record time for that track in the game’s 150cc mode as of June 7, 2023? Express your answer in minutes and seconds, rounding the seconds to the nearest hundredth, e.g. 1:01.001.", + "answer": "1:41.614", + "task_id": "7a4a336d-dcfa-45a0-b014-824c7619e8de", + "level": "2", + "split": "validation", + "metadata": { + "steps": "1. Search the web for “gamegrumps mario kart 8 deluxe may 14 2017”.\n2. Click on the YouTube video result.\n3. Navigate to two minutes into the video.\n4. Scroll further back until I see the name of the racecourse, Yoshi Circuit.\n5. Search the web for “mario kart 8 deluxe yoshi circuit world record 150cc”\n6. Scroll down until I find a reliable world record listing site.\n7. Navigate through the site until I find the record that meets the specified criteria.\n8. Read the date the record was set to confirm that it applies to the question’s specified date.", + "num_steps": "8", + "tools": "1. Search engine\n2. Web browser\n3. YouTube\n4. OCR", + "num_tools": "4", + "time_taken": "5-10 minutes" + } + } +] \ No newline at end of file diff --git a/data/gaia/val.parquet b/data/gaia/val.parquet new file mode 100644 index 00000000..340b4faf Binary files /dev/null and b/data/gaia/val.parquet differ diff --git a/docs/DOCKER_SETUP.md b/docs/DOCKER_SETUP.md new file mode 100644 index 00000000..a6aabff6 --- /dev/null +++ b/docs/DOCKER_SETUP.md @@ -0,0 +1,141 @@ +# OpenManus-RL Docker Setup for AMD GPUs + +This setup allows you to run OpenManus-RL alfworld rollouts in a Docker container without affecting your existing verl-agent environment. + +## Prerequisites + +- Docker installed and running +- AMD GPU with ROCm support +- The `verl-agent:rocm-snap1` Docker image (from your previous verl-agent setup) +- Models stored in `/root/models/` + +## Setup Instructions + +### 1. Initial Setup + +First, run the setup script to create and configure the Docker container: + +```bash +cd /root/OpenManus-RL +./scripts/docker_setup.sh +``` + +This will: +- Create a new Docker container named `openmanus-rl` +- Install all required dependencies +- Set up a virtual environment at `/opt/openmanus-venv` +- Port 8001 on host will map to 8000 in container (to avoid conflict with verl-agent) + +### 2. Start/Access the Container + +If you need to enter the container manually: + +```bash +docker exec -it openmanus-rl bash +source /opt/openmanus-venv/bin/activate +cd /workspace +``` + +Then you can run commands directly. + +### 3. Run Rollouts (Unified Script) + +See ROLLOUT_GUIDE.md for detailed examples. A few quick starters: + +- GAIA dry‑run: + - `python scripts/rollout/unified_rollout.py --env gaia --batch_size 2 --total_envs 4 --dry_run` + +- AlfWorld small run (OpenAI): + - `python scripts/rollout/unified_rollout.py --env alfworld --model gpt-4o-mini --batch_size 1 --total_envs 2 --max_steps 20 --dump_path logs/alfworld/trajectory_$(date +%Y%m%d_%H%M%S).jsonl --chat_root .` + +- GAIA small run (local vLLM): + - `./scripts/serve_model.sh` (in another shell) + - `python scripts/rollout/unified_rollout.py --env gaia --model qwen2.5-7b-alfworld --base_url http://127.0.0.1:8000/v1 --gaia_tools python_code_generator --batch_size 1 --total_envs 2 --max_steps 30 --dump_path logs/gaia/trajectory_$(date +%Y%m%d_%H%M%S).jsonl --chat_root .` + +### 4. Running GAIA (Tool-Use) Rollouts + +GAIA uses the tool-use environment and the dataset in `data/gaia/val.json`. Some tools need extra API keys. + +Required packages for common tools are already listed in `requirements_docker.txt` (requests, python-dotenv, wikipedia). For Google search, set: + +```bash +export GOOGLE_API_KEY=your-google-api-key +export GOOGLE_CX=your-custom-search-engine-id +``` + +There are two ways to run GAIA: + +Use the unified script. Examples: + +1) OpenAI API +```bash +export OPENAI_API_KEY="your-openai-api-key" +python scripts/rollout/unified_rollout.py \ + --env gaia --model gpt-4o-mini \ + --gaia_tools python_code_generator \ + --total_envs 50 --batch_size 10 --max_steps 30 --concurrency 8 \ + --dump_path logs/gaia/trajectory_$(date +%Y%m%d_%H%M%S).jsonl \ + --chat_root /workspace +``` + +2) Local model via vLLM (OpenAI-compatible) + +First start the vLLM server (see above), then: +```bash +python scripts/rollout/unified_rollout.py \ + --env gaia --model qwen2.5-7b-alfworld --base_url http://127.0.0.1:8000/v1 \ + --gaia_tools python_code_generator \ + --total_envs 50 --batch_size 10 --max_steps 30 --concurrency 8 \ + --dump_path logs/gaia/trajectory_$(date +%Y%m%d_%H%M%S).jsonl \ + --chat_root /workspace +``` + +Notes: +- Default GAIA tools used in examples: `python_code_generator`（避免外部 API 依赖）。 +- If a tool needs external access (web APIs), ensure the container has outbound network connectivity and env vars are set. +- Chat histories and logs are saved under `logs/gaia` and `trajectories//gaia//` when `--chat_root` is provided. + +## Container Management + +### Stop the container +```bash +docker stop openmanus-rl +``` + +### Start the container again +```bash +docker start openmanus-rl +``` + +### Remove the container +```bash +docker stop openmanus-rl +docker rm openmanus-rl +``` + +### Check container logs +```bash +docker logs openmanus-rl +``` + +## Troubleshooting + +### If vLLM fails to start +1. Check GPU memory usage: `rocm-smi` +2. Adjust `--gpu-memory-utilization` in `serve_model.sh` +3. Make sure no other process is using port 8000 in the container + +### If rollout fails +1. Check that all dependencies are installed: `pip list` +2. Verify AlfWorld data is downloaded: `ls ~/.cache/alfworld` or re‑run `alfworld-download -f` +3. Check logs under `/workspace/logs//` + +### Port conflicts +- Default: container 8000 → host 8001 (configured by `docker_setup.sh`) +- Adjust mapping via `-p` flag if needed. + +## Output Files + +- Trajectory files: `/root/OpenManus-RL/logs/alfworld/trajectory_*.jsonl` +- Chat histories: `/root/OpenManus-RL/trajectories//` +- Log files: `/root/OpenManus-RL/logs/alfworld/run_log_*.log` diff --git a/docs/ROLLOUT_GUIDE.md b/docs/ROLLOUT_GUIDE.md new file mode 100644 index 00000000..7db0cbc5 --- /dev/null +++ b/docs/ROLLOUT_GUIDE.md @@ -0,0 +1,90 @@ +# Rollout Guide (AlfWorld, GAIA, WebShop) + +This guide shows how to run rollouts for the three environments using a single unified script. The script supports both OpenAI API and local OpenAI‑compatible endpoints (e.g., vLLM). + +## Prerequisites + +- Python venv prepared via Docker setup (see DOCKER_SETUP.md) +- .env at repo root (auto‑loaded) for API keys: + - `OPENAI_API_KEY` for OpenAI + - Optional tool keys (e.g., GAIA Google tools): `GOOGLE_API_KEY`, `GOOGLE_CX` +- For local inference (vLLM), start the server first (see DOCKER_SETUP.md or `serve_model.sh`). + +## Unified Script + +- Entry: `python scripts/rollout/unified_rollout.py` +- Core flags: + - `--env {alfworld,gaia,webshop}` choose environment + - `--model ` model name (OpenAI or local) + - `--base_url ` set when using local server (e.g., `http://127.0.0.1:8000/v1`) + - `--batch_size`, `--total_envs`, `--max_steps`, `--concurrency` + - `--dump_path ` save trajectories + - `--chat_root ` save chat histories under `trajectories////` + - `--dry_run` plan batches without creating envs/calling models + - `--unique_envs` ensure unique task/game sampling where supported + +## GAIA + +Data path default: `data/gaia/val.json` + +- Dry‑run (no model calls): + - `python scripts/rollout/openmanus_rollout.py --env gaia --batch_size 2 --total_envs 4 --dry_run` + +- OpenAI small run (minimal tools): + - `python scripts/rollout/openmanus_rollout.py \ + --env gaia --model gpt-4o \ + --gaia_tools python_code_generator \ + --batch_size 1 --total_envs 2 --max_steps 30 --concurrency 2 \ + --dump_path logs/gaia/trajectory_$(date +%Y%m%d_%H%M%S).jsonl --chat_root .` + +- Local vLLM small run: + - `python scripts/rollout/openmanus_rollout.py \ + --env gaia --model qwen2.5-7b-alfworld --base_url http://127.0.0.1:8000/v1 \ + --gaia_tools python_code_generator \ + --batch_size 1 --total_envs 2 --max_steps 30 --concurrency 2 \ + --dump_path logs/gaia/trajectory_$(date +%Y%m%d_%H%M%S).jsonl --chat_root .` + +## AlfWorld + +Make sure AlfWorld is installed and game data downloaded (`alfworld-download -f`). + +- Dry‑run (unique game files sampling): + - `python scripts/rollout/unified_rollout.py --env alfworld --unique_envs --batch_size 2 --total_envs 4 --dry_run` + +- OpenAI small run: + - `python scripts/rollout/openmanus_rollout.py \ + --env alfworld --model gpt-4o \ + --batch_size 1 --total_envs 2 --max_steps 30 --concurrency 2 \ + --dump_path logs/alfworld/trajectory_$(date +%Y%m%d_%H%M%S).jsonl --chat_root .` + +- Local vLLM small run: + - `python scripts/rollout/openmanus_rollout.py \ + --env alfworld --model qwen2.5-7b-alfworld --base_url http://127.0.0.1:8000/v1 \ + --batch_size 1 --total_envs 2 --max_steps 20 --concurrency 2 \ + --dump_path logs/alfworld/trajectory_$(date +%Y%m%d_%H%M%S).jsonl --chat_root .` + +## WebShop (optional) + +To run WebShop, follow data/index setup in DOCKER_SETUP.md, then use: + +- Dry‑run: + - `python scripts/rollout/openmanus_rollout.py --env webshop --batch_size 2 --total_envs 4 --dry_run` + +- OpenAI: + - `python scripts/rollout/openmanus_rollout.py \ + --env webshop --model gpt-4o \ + --batch_size 2 --total_envs 4 --max_steps 30 --concurrency 2 \ + --dump_path logs/webshop/trajectory_$(date +%Y%m%d_%H%M%S).jsonl --chat_root .` + +- Local vLLM: + - `python scripts/rollout/openmanus_rollout.py \ + --env webshop --model qwen2.5-7b-alfworld --base_url http://127.0.0.1:8000/v1 \ + --batch_size 2 --total_envs 4 --max_steps 30 --concurrency 2 \ + --dump_path logs/webshop/trajectory_$(date +%Y%m%d_%H%M%S).jsonl --chat_root .` + +## Outputs + +- Logs: `logs//unified_run_*.log` +- Trajectory: `--dump_path` JSONL +- Chats: `trajectories////` when `--chat_root` is set + diff --git a/openmanus_rl/engines/__init__.py b/openmanus_rl/engines/__init__.py new file mode 100644 index 00000000..0f785c92 --- /dev/null +++ b/openmanus_rl/engines/__init__.py @@ -0,0 +1,6 @@ +"""LLM engine interfaces and factories. + +This package provides lightweight wrappers around OpenAI-compatible +chat completion APIs and a simple factory used by tool modules. +""" + diff --git a/openmanus_rl/engines/factory.py b/openmanus_rl/engines/factory.py new file mode 100644 index 00000000..939e9ac7 --- /dev/null +++ b/openmanus_rl/engines/factory.py @@ -0,0 +1,21 @@ +"""Engine factory helpers. + +Exposes `create_llm_engine` returning a callable that maps prompt -> text using +the minimal `ChatOpenAI` wrapper. Keep the surface small and stable so tools +can depend on it without heavy coupling. +""" + +from typing import Callable, Optional +from .openai import ChatOpenAI + + +def create_llm_engine(model_string: str = "gpt-4o-mini", is_multimodal: bool = False, base_url: Optional[str] = None) -> Callable[[str], str]: + chat = ChatOpenAI(model=model_string, base_url=base_url) + + def _engine(prompt: str) -> str: + # Tools currently call engine(prompt) for text-only flows. + # If multimodal is needed later, extend by adding optional image args. + return chat(prompt) + + return _engine + diff --git a/openmanus_rl/engines/openai.py b/openmanus_rl/engines/openai.py new file mode 100644 index 00000000..1917f316 --- /dev/null +++ b/openmanus_rl/engines/openai.py @@ -0,0 +1,124 @@ +"""Minimal OpenAI chat wrapper. + +Provides a small surface compatible with internal code paths that expect +`ChatOpenAI` with a callable interface. Supports OpenAI-compatible backends +such as vLLM by honoring `OPENAI_BASE_URL`. +""" + +from typing import Optional, List, Dict, Any, Type +import json +import re +try: + from pydantic import BaseModel # type: ignore +except Exception: # pragma: no cover + BaseModel = object # type: ignore +import os + +try: + from openai import OpenAI # type: ignore +except Exception as exc: # pragma: no cover + OpenAI = None # type: ignore + + +class ChatOpenAI: + """Thin wrapper around OpenAI's Chat Completions API. + + The instance is callable and returns plain text. Images are not sent as + binary by design to remain compatible with OpenAI-compatible servers that + do not support multimodal content; image paths are appended as text hints. + """ + + def __init__( + self, + model: str = "gpt-4o-mini", + base_url: Optional[str] = None, + api_key: Optional[str] = None, + temperature: float = 0.0, + ) -> None: + if OpenAI is None: + raise RuntimeError("openai package is not installed") + + self.model = model + self.temperature = temperature + self.base_url = base_url or os.getenv("OPENAI_BASE_URL") + self.api_key = api_key or os.getenv("OPENAI_API_KEY", "EMPTY") + self.client = OpenAI(api_key=self.api_key, base_url=self.base_url) + + def __call__( + self, + prompt: str, + images: Optional[List[str]] = None, + system: Optional[str] = None, + response_format: Optional[Type] = None, + **_: Any, + ) -> Any: + messages: List[Dict[str, Any]] = [] + if system: + messages.append({"role": "system", "content": system}) + + if not images: + messages.append({"role": "user", "content": prompt}) + else: + # Safe multimodal fallback: append image paths as text hints. + content = prompt + for p in images: + content += f"\n[Image: {p}]" + messages.append({"role": "user", "content": content}) + + resp = self.client.chat.completions.create( + model=self.model, + messages=messages, + temperature=self.temperature, + n=1, + ) + text = (resp.choices[0].message.content or "").strip() + + # Best-effort structured parsing when a pydantic model is requested + try: + if response_format and isinstance(response_format, type) and issubclass(response_format, BaseModel): + # Try JSON first + try: + data = json.loads(text) + if isinstance(data, dict): + return response_format(**data) + if isinstance(data, list): + # Common pattern: patch list + payload: Dict[str, Any] = {} + if hasattr(response_format, "model_fields") and "patch" in response_format.model_fields: # pydantic v2 + payload["patch"] = data + elif hasattr(response_format, "__fields__") and "patch" in getattr(response_format, "__fields__"): + payload["patch"] = data + if payload: + return response_format(**payload) + except Exception: + pass + + # Special-case: AnswerVerification(analysis: str, true_false: bool) + if getattr(response_format, "__name__", "") == "AnswerVerification": + analysis = "" + tf = False + m = re.search(r"\s*(.*?)\s*", text, re.DOTALL) + if m: + analysis = m.group(1).strip() + m2 = re.search(r"\s*(.*?)\s*", text, re.DOTALL) + if m2: + val = m2.group(1).strip().lower() + tf = val in ("true", "1", "yes") + if not analysis: + analysis = text + return response_format(analysis=analysis, true_false=tf) + + # Fallback: try to populate known common fields + payload: Dict[str, Any] = {} + for field in ("analysis", "text"): + if (hasattr(response_format, "model_fields") and field in response_format.model_fields) or ( + hasattr(response_format, "__fields__") and field in getattr(response_format, "__fields__") + ): + payload[field] = text + if payload: + return response_format(**payload) + except Exception: + # Swallow parsing errors and return raw text + pass + + return text diff --git a/openmanus_rl/environments/env_manager.py b/openmanus_rl/environments/env_manager.py index fef5e73c..4405f8c6 100644 --- a/openmanus_rl/environments/env_manager.py +++ b/openmanus_rl/environments/env_manager.py @@ -6,7 +6,7 @@ import os from openmanus_rl.environments.prompts import * from openmanus_rl.environments.base import EnvironmentManagerBase, to_numpy -from openmanus_rl.memory import SimpleMemory +from openmanus_rl.memory import SimpleMemory, SummarizedMemory def parse_gamefile(infos): gamefile = [] @@ -28,7 +28,11 @@ def set_gamefile(infos, gamefile): class AlfWorldEnvironmentManager(EnvironmentManagerBase): def __init__(self, envs, projection_f, config): - self.memory = SimpleMemory() + # Choose memory type based on config + if hasattr(config.env, 'use_summary') and config.env.use_summary: + self.memory = SummarizedMemory() + else: + self.memory = SimpleMemory() super().__init__(envs, projection_f, config) def reset(self): @@ -79,10 +83,24 @@ def build_text_obs(self, text_obs: List[str], admissible_actions: List[List[str] """ postprocess_text_obs = [] if not init and self.config.env.history_length > 0: - memory_contexts, valid_lens = self.memory.fetch( + # Check if using summary mode + use_summary = hasattr(self.config.env, 'use_summary') and self.config.env.use_summary + + if use_summary: + memory_contexts, valid_lens = self.memory.fetch( self.config.env.history_length, obs_key="text_obs", - action_key="action") + action_key="action", + use_summary=True, + summary_api_key=getattr(self.config.env, 'summary_api_key', None), + summary_endpoint=getattr(self.config.env, 'summary_endpoint', None) + ) + else: + memory_contexts, valid_lens = self.memory.fetch( + self.config.env.history_length, + obs_key="text_obs", + action_key="action" + ) for i in range(len(text_obs)): # exclude 'help' in admissible_actions[i] @@ -140,7 +158,11 @@ def _process_gamefile(self, gamefile, won_value, success): class WebshopEnvironmentManager(EnvironmentManagerBase): def __init__(self, envs, projection_f, config): - self.memory = SimpleMemory() + # Choose memory type based on config + if hasattr(config.env, 'use_summary') and config.env.use_summary: + self.memory = SummarizedMemory() + else: + self.memory = SimpleMemory() super().__init__(envs, projection_f, config) def reset(self) -> Dict[str, Any]: @@ -223,7 +245,19 @@ def build_text_obs(self, text_obs: List[str], infos: List[List[str]], init: bool """ postprocess_text_obs = [] if not init and self.config.env.history_length > 0: - memory_contexts, valid_lens = self.memory.fetch( + # Check if using summary mode + use_summary = hasattr(self.config.env, 'use_summary') and self.config.env.use_summary + if use_summary: + memory_contexts, valid_lens = self.memory.fetch( + self.config.env.history_length, + obs_key="text_obs", + action_key="action", + use_summary=True, + summary_api_key=getattr(self.config.env, 'summary_api_key', None), + summary_endpoint=getattr(self.config.env, 'summary_endpoint', None), + ) + else: + memory_contexts, valid_lens = self.memory.fetch( self.config.env.history_length, obs_key="text_obs", action_key="action") @@ -326,6 +360,44 @@ def make_envs(config): import time time.sleep((config.data.train_batch_size * group_n + config.data.val_batch_size) * 0.1) # wait for the envs to be ready return envs, val_envs + elif "tool_use" in config.env.env_name.lower(): + from openmanus_rl.environments.env_package.tool_use.envs import build_tool_use_envs + from openmanus_rl.environments.env_package.tool_use.projection import tool_use_projection + from openmanus_rl.environments.env_package.tool_use.manager import ToolUseEnvironmentManager + + # Load task data + import json + data_path = getattr(config.env, 'data_path', 'data/gaia/val.json') + with open(data_path, 'r') as f: + tasks_data = json.load(f) + + # Get available tools from config + available_tools = getattr(config.env, 'available_tools', [ + 'google_search', 'wikipedia_knowledge_searcher', 'arxiv_paper_searcher' + ]) + + # Build environments + _envs = build_tool_use_envs( + tasks_data=tasks_data, + available_tools=available_tools, + seed=config.env.seed, + env_num=config.data.train_batch_size, + group_n=group_n, + is_train=True + ) + _val_envs = build_tool_use_envs( + tasks_data=tasks_data, + available_tools=available_tools, + seed=config.env.seed + 1000, + env_num=config.data.val_batch_size, + group_n=1, + is_train=False + ) + + projection_f = partial(tool_use_projection) + envs = ToolUseEnvironmentManager(_envs, projection_f, config) + val_envs = ToolUseEnvironmentManager(_val_envs, projection_f, config) + return envs, val_envs else: print("Environment not supported") - exit(1) \ No newline at end of file + exit(1) diff --git a/openmanus_rl/environments/env_package/alfworld/alfworld/agents/environment/alfred_tw_env.py b/openmanus_rl/environments/env_package/alfworld/alfworld/agents/environment/alfred_tw_env.py index 72ada5f7..3aed7269 100644 --- a/openmanus_rl/environments/env_package/alfworld/alfworld/agents/environment/alfred_tw_env.py +++ b/openmanus_rl/environments/env_package/alfworld/alfworld/agents/environment/alfred_tw_env.py @@ -114,7 +114,7 @@ class AlfredTWEnv(object): Interface for Textworld Env ''' - def __init__(self, config, train_eval="train"): + def __init__(self, config, train_eval="train", game_files=None): print("Initializing AlfredTWEnv...") self.config = config self.train_eval = train_eval @@ -124,7 +124,13 @@ def __init__(self, config, train_eval="train"): " the script `alfworld-generate`. Ignoring it and loading games as they are.") print(colored(msg, "yellow")) - self.collect_game_files() + if game_files is not None: + # Use provided game files directly + self.game_files = list(game_files) + self.num_games = len(self.game_files) + print(f"Using provided game_files: {self.num_games}") + else: + self.collect_game_files() self.use_expert = False print(f"use_expert = {self.use_expert}") def collect_game_files(self, verbose=False): diff --git a/openmanus_rl/environments/env_package/alfworld/envs.py b/openmanus_rl/environments/env_package/alfworld/envs.py index abcb5a5c..af70e403 100644 --- a/openmanus_rl/environments/env_package/alfworld/envs.py +++ b/openmanus_rl/environments/env_package/alfworld/envs.py @@ -1,10 +1,53 @@ +# Copyright 2025 Nanyang Technological University (NTU), Singapore +# and the verl-agent (GiGPO) team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import yaml import gymnasium as gym from gymnasium import spaces import numpy as np import torch -import torchvision.transforms as T +# Try to import torchvision transforms lazily and robustly. +# On ROCm builds, importing torchvision may fail due to missing compiled ops +# (e.g., torchvision::nms). We only need basic ToTensor for multi-modal Thor env. +try: + import torchvision.transforms as T # type: ignore +except Exception: # Fallback lightweight transforms to avoid hard dependency + class _ToTensor: + def __call__(self, img): + import numpy as _np + import torch as _torch + arr = _np.array(img) + if arr.ndim == 2: # grayscale -> HxW -> 1xHxW + arr = arr[:, :, None] + t = _torch.from_numpy(arr).permute(2, 0, 1).float() / 255.0 + return t + + class _Compose: + def __init__(self, ts): + self.ts = ts + def __call__(self, x): + for f in self.ts: + x = f(x) + return x + + class _T: + Compose = _Compose + ToTensor = _ToTensor + + T = _T() # minimal drop-in replacement import ray from openmanus_rl.environments.env_package.alfworld.alfworld.agents.environment import get_environment @@ -44,8 +87,17 @@ class AlfworldWorker: Each actor holds one environment instance. """ - def __init__(self, config, seed, base_env): - self.env = base_env.init_env(batch_size=1) # Each worker holds only one sub-environment + def __init__(self, config, seed, base_env=None, env_type=None, single_gamefile=None, is_train=True, eval_dataset='eval_in_distribution'): + if base_env is not None: + # Legacy path: share a base_env and instantiate sub-env from it + self.env = base_env.init_env(batch_size=1) + else: + # Unique path: each worker binds to exactly one gamefile + assert env_type is not None, "env_type is required when base_env is None" + BaseEnvCls = get_environment(env_type) + game_files_override = [single_gamefile] if single_gamefile is not None else None + be = BaseEnvCls(config, train_eval='train' if is_train else eval_dataset, game_files=game_files_override) + self.env = be.init_env(batch_size=1) self.env.seed(seed) def step(self, action): @@ -69,7 +121,7 @@ def getobs(self): return image class AlfworldEnvs(gym.Env): - def __init__(self, alf_config_path, seed=0, env_num=1, group_n=1, is_train=True, env_kwargs={}): + def __init__(self, alf_config_path, seed=0, env_num=1, group_n=1, is_train=True, env_kwargs={}, game_files=None): super().__init__() # Initialize Ray if not already initialized @@ -79,16 +131,31 @@ def __init__(self, alf_config_path, seed=0, env_num=1, group_n=1, is_train=True, eval_dataset = env_kwargs.get('eval_dataset', 'eval_in_distribution') config = load_config_file(alf_config_path) env_type = config['env']['type'] - base_env = get_environment(env_type)(config, train_eval='train' if is_train else eval_dataset) self.multi_modal = (env_type == 'AlfredThorEnv') self.num_processes = env_num * group_n self.group_n = group_n # Create Ray remote actors instead of processes self.workers = [] - for i in range(self.num_processes): - worker = AlfworldWorker.remote(config, seed + (i // self.group_n), base_env) - self.workers.append(worker) + if game_files is not None: + assert len(game_files) == self.num_processes, "game_files length must equal env_num * group_n" + for i in range(self.num_processes): + single = game_files[i] + worker = AlfworldWorker.remote( + config, + seed + (i // self.group_n), + base_env=None, + env_type=env_type, + single_gamefile=single, + is_train=is_train, + eval_dataset=eval_dataset, + ) + self.workers.append(worker) + else: + base_env = get_environment(env_type)(config, train_eval='train' if is_train else eval_dataset) + for i in range(self.num_processes): + worker = AlfworldWorker.remote(config, seed + (i // self.group_n), base_env=base_env) + self.workers.append(worker) self.prev_admissible_commands = [None for _ in range(self.num_processes)] @@ -160,16 +227,18 @@ def reset(self): def getobs(self): """ - Ask each worker to return its current frame image. - Usually needed only for multi-modal environments; otherwise can return None. + Collect all image observations from workers. """ + # Send getobs commands to all workers futures = [] for worker in self.workers: future = worker.getobs.remote() futures.append(future) - images = ray.get(futures) - return images + # Collect and stack results + results = ray.get(futures) + image_obs_list = torch.cat(results, dim=0) + return image_obs_list @property def get_admissible_commands(self): @@ -180,12 +249,9 @@ def get_admissible_commands(self): return self.prev_admissible_commands def close(self): - """ - Close all workers - """ - # Kill all Ray actors + """Clean up Ray actors""" for worker in self.workers: ray.kill(worker) -def build_alfworld_envs(alf_config_path, seed, env_num, group_n, is_train=True, env_kwargs={}): - return AlfworldEnvs(alf_config_path, seed, env_num, group_n, is_train, env_kwargs) \ No newline at end of file +def build_alfworld_envs(alf_config_path, seed, env_num, group_n, is_train=True, env_kwargs={}, game_files=None): + return AlfworldEnvs(alf_config_path, seed, env_num, group_n, is_train, env_kwargs, game_files) \ No newline at end of file diff --git a/openmanus_rl/environments/env_package/alfworld/projection.py b/openmanus_rl/environments/env_package/alfworld/projection.py index 64356ef3..5c43d18e 100644 --- a/openmanus_rl/environments/env_package/alfworld/projection.py +++ b/openmanus_rl/environments/env_package/alfworld/projection.py @@ -34,9 +34,11 @@ def alfworld_projection(actions: List[str], action_pools: List[List[str]]): except: actions[i] = actions[i][-30:] - # check ... - think_start_idx = original_str.find("") - think_end_idx = original_str.find("") + # # check ... + # think_start_idx = original_str.find("") + # think_end_idx = original_str.find("") + think_start_idx = original_str.find("") + think_end_idx = original_str.find("") if think_start_idx == -1 or think_end_idx == -1: valids[i] = 0 diff --git a/openmanus_rl/environments/env_package/tool_use/envs.py b/openmanus_rl/environments/env_package/tool_use/envs.py new file mode 100644 index 00000000..a2a05e3b --- /dev/null +++ b/openmanus_rl/environments/env_package/tool_use/envs.py @@ -0,0 +1,204 @@ +""" +Tool Use Environment for complex reasoning tasks with tool calling capability. +Provides tasks from dataset and handles tool execution results. +""" + +import json +import random +import importlib +from typing import List, Dict, Any, Tuple +import numpy as np + + +class ToolUseEnv: + """ + Simple mock environment for tool use tasks. + Provides tasks from dataset and handles tool execution. + """ + + def __init__(self, tasks_data: List[Dict], available_tools: List[str], seed: int = 42): + self.tasks_data = tasks_data + self.available_tools = available_tools + self.tool_manager = ToolManager(available_tools) + self.current_task_idx = 0 + self.seed = seed + random.seed(seed) + np.random.seed(seed) + + def reset(self, task_idx: int = None) -> Tuple[str, Dict]: + """Reset environment with a new task""" + if task_idx is not None: + self.current_task_idx = task_idx + elif self.current_task_idx >= len(self.tasks_data): + self.current_task_idx = 0 + + task_data = self.tasks_data[self.current_task_idx] + self.current_task_idx += 1 + + # Return empty observation (task info is in info dict) + info = { + 'task': task_data['question'], + 'answer': task_data['answer'], + 'pid': task_data['pid'], + 'available_tools': self.available_tools, + 'tool_metadata': self.tool_manager.get_tools_metadata() + } + + return "", info + + def step(self, action: str) -> Tuple[str, float, bool, Dict]: + """ + Execute action and return observation, reward, done, info. + For tool use environment, we don't actually step - the environment manager handles everything. + """ + return "", 0.0, False, {} + + def close(self): + """Close environment""" + pass + + +class ToolUseEnvs: + """ + Vectorized wrapper for tool use environments. + Similar to AlfworldEnvs but simpler since we don't need Ray workers. + """ + + def __init__(self, tasks_data: List[Dict], available_tools: List[str], + seed: int = 0, env_num: int = 1, group_n: int = 1, is_train: bool = True): + self.tasks_data = tasks_data + self.available_tools = available_tools + self.num_processes = env_num * group_n + self.group_n = group_n + self.is_train = is_train + + # Create individual environments + self.envs = [] + for i in range(self.num_processes): + env = ToolUseEnv(tasks_data, available_tools, seed + i) + self.envs.append(env) + + # Track current task indices for each environment + self.current_indices = list(range(self.num_processes)) + + def reset(self) -> Tuple[List[str], List[Dict]]: + """Reset all environments""" + obs_list = [] + info_list = [] + + for i, env in enumerate(self.envs): + # Use different tasks for different environments + task_idx = (self.current_indices[i]) % len(self.tasks_data) + obs, info = env.reset(task_idx=task_idx) + obs_list.append(obs) + info_list.append(info) + self.current_indices[i] += 1 + + return obs_list, info_list + + def step(self, actions: List[str]) -> Tuple[List[str], List[float], List[bool], List[Dict]]: + """Step all environments - placeholder since real logic is in environment manager""" + obs_list = [] + rewards_list = [] + dones_list = [] + info_list = [] + + for i, (env, action) in enumerate(zip(self.envs, actions)): + obs, reward, done, info = env.step(action) + obs_list.append(obs) + rewards_list.append(reward) + dones_list.append(done) + info_list.append(info) + + return obs_list, rewards_list, dones_list, info_list + + def close(self): + """Close all environments""" + for env in self.envs: + env.close() + + +class ToolManager: + """Manages available tools and their execution""" + + def __init__(self, tool_names: List[str]): + self.tool_names = tool_names + self.available_tools = {} + self._load_tools() + + def _load_tools(self): + """Load tools specified in tool_names""" + for tool_name in self.tool_names: + try: + self._load_tool(tool_name) + except Exception as e: + print(f"Warning: Failed to load tool '{tool_name}': {e}") + + def _load_tool(self, tool_name: str): + """Load a specific tool""" + # Map tool names to their module paths + tool_mapping = { + 'google_search': 'openmanus_rl.tools.google_search.tool.Google_Search_Tool', + 'wikipedia_knowledge_searcher': 'openmanus_rl.tools.wikipedia_knowledge_searcher.tool.Wikipedia_Knowledge_Searcher_Tool', + 'arxiv_paper_searcher': 'openmanus_rl.tools.arxiv_paper_searcher.tool.Arxiv_Paper_Searcher_Tool', + 'pubmed_search': 'openmanus_rl.tools.pubmed_search.tool.Pubmed_Search_Tool', + 'url_text_extractor': 'openmanus_rl.tools.url_text_extractor.tool.URL_Text_Extractor_Tool', + 'python_code_generator': 'openmanus_rl.tools.python_code_generator.tool.Python_Code_Generator_Tool', + } + + if tool_name not in tool_mapping: + print(f"Unknown tool: {tool_name}, skipping...") + return + + module_path = tool_mapping[tool_name] + module_name, class_name = module_path.rsplit('.', 1) + + # Import and instantiate the tool + module = importlib.import_module(module_name) + tool_class = getattr(module, class_name) + tool_instance = tool_class() + + self.available_tools[tool_name] = tool_instance + + def get_tools_metadata(self) -> str: + """Generate formatted metadata for all available tools""" + if not self.available_tools: + return "No tools available." + + metadata_lines = [] + for tool_name, tool_instance in self.available_tools.items(): + metadata = tool_instance.get_metadata() + + tool_info = f"""Tool: {tool_name} +Description: {metadata.get('tool_description', 'No description')} +Input Types: {metadata.get('input_types', 'No input types specified')} +Usage: +tool: {tool_name} +parameters: {{"param_name": "param_value"}} +""" + + metadata_lines.append(tool_info.strip()) + + return "\n\n".join(metadata_lines) + + def execute_tool(self, tool_name: str, params: Dict) -> str: + """Execute a tool with given parameters""" + if tool_name not in self.available_tools: + return f"Error: Tool '{tool_name}' not available. Available tools: {list(self.available_tools.keys())}" + + try: + tool_instance = self.available_tools[tool_name] + result = tool_instance.execute(**params) + + # Convert result to string if needed + if isinstance(result, (list, dict)): + return json.dumps(result, indent=2, ensure_ascii=False) + return str(result) + except Exception as e: + return f"Error executing tool '{tool_name}': {str(e)}" + + +def build_tool_use_envs(tasks_data: List[Dict], available_tools: List[str], + seed: int, env_num: int, group_n: int, is_train: bool = True): + """Build tool use environments""" + return ToolUseEnvs(tasks_data, available_tools, seed, env_num, group_n, is_train) diff --git a/openmanus_rl/environments/env_package/tool_use/manager.py b/openmanus_rl/environments/env_package/tool_use/manager.py new file mode 100644 index 00000000..64ebb37e --- /dev/null +++ b/openmanus_rl/environments/env_package/tool_use/manager.py @@ -0,0 +1,172 @@ +""" +Tool Use Environment Manager +""" + +import json +import re +from typing import List, Dict, Any +from collections import defaultdict +import numpy as np + +from openmanus_rl.environments.base import EnvironmentManagerBase, to_numpy +from openmanus_rl.memory import SimpleMemory +from openmanus_rl.environments.prompts import * + + +class ToolUseEnvironmentManager(EnvironmentManagerBase): + """Environment manager for tool use tasks""" + + def __init__(self, envs, projection_f, config): + super().__init__(envs, projection_f, config) + self.memory = SimpleMemory() + self.current_tasks = [] + self.ground_truths = [] + self.step_counts = [] + self.task_completed = [] + + def reset(self): + """Reset environment and get new tasks""" + obs, infos = self.envs.reset() + + # Extract task information + self.current_tasks = [info.get('task', '') for info in infos] + self.ground_truths = [info.get('answer', '') for info in infos] + self.tool_metadata = infos[0].get('tool_metadata', '') if infos else '' + + batch_size = len(self.current_tasks) + self.step_counts = [0] * batch_size + self.task_completed = [False] * batch_size + + # Initialize memory + self.memory.reset(batch_size=batch_size) + + # Build initial text observation + full_text_obs = self.build_text_obs(init=True) + + return {'text': full_text_obs, 'image': None, 'anchor': self.current_tasks.copy()}, infos + + def step(self, text_actions: List[str]): + """Execute text actions""" + actions, valids = self.projection_f(text_actions) + batch_size = len(text_actions) + + # Process actions and execute tools + observations = [] + rewards = np.zeros(batch_size) + dones = np.zeros(batch_size, dtype=bool) + infos = [] + + for i, (action, valid) in enumerate(zip(actions, valids)): + if self.task_completed[i]: + observations.append("Task completed.") + infos.append({'is_action_valid': True, 'won': True}) + continue + + self.step_counts[i] += 1 + + # Process action + obs, info = self._process_action(action, i) + observations.append(obs) + + # Check completion + if self._is_completion_action(action): + self.task_completed[i] = True + dones[i] = True + elif self.step_counts[i] >= self.config.env.max_steps: + obs += "\n\nMaximum steps reached. Please provide your final answer in tags." + dones[i] = True + + info['is_action_valid'] = to_numpy(valid) + info['won'] = self.task_completed[i] + info['step_count'] = self.step_counts[i] + infos.append(info) + + # After processing all envs, store this step's observations and actions into memory + try: + self.memory.store({'text_obs': observations, 'action': text_actions}) + except Exception: + # Be permissive: if memory storage fails, continue without history + pass + + # Build text observations + full_text_obs = self.build_text_obs(observations=observations) + + next_observations = {'text': full_text_obs, 'image': None, 'anchor': observations.copy()} + rewards = to_numpy(rewards) + dones = to_numpy(dones) + + return next_observations, rewards, dones, infos + + def _process_action(self, action: str, batch_idx: int) -> tuple: + """Process a single action""" + info = {'is_action_valid': True} + + try: + # Try to parse as JSON (from projection) + action_data = json.loads(action) + if action_data.get('type') == 'tool_call': + # Execute tool + tool_name = action_data['tool'] + params = action_data['parameters'] + result = self.envs.envs[batch_idx].tool_manager.execute_tool(tool_name, params) + observation = f"Tool '{tool_name}' executed.\nResult: {result}" + else: + observation = "Action acknowledged." + except (json.JSONDecodeError, KeyError): + # Regular action or final answer + if action.startswith("FINAL_ANSWER:"): + observation = "Final answer provided. Task completed." + else: + observation = "Action acknowledged. Continue reasoning or use tools to gather information." + + return observation, info + + def _is_completion_action(self, action: str) -> bool: + """Check if action indicates task completion""" + return action.startswith("FINAL_ANSWER:") or "" in action + + def build_text_obs(self, observations: List[str] = None, init: bool = False) -> List[str]: + """Build text observations for agent""" + batch_size = len(self.current_tasks) + postprocess_text_obs = [] + + for i in range(batch_size): + if init or self.config.env.history_length <= 0: + obs = TOOL_USE_TEMPLATE_NO_HIS.format( + task_description=self.current_tasks[i], + available_tools=self.tool_metadata, + current_observation="Start working on the task." + ) + else: + # Get history + memory_contexts, valid_lens = self.memory.fetch( + self.config.env.history_length, + obs_key="text_obs", + action_key="action" + ) + + current_obs = observations[i] if observations else "Continue with your task." + + obs = TOOL_USE_TEMPLATE.format( + task_description=self.current_tasks[i], + step_count=self.step_counts[i], + history_length=valid_lens[i], + action_history=memory_contexts[i], + current_step=self.step_counts[i] + 1, + current_observation=current_obs, + available_tools=self.tool_metadata + ) + + postprocess_text_obs.append(obs) + + return postprocess_text_obs + + def _process_batch(self, batch_idx, total_batch_list, total_infos, success): + """Process batch for success evaluation""" + for i in reversed(range(len(total_batch_list[batch_idx]))): + batch_item = total_batch_list[batch_idx][i] + if batch_item['active_masks']: + info = total_infos[batch_idx][i] + won_value = float(info['won']) + success['success_rate'].append(won_value) + return diff --git a/openmanus_rl/environments/env_package/tool_use/projection.py b/openmanus_rl/environments/env_package/tool_use/projection.py new file mode 100644 index 00000000..49c72446 --- /dev/null +++ b/openmanus_rl/environments/env_package/tool_use/projection.py @@ -0,0 +1,133 @@ +""" +Tool Use Projection function for processing text actions into tool calls or regular actions. +""" + +import json +import re +from typing import List, Tuple + + +def tool_use_projection(actions: List[str]) -> Tuple[List[str], List[int]]: + """ + Process text actions for tool use environment. + + Args: + actions: List of text actions from the agent + + Returns: + Tuple of (processed_actions, valids) where: + - processed_actions: List of processed action strings + - valids: List of 1s and 0s indicating valid/invalid actions + """ + valids = [0] * len(actions) + processed_actions = [] + + for i, action in enumerate(actions): + try: + # Start with assuming action is valid + is_valid = True + original_action = action + + # Check for Chinese characters - mark as invalid if found + if re.search(r'[\u4e00-\u9fff]', action): + is_valid = False + + # Check if action contains tool call + if _has_tool_call(action): + # Parse and validate tool call + tool_action, tool_valid = _parse_tool_call(action) + processed_actions.append(tool_action) + is_valid = is_valid and tool_valid + + # Check if action contains final answer + elif _has_answer(action): + # Extract answer and mark as completion action + answer = _extract_answer(action) + processed_actions.append(f"FINAL_ANSWER: {answer}") + is_valid = True # Answer actions are always valid + + else: + # Regular reasoning action - just pass through + processed_actions.append(action) + # Regular actions are valid as long as they don't have Chinese + + valids[i] = 1 if is_valid else 0 + + except Exception as e: + # If any error occurs, mark as invalid + processed_actions.append(action) + valids[i] = 0 + + return processed_actions, valids + + +def _has_tool_call(action: str) -> bool: + """Check if action contains a tool call""" + return bool(re.search(r'.*?', action, re.DOTALL)) + + +def _has_answer(action: str) -> bool: + """Check if action contains a final answer""" + return bool(re.search(r'.*?', action, re.DOTALL)) + + +def _parse_tool_call(action: str) -> Tuple[str, bool]: + """ + Parse tool call from action text. + + Returns: + Tuple of (parsed_action, is_valid) + """ + try: + # Extract tool call content + tool_match = re.search(r'(.*?)', action, re.DOTALL) + if not tool_match: + return action, False + + tool_content = tool_match.group(1).strip() + + # Parse tool name and parameters + tool_name = None + params = {} + + lines = tool_content.split('\n') + for line in lines: + line = line.strip() + if line.lower().startswith('tool:'): + tool_name = line.split(':', 1)[1].strip() + elif line.lower().startswith('parameters:'): + try: + params_str = line.split(':', 1)[1].strip() + # Try to parse as JSON + params = json.loads(params_str) + except (json.JSONDecodeError, IndexError): + # Fallback to treating the whole thing as a query + params = {'query': params_str} + elif ':' in line and not tool_name: + # Handle simple key:value format + key, value = line.split(':', 1) + params[key.strip()] = value.strip() + + if not tool_name: + return action, False + + # Format as structured action + formatted_action = json.dumps({ + 'type': 'tool_call', + 'tool': tool_name, + 'parameters': params, + 'original': action + }) + + return formatted_action, True + + except Exception: + return action, False + + +def _extract_answer(action: str) -> str: + """Extract final answer from action text""" + answer_match = re.search(r'(.*?)', action, re.DOTALL) + if answer_match: + return answer_match.group(1).strip() + return "" diff --git a/openmanus_rl/environments/env_package/tool_use/reward_manager.py b/openmanus_rl/environments/env_package/tool_use/reward_manager.py new file mode 100644 index 00000000..5c427b34 --- /dev/null +++ b/openmanus_rl/environments/env_package/tool_use/reward_manager.py @@ -0,0 +1,199 @@ +""" +LLM Judge Reward Manager for Tool Use Environment +Evaluates agent answers against ground truth using LLM judge similar to calculate_score.py +""" + +import re +import json +from typing import List, Dict, Any, Optional +import logging +from pydantic import BaseModel +from collections import defaultdict + +try: + from openmanus_rl.engines.openai import ChatOpenAI + HAS_LLM_ENGINE = True +except ImportError: + HAS_LLM_ENGINE = False + logging.warning("LLM engine not available. LLM judge reward will not work.") + + +class AnswerVerification(BaseModel): + analysis: str + true_false: bool + + +class LLMJudgeRewardManager: + """ + Manages LLM-based reward calculation for tool use tasks. + Evaluates agent answers against ground truth using GPT-based judging. + """ + + def __init__(self, model_string: str = "gpt-4o-mini"): + self.model_string = model_string + self.llm_engine = None + if HAS_LLM_ENGINE: + try: + self.llm_engine = ChatOpenAI( + model_string=model_string, + is_multimodal=False, + enable_cache=True + ) + print(f"LLM Judge initialized with {model_string}") + except Exception as e: + print(f"Warning: Failed to initialize LLM engine: {e}") + + def extract_final_answers(self, responses: List[str]) -> List[str]: + """ + Extract final answers from agent responses. + Looks for ... tags or FINAL_ANSWER: format. + """ + final_answers = [] + + for response in responses: + answer = "" + + # Try to find tags first + answer_match = re.search(r'(.*?)', response, re.DOTALL) + if answer_match: + answer = answer_match.group(1).strip() + + # Fallback to FINAL_ANSWER format + elif "FINAL_ANSWER:" in response: + lines = response.split('\n') + for line in lines: + if line.strip().startswith("FINAL_ANSWER:"): + answer = line.split("FINAL_ANSWER:", 1)[1].strip() + break + + # If no structured answer found, try to extract from end of response + if not answer: + # Look for common answer patterns at the end + lines = response.split('\n') + for line in reversed(lines[-5:]): # Check last 5 lines + line = line.strip() + if line and not line.startswith(('<', 'Tool', 'Result:')): + answer = line + break + + final_answers.append(answer) + + return final_answers + + def evaluate_answers(self, agent_answers: List[str], ground_truths: List[str], + pids: List[str] = None) -> Dict[str, Any]: + """ + Evaluate agent answers against ground truth using LLM judge. + + Returns: + Dict containing: + - individual_scores: List of 0/1 scores for each answer + - individual_analyses: List of analysis texts + - overall_accuracy: Float accuracy score + """ + if not self.llm_engine: + print("Warning: LLM engine not available, returning zero scores") + return { + 'individual_scores': [0.0] * len(agent_answers), + 'individual_analyses': ['LLM judge not available'] * len(agent_answers), + 'overall_accuracy': 0.0 + } + + individual_scores = [] + individual_analyses = [] + + for i, (agent_answer, ground_truth) in enumerate(zip(agent_answers, ground_truths)): + if not agent_answer.strip(): + # Empty answer - automatically wrong + individual_scores.append(0.0) + individual_analyses.append("Empty answer provided") + continue + + try: + analysis, is_correct = self._judge_single_answer(agent_answer, ground_truth) + individual_scores.append(1.0 if is_correct else 0.0) + individual_analyses.append(analysis) + except Exception as e: + print(f"Error evaluating answer {i}: {e}") + individual_scores.append(0.0) + individual_analyses.append(f"Evaluation error: {str(e)}") + + overall_accuracy = sum(individual_scores) / len(individual_scores) if individual_scores else 0.0 + + return { + 'individual_scores': individual_scores, + 'individual_analyses': individual_analyses, + 'overall_accuracy': overall_accuracy, + 'correct_count': sum(individual_scores), + 'total_count': len(individual_scores) + } + + def _judge_single_answer(self, agent_answer: str, ground_truth: str) -> tuple: + """ + Judge a single answer against ground truth using LLM. + Returns (analysis, is_correct). + """ + query_prompt = f""" + Compare the model's response against the correct answer following these evaluation rules: + + Model response: {agent_answer} + Correct answer: {ground_truth} + + Evaluation rules: + 1. Extract the core answer from the model response (ignore explanations or additional context) + 2. The answer is correct if it EXACTLY matches the correct answer: + - Numbers must match precisely (e.g., "142" = "142") + - Text must match case-sensitive (e.g., "Time-Parking 2: Parallel Universe") + - Zip codes must be exact (e.g., "34689") + - No partial credit for similar or related answers + 3. The answer is incorrect if: + - It contains any additional or missing information + - It uses different formatting or representations + - It's semantically equivalent but not identical + + Response Format: + : Extract the core answer and explain exact match comparison + : Return "True" only for exact matches, otherwise "False" + """ + + verification = self.llm_engine(query_prompt, response_format=AnswerVerification) + + analysis = verification.analysis.strip() + is_correct = verification.true_false + + return analysis, is_correct + + +def calculate_delayed_rewards(episode_data: List[Dict], reward_manager: LLMJudgeRewardManager) -> Dict[str, Any]: + """ + Calculate delayed rewards for completed episodes using LLM judge. + + Args: + episode_data: List of episode dictionaries containing responses and ground truth + reward_manager: LLMJudgeRewardManager instance + + Returns: + Dict containing reward scores and metadata + """ + # Extract final answers and ground truths + responses = [ep.get('final_response', '') for ep in episode_data] + ground_truths = [ep.get('ground_truth', '') for ep in episode_data] + pids = [ep.get('pid', str(i)) for i, ep in enumerate(episode_data)] + + # Extract final answers from responses + final_answers = reward_manager.extract_final_answers(responses) + + # Evaluate using LLM judge + evaluation_results = reward_manager.evaluate_answers(final_answers, ground_truths, pids) + + # Convert to reward format + rewards = evaluation_results['individual_scores'] + + return { + 'rewards': rewards, + 'final_answers': final_answers, + 'analyses': evaluation_results['individual_analyses'], + 'accuracy': evaluation_results['overall_accuracy'], + 'correct_count': evaluation_results['correct_count'], + 'total_count': evaluation_results['total_count'] + } diff --git a/openmanus_rl/environments/prompts/__init__.py b/openmanus_rl/environments/prompts/__init__.py index 21699317..66986e54 100644 --- a/openmanus_rl/environments/prompts/__init__.py +++ b/openmanus_rl/environments/prompts/__init__.py @@ -1,2 +1,3 @@ from .alfworld import * -from .webshop import * \ No newline at end of file +from .webshop import * +from .tool_use import * \ No newline at end of file diff --git a/openmanus_rl/environments/prompts/alfworld.py b/openmanus_rl/environments/prompts/alfworld.py index 568e3534..2a47c4f0 100644 --- a/openmanus_rl/environments/prompts/alfworld.py +++ b/openmanus_rl/environments/prompts/alfworld.py @@ -1,14 +1,33 @@ -# --------------------- ALFWorld --------------------- # +# # --------------------- ALFWorld --------------------- # +# ALFWORLD_TEMPLATE_NO_HIS = """ +# You are an expert agent operating in the ALFRED Embodied Environment. +# Your current observation is: {current_observation} +# Your admissible actions of the current situation are: [{admissible_actions}]. + +# Now it's your turn to take an action. +# You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within tags. +# Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. +# """ + ALFWORLD_TEMPLATE_NO_HIS = """ You are an expert agent operating in the ALFRED Embodied Environment. +Your task is: {task_description} Your current observation is: {current_observation} Your admissible actions of the current situation are: [{admissible_actions}]. -Now it's your turn to take an action. -You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within tags. -Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. +Please begin by analyzing the situation and planning your approach: + + +Analyze the current situation and devise a plan to accomplish the task: +What are the key steps needed to complete this task? +How to advance our plan toward completing the task in immediate next step? +Based on the current observation, what should be our immediate next step? + + +Finally, choose ONE admissible action for the current step and present it within tags. """ + ALFWORLD_TEMPLATE = """ You are an expert agent operating in the ALFRED Embodied Environment. Your task is to: {task_description} Prior to this step, you have already taken {step_count} step(s). Below are the most recent {history_length} observaitons and the corresponding actions you took: {action_history} @@ -16,41 +35,16 @@ Your admissible actions of the current situation are: [{admissible_actions}]. Now it's your turn to take an action. -You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within tags. -Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. -""" -ALFWORLD_OPENMANUS_INITIAL_TEMPLATE = """ -You are an expert agent operating in the ALFRED Embodied Environment. Your task is to: {task_description} +You should first recall relevant past experiences and reason from our conversation history, then MUST summarize within tags like this: -Current observation: {current_observation} -Available actions: [{admissible_actions}] + +[Recall relevant past experiences and reason from our conversation history] +- Please summarize the most relavent memory for this step. +- Please explain why this memory is helpful for the next reflection and planning. + -Please begin by analyzing the situation and planning your approach: - - -Analyze the current situation and devise a plan to accomplish the task: {task_description} -What are the key steps needed to complete this task? -Based on the current observation, what should be our immediate next step? -How does this action advance our plan toward completing the task? - - -Now, present your chosen action: - - -action_choice: [selected admissible action from the list] -action_parameters: {{relevant details about the action if applicable}} - - -From now on, I will provide you with observations after each action, and you should respond with memory recall, reflection, thinking, and your next action in this format: - - -[Recall relevant past experiences and reasoning from our conversation history] -- What similar situations have I encountered? -- What strategies worked or failed before? -- What objects or locations have I discovered? -- What was my previous reasoning and plans? - +After that, you should reflect on the last action and its outcome, then MUST summarize within tags like this: [Reflect on the last action and its outcome] @@ -60,18 +54,15 @@ - Am I making progress toward the task goal? - +After that, you should plan the next step based on memory and reflection, then MUST summarize within tags like this: + + [Plan the next step based on memory and reflection] - Given what I've learned, what should I do next? +- Please explain why this plan is helpful for the next action? - How does this action fit into my overall strategy? - What do I expect this action to achieve? - + - -action_choice: [selected admissible action from the list] -action_parameters: {{relevant details about the action if applicable}} - +Finally, choose ONE admissible action for the current step and present it within tags. """ - -# Keep the old template name for backward compatibility -ALFWORLD_OPENMANUS_TEMPLATE = ALFWORLD_OPENMANUS_INITIAL_TEMPLATE \ No newline at end of file diff --git a/openmanus_rl/environments/prompts/tool_use.py b/openmanus_rl/environments/prompts/tool_use.py new file mode 100644 index 00000000..5bba9a0d --- /dev/null +++ b/openmanus_rl/environments/prompts/tool_use.py @@ -0,0 +1,89 @@ +# --------------------- Tool Use --------------------- # + +TOOL_USE_TEMPLATE_NO_HIS = """ +You are an expert research assistant capable of using various tools to gather information and solve complex problems. + +Task: {task_description} + +Available Tools: +{available_tools} + +Current Observation: {current_observation} + +Instructions: +1. Analyze the task and determine what information you need +2. Use available tools to gather information when needed +3. Reason through the information step by step +4. When you have sufficient information, provide your final answer in tags + +Format for tool usage: + +tool: [tool_name] +parameters: {{"param1": "value1", "param2": "value2"}} + + +Now it's your turn to take an action. You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within tags. +Once you've finished your reasoning, you should either use a tool or provide your final answer within tags. +""" +TOOL_USE_TEMPLATE_LAST_STEP = """ +You are an expert research assistant capable of using various tools to gather information and solve complex problems. + +Task: {task_description} + +Prior to this step, you have already taken {step_count} step(s). Below are the full {history_length} observations and the corresponding actions you took: {action_history} + +You are now at step {current_step} and this is the final step. +Current Observation: {current_observation} +You must provide your final answer within tags. +""" + +TOOL_USE_TEMPLATE = """ +You are an expert research assistant capable of using various tools to gather information and solve complex problems. + +Task: {task_description} + +Prior to this step, you have already taken {step_count} step(s). Below are the most recent {history_length} observations and the corresponding actions you took: {action_history} + +You are now at step {current_step}. +Current Observation: {current_observation} + +Available Tools: +{available_tools} + +You should first recall relevant past experiences and reason from our conversation history, then MUST summarize within tags like this: + + +[Recall relevant past experiences and reason from our conversation history] +- Please summarize the most relavent memory for this step. +- Please explain why this memory is helpful for the next reflection and planning. + + +After that, you should reflect on the last action and its outcome, then MUST summarize within tags like this: + + +[Reflect on the last action and its outcome] +- What did my last action accomplish? +- Was it successful or did it encounter issues? +- How does this outcome affect my plan? +- Am I making progress toward the task goal? + + +Given from the analysis from the memory analysis and reflection, if we get the final answer, we should provide it within tags. +If we don't get the final answer, you should plan the next step based on memory and reflection, then MUST summarize within tags like this: + + +[Plan the next step based on memory and reflection] +- Given what I've learned, what should I do next? +- Please explain why this plan is helpful for the next action? +- How does this action fit into my overall strategy? +- What do I expect this action to achieve? + + +Finally, choose ONE admissible action for the current step and present it within the tags. + +action: [tool_name] +parameters: {{"param1": "value1", "param2": "value2"}} + + +""" + diff --git a/openmanus_rl/environments/prompts/webshop.py b/openmanus_rl/environments/prompts/webshop.py index 927b6618..46112901 100644 --- a/openmanus_rl/environments/prompts/webshop.py +++ b/openmanus_rl/environments/prompts/webshop.py @@ -1,29 +1,79 @@ +# Copyright 2025 Nanyang Technological University (NTU), Singapore +# and the verl-agent (GiGPO) team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # --------------------- WebShop --------------------- # WEBSHOP_TEMPLATE_NO_HIS = """ -You are an expert autonomous agent operating in the WebShop e‑commerce environment. -Your task is to: {task_description}. -Your current observation is: {current_observation}. -Your admissible actions of the current situation are: -[ +You are an expert agent operating in the WebShop e‑commerce environment. +Your task is: {task_description} +Your current observation is: {current_observation} +Your admissible actions of the current situation are: [ {available_actions} ]. -Now it's your turn to take one action for the current step. -You should first reason step-by-step about the current situation, then think carefully which admissible action best advances the shopping goal. This reasoning process MUST be enclosed within tags. -Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. +Please begin by analyzing the situation and planning your approach: + + +Analyze the current shopping situation and devise a plan to accomplish the task: {task_description} +What are the key steps needed to complete this task (e.g., search with the right keywords, open a relevant product, compare options, select attributes, finalize)? +Based on the current observation, what should be my immediate next step? +How does this action advance my plan toward completing the shopping goal? + + +Finally, choose ONE admissible action for the current step and present it within tags. """ WEBSHOP_TEMPLATE = """ -You are an expert autonomous agent operating in the WebShop e‑commerce environment. -Your task is to: {task_description}. +You are an expert agent operating in the WebShop e‑commerce environment. +Your task is: {task_description} Prior to this step, you have already taken {step_count} step(s). Below are the most recent {history_length} observations and the corresponding actions you took: {action_history} -You are now at step {current_step} and your current observation is: {current_observation}. -Your admissible actions of the current situation are: -[ +You are now at step {current_step} and your current observation is: {current_observation} +Your admissible actions of the current situation are: [ {available_actions} ]. -Now it's your turn to take one action for the current step. -You should first reason step-by-step about the current situation, then think carefully which admissible action best advances the shopping goal. This reasoning process MUST be enclosed within tags. -Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. -""" \ No newline at end of file +Now it's your turn to take an action. + +You should first recall relevant past experience and reason from our conversation history, then MUST summarize within tags like this: + + +[Recall relevant past experiences and reason from our conversation history] +Recent action history ({step_count} steps taken): {action_history} +- What similar shopping situations have I encountered? +- What strategies worked or failed before (e.g., search terms, product filtering, option selection)? +- What products, attributes, or pages have I already explored? +- What was my previous reasoning and plan? + + +After that, you should reflect on the last action and its outcome, then MUST summarize within tags like this: + + +[Reflect on the last action and its outcome] +- What did my last action accomplish? +- Was it successful or did it encounter issues? +- How does this outcome affect my plan? +- Am I making progress toward the task goal: {task_description}? + + +After that, you should plan the next step based on memory and reflection, then MUST summarize within tags like this: + + +[Plan the next step based on memory and reflection] +- Given what I've learned, what should I do next? +- How does this action fit into my overall shopping strategy? +- What do I expect this action to achieve now? + + +Finally, choose ONE admissible action for the current step and present it within tags. +""" diff --git a/openmanus_rl/memory/__init__.py b/openmanus_rl/memory/__init__.py index d175ad7b..e93a610e 100644 --- a/openmanus_rl/memory/__init__.py +++ b/openmanus_rl/memory/__init__.py @@ -1,4 +1,5 @@ from .memory import SimpleMemory from .file_memory import FileMemory +from .summarized_memory import SummarizedMemory -__all__ = ['SimpleMemory', 'FileMemory'] \ No newline at end of file +__all__ = ['SimpleMemory', 'FileMemory', 'SummarizedMemory'] \ No newline at end of file diff --git a/openmanus_rl/memory/summarized_memory.py b/openmanus_rl/memory/summarized_memory.py new file mode 100644 index 00000000..82fcb9ff --- /dev/null +++ b/openmanus_rl/memory/summarized_memory.py @@ -0,0 +1,190 @@ +import requests +import logging +from typing import List, Tuple, Optional +from .memory import SimpleMemory + +logger = logging.getLogger(__name__) + + +def simple_summarize(history_steps: List[str], api_key: str = None, endpoint: str = None) -> str: + """ + Simple function to summarize history steps using LLM API. + + Args: + history_steps: List of formatted history strings + api_key: OpenAI API key + endpoint: API endpoint URL + + Returns: + Summarized history string + """ + if not api_key or not endpoint: + # Fallback: return truncated recent history + return "\n".join(history_steps[-3:]) # Last 3 steps + + # Join all history into one text + full_history = "\n".join(history_steps) + + prompt = f"""Compress this ALFRED history into a current state snapshot. + +Output EXACTLY these labeled lines (one line each, ASCII only): +Task: +Location: +Inventory: +Discovered: +KeyEvents: <1-2 important actions and outcomes> + +Rules: +- Facts only; no suggestions or analysis. +- Do not copy long quotes; use key nouns. +- If unknown, write 'unknown'. +- Total length <= 600 characters. + +History to summarize: +{full_history}""" + + try: + headers = { + "api-key": api_key, + "Content-Type": "application/json" + } + + # Azure OpenAI format + url = f"{endpoint}/openai/deployments/gpt-4o/chat/completions?api-version=2024-05-13" + + payload = { + "messages": [ + {"role": "system", "content": "You are a helpful assistant that summarizes task progress concisely."}, + {"role": "user", "content": prompt} + ], + "max_tokens": 300, + "temperature": 0.1 + } + + response = requests.post(url, headers=headers, json=payload, timeout=15) + + if response.status_code == 200: + content = response.json()['choices'][0]['message']['content'] + logger.debug(f"Summary generated: {len(content)} chars") + return content.strip() + else: + logger.warning(f"API error {response.status_code}, using fallback") + return "\n".join(history_steps[-3:]) + + except Exception as e: + logger.warning(f"Summarization failed: {e}, using fallback") + return "\n".join(history_steps[-3:]) + + +class SummarizedMemory(SimpleMemory): + """ + Memory manager with summarization capability. + Inherits from SimpleMemory and adds optional history summarization. + """ + + def __init__(self): + super().__init__() + self.summaries = [] # Cache summaries for each environment + self.last_summary_step = [] # Track when each env was last summarized + + def reset(self, batch_size: int): + """Reset memory and summary caches.""" + super().reset(batch_size) + self.summaries = [None] * batch_size + self.last_summary_step = [0] * batch_size + + def fetch( + self, + history_length: int, + obs_key: str = "text_obs", + action_key: str = "action", + use_summary: bool = False, + summary_api_key: str = None, + summary_endpoint: str = None, + summary_threshold: Optional[int] = None, # kept for backward compatibility, ignored + ) -> Tuple[List[str], List[int]]: + """ + Fetch history with optional summarization. + + Strategy: + - 1 step: return original history (no summarization needed) + - >1 steps: return summarized history (information compression) + + Args: + history_length: Max steps for regular mode (ignored in summary mode) + obs_key: Key for observations + action_key: Key for actions + use_summary: Whether to use summarization + summary_api_key: API key for LLM + summary_endpoint: API endpoint for LLM + + Returns: + Tuple of (memory_contexts, valid_lengths) + """ + if not use_summary: + # Use original SimpleMemory behavior + return super().fetch(history_length, obs_key, action_key) + + return self._fetch_with_summary( + obs_key, action_key, summary_api_key, summary_endpoint + ) + + def _fetch_with_summary( + self, + obs_key: str, + action_key: str, + api_key: str, + endpoint: str + ) -> Tuple[List[str], List[int]]: + """Fetch history using summarization strategy.""" + memory_contexts, valid_lengths = [], [] + + for env_idx in range(self.batch_size): + total_steps = len(self._data[env_idx]) + + if total_steps <= 1: + # Only 1 step, use regular history (no need to summarize) + ctx, vlen = super().fetch(1, obs_key=obs_key, action_key=action_key) + memory_contexts.append(ctx[0]) + valid_lengths.append(vlen[0]) + else: + # More than 1 step, use summarization + summary_context = self._get_or_create_summary( + env_idx, obs_key, action_key, api_key, endpoint + ) + memory_contexts.append(summary_context) + valid_lengths.append(total_steps) # Return total steps covered + + return memory_contexts, valid_lengths + + def _get_or_create_summary( + self, + env_idx: int, + obs_key: str, + action_key: str, + api_key: str, + endpoint: str + ) -> str: + """Get existing summary or create a new one.""" + total_steps = len(self._data[env_idx]) + + # Update summary whenever step count has advanced (or first time) + if self.summaries[env_idx] is None or total_steps != self.last_summary_step[env_idx]: + + # Create formatted history for all steps + all_history = [] + for j, rec in enumerate(self._data[env_idx]): + step_num = j + 1 + act = rec[action_key] + obs = rec[obs_key] + all_history.append( + f"[Observation {step_num}: '{obs}', Action {step_num}: '{act}']" + ) + + # Generate summary + self.summaries[env_idx] = simple_summarize(all_history, api_key, endpoint) + self.last_summary_step[env_idx] = total_steps + + logger.debug(f"Updated summary for env {env_idx}, covering {total_steps} steps") + + return self.summaries[env_idx] diff --git a/openmanus_rl/tools/advanced_object_detector/tool.py b/openmanus_rl/tools/advanced_object_detector/tool.py index 9e4d279d..d83135c3 100644 --- a/openmanus_rl/tools/advanced_object_detector/tool.py +++ b/openmanus_rl/tools/advanced_object_detector/tool.py @@ -4,7 +4,7 @@ import os import time -from octotools.tools.base import BaseTool +from openmanus_rl.tools.base import BaseTool from PIL import Image, ImageOps import os @@ -233,4 +233,4 @@ def get_metadata(self): except ValueError as e: print(f"Execution failed: {e}") - print("Done!") \ No newline at end of file + print("Done!") diff --git a/openmanus_rl/tools/arxiv_paper_searcher/tool.py b/openmanus_rl/tools/arxiv_paper_searcher/tool.py index 77391514..88da75b4 100644 --- a/openmanus_rl/tools/arxiv_paper_searcher/tool.py +++ b/openmanus_rl/tools/arxiv_paper_searcher/tool.py @@ -2,7 +2,7 @@ import requests from bs4 import BeautifulSoup -from octotools.tools.base import BaseTool +from openmanus_rl.tools.base import BaseTool class ArXiv_Paper_Searcher_Tool(BaseTool): def __init__(self): diff --git a/openmanus_rl/tools/generalist_solution_generator/tool.py b/openmanus_rl/tools/generalist_solution_generator/tool.py index 9d970c2f..eab5dc14 100644 --- a/openmanus_rl/tools/generalist_solution_generator/tool.py +++ b/openmanus_rl/tools/generalist_solution_generator/tool.py @@ -1,6 +1,6 @@ import os -from octotools.tools.base import BaseTool -from octotools.engine.factory import create_llm_engine +from openmanus_rl.tools.base import BaseTool +from openmanus_rl.engines.factory import create_llm_engine class Generalist_Solution_Generator_Tool(BaseTool): require_llm_engine = True diff --git a/openmanus_rl/tools/google_search/tool.py b/openmanus_rl/tools/google_search/tool.py index ee524fc4..bb518d25 100644 --- a/openmanus_rl/tools/google_search/tool.py +++ b/openmanus_rl/tools/google_search/tool.py @@ -2,7 +2,7 @@ import requests from typing import List, Dict, Any -from octotools.tools.base import BaseTool +from openmanus_rl.tools.base import BaseTool from dotenv import load_dotenv load_dotenv() diff --git a/openmanus_rl/tools/image_captioner/tool.py b/openmanus_rl/tools/image_captioner/tool.py index 607935ad..74f469f5 100644 --- a/openmanus_rl/tools/image_captioner/tool.py +++ b/openmanus_rl/tools/image_captioner/tool.py @@ -1,6 +1,6 @@ import os -from octotools.tools.base import BaseTool -from octotools.engine.factory import create_llm_engine +from openmanus_rl.tools.base import BaseTool +from openmanus_rl.engines.factory import create_llm_engine class Image_Captioner_Tool(BaseTool): require_llm_engine = True diff --git a/openmanus_rl/tools/nature_news_fetcher/tool.py b/openmanus_rl/tools/nature_news_fetcher/tool.py index 6e1ed528..bfe77143 100644 --- a/openmanus_rl/tools/nature_news_fetcher/tool.py +++ b/openmanus_rl/tools/nature_news_fetcher/tool.py @@ -3,7 +3,7 @@ from bs4 import BeautifulSoup import time -from octotools.tools.base import BaseTool +from openmanus_rl.tools.base import BaseTool class Nature_News_Fetcher_Tool(BaseTool): def __init__(self): diff --git a/openmanus_rl/tools/object_detector/tool.py b/openmanus_rl/tools/object_detector/tool.py index b07fbaba..15291dd0 100644 --- a/openmanus_rl/tools/object_detector/tool.py +++ b/openmanus_rl/tools/object_detector/tool.py @@ -6,7 +6,7 @@ import torch from transformers import pipeline -from octotools.tools.base import BaseTool +from openmanus_rl.tools.base import BaseTool from PIL import Image, ImageOps import os @@ -176,4 +176,4 @@ def get_metadata(self): except ValueError as e: print(f"Execution failed: {e}") - print("Done!") \ No newline at end of file + print("Done!") diff --git a/openmanus_rl/tools/pubmed_search/tool.py b/openmanus_rl/tools/pubmed_search/tool.py index 003ed365..cc12d861 100644 --- a/openmanus_rl/tools/pubmed_search/tool.py +++ b/openmanus_rl/tools/pubmed_search/tool.py @@ -2,7 +2,7 @@ import json from pymed import PubMed from metapub import PubMedFetcher -from octotools.tools.base import BaseTool +from openmanus_rl.tools.base import BaseTool from tenacity import ( retry, stop_after_attempt, @@ -109,4 +109,4 @@ def get_metadata(self): except ValueError as e: print(f"Execution failed: {e}") - print("Done!") \ No newline at end of file + print("Done!") diff --git a/openmanus_rl/tools/python_code_generator/tool.py b/openmanus_rl/tools/python_code_generator/tool.py index 3f417073..cc3ed74a 100644 --- a/openmanus_rl/tools/python_code_generator/tool.py +++ b/openmanus_rl/tools/python_code_generator/tool.py @@ -7,8 +7,8 @@ import contextlib import threading -from octotools.tools.base import BaseTool -from octotools.engine.factory import create_llm_engine +from openmanus_rl.tools.base import BaseTool +from openmanus_rl.engines.factory import create_llm_engine import signal from contextlib import contextmanager diff --git a/openmanus_rl/tools/relevant_patch_zoomer/tool.py b/openmanus_rl/tools/relevant_patch_zoomer/tool.py index fc53e5f2..401c7f70 100644 --- a/openmanus_rl/tools/relevant_patch_zoomer/tool.py +++ b/openmanus_rl/tools/relevant_patch_zoomer/tool.py @@ -1,8 +1,8 @@ import os import cv2 from pydantic import BaseModel -from octotools.tools.base import BaseTool -from octotools.engine.factory import create_llm_engine +from openmanus_rl.tools.base import BaseTool +from openmanus_rl.engines.factory import create_llm_engine class PatchZoomerResponse(BaseModel): analysis: str diff --git a/openmanus_rl/tools/text_detector/tool.py b/openmanus_rl/tools/text_detector/tool.py index b98b0aa9..f53fbfea 100644 --- a/openmanus_rl/tools/text_detector/tool.py +++ b/openmanus_rl/tools/text_detector/tool.py @@ -2,7 +2,7 @@ import os import time -from octotools.tools.base import BaseTool +from openmanus_rl.tools.base import BaseTool import warnings warnings.filterwarnings("ignore") diff --git a/openmanus_rl/tools/url_text_extractor/tool.py b/openmanus_rl/tools/url_text_extractor/tool.py index 1fc7ac2b..21fd23a3 100644 --- a/openmanus_rl/tools/url_text_extractor/tool.py +++ b/openmanus_rl/tools/url_text_extractor/tool.py @@ -2,7 +2,7 @@ import requests from bs4 import BeautifulSoup -from octotools.tools.base import BaseTool +from openmanus_rl.tools.base import BaseTool class URL_Text_Extractor_Tool(BaseTool): def __init__(self): diff --git a/openmanus_rl/tools/wikipedia_knowledge_searcher/tool.py b/openmanus_rl/tools/wikipedia_knowledge_searcher/tool.py index 8cb2deb6..e62a2670 100644 --- a/openmanus_rl/tools/wikipedia_knowledge_searcher/tool.py +++ b/openmanus_rl/tools/wikipedia_knowledge_searcher/tool.py @@ -1,7 +1,7 @@ import os import wikipedia -from octotools.tools.base import BaseTool +from openmanus_rl.tools.base import BaseTool class Wikipedia_Knowledge_Searcher_Tool(BaseTool): def __init__(self): diff --git a/requirements_docker.txt b/requirements_docker.txt new file mode 100644 index 00000000..415c0b82 --- /dev/null +++ b/requirements_docker.txt @@ -0,0 +1,17 @@ +# Core dependencies for OpenManus-RL in AMD vllm Docker +gymnasium==0.29.1 +stable-baselines3==2.6.0 +alfworld +pyyaml +openai +together +ray +numpy +pandas +tqdm +transformers +torch +vllm +requests +python-dotenv +wikipedia diff --git a/scripts/docker_setup.sh b/scripts/docker_setup.sh new file mode 100755 index 00000000..2b371afa --- /dev/null +++ b/scripts/docker_setup.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Setup script for OpenManus-RL Docker environment on AMD GPUs + +set -e + +echo "=========================================" +echo "OpenManus-RL Docker Setup for AMD GPUs" +echo "=========================================" + +# Step 1: Stop and remove existing OpenManus container if it exists +echo "Cleaning up existing OpenManus container..." +docker stop openmanus-rl 2>/dev/null || true +docker rm openmanus-rl 2>/dev/null || true + +# Step 2: Create a new container from the existing snapshot image +echo "Starting new OpenManus-RL container..." +docker run -it -d --name openmanus-rl \ + --ipc=host --shm-size=64g \ + --device=/dev/kfd --device=/dev/dri --group-add video \ + -e HIP_VISIBLE_DEVICES=0 \ + -v "$PWD:/workspace" \ + -v "/root/models:/root/models" \ + -v "$HOME/.cache/huggingface:/root/.cache/huggingface" \ + -p 8001:8000 \ + -w /workspace \ + verl-agent:rocm-snap1 bash + +echo "Container started. Setting up environment..." + +# Step 3: Install dependencies inside the container +docker exec -it openmanus-rl bash -c ' +export PATH="$HOME/.local/bin:$PATH" +command -v uv || (curl -LsSf https://astral.sh/uv/install.sh | sh) + +# Create virtual environment +uv venv /opt/openmanus-venv +. /opt/openmanus-venv/bin/activate + +# Install required packages +uv pip install gymnasium==0.29.1 stable-baselines3==2.6.0 alfworld +alfworld-download -f +uv pip install -e . --no-deps +uv pip install pyyaml +uv pip install -U openai +uv pip install Ray +uv pip install together +uv pip install wikipedia python-dotenv requests + +echo "Environment setup complete!" +' + +echo "=========================================" +echo "Setup complete! You can now:" +echo "1. Enter the container: docker exec -it openmanus-rl bash" +echo "2. Activate the environment: source /opt/openmanus-venv/bin/activate" +echo "3. Run the unified script from /workspace" +echo "=========================================" + diff --git a/scripts/gaia_calculate_score.py b/scripts/gaia_calculate_score.py new file mode 100644 index 00000000..330f5c9b --- /dev/null +++ b/scripts/gaia_calculate_score.py @@ -0,0 +1,212 @@ +""" +Reference GAIA scoring script. + +Note: This file depends on external utilities (e.g., tasks.utils.ResultAnalyzer) +that are not part of this repository. It is provided for reference only. +""" + +import concurrent.futures +import os +import json +import argparse +import tqdm + +from pydantic import BaseModel +from openmanus_rl.engines.openai import ChatOpenAI + +try: + from tasks.utils import ResultAnalyzer # external utility, may not exist in this repo +except Exception: + ResultAnalyzer = None + +class AnswerVerification(BaseModel): + analysis: str + true_false: bool + +class BinaryAnswerVerification(BaseModel): + true_false: bool + +class ResultScorer: + def __init__(self, llm_engine=None): + self.llm_engine = llm_engine or ChatOpenAI(model="gpt-4o-mini") + try: + model_name = getattr(self.llm_engine, "model", "unknown") + except Exception: + model_name = "unknown" + print(f"\nLocal OpenAI engine {model_name} initialized.\n") + + def answer_verification(self, response, correct_answer): + query_prompt = f""" + Compare the model's response against the correct answer following these evaluation rules: + + Model response: {response} + Correct answer: {correct_answer} + + Evaluation rules: + 1. Extract the core answer from the model response (ignore explanations or additional context) + 2. The answer is correct if it EXACTLY matches the correct answer: + - Numbers must match precisely (e.g., "142" = "142") + - Text must match case-sensitive (e.g., "Time-Parking 2: Parallel Universe") + - Zip codes must be exact (e.g., "34689") + - No partial credit for similar or related answers + 3. The answer is incorrect if: + - It contains any additional or missing information + - It uses different formatting or representations + - It's semantically equivalent but not identical + + Response Format: + : Extract the core answer and explain exact match comparison + : Return "True" only for exact matches, otherwise "False" + """ + + verification = self.llm_engine(query_prompt, response_format=AnswerVerification) + + analysis = verification.analysis.strip() + true_false = verification.true_false + + return analysis, true_false + + def score_results(self, results, max_workers=10): + correct = 0 + + def process_single_result(pid_data): + pid, question_data = pid_data + response = question_data["response"] + correct_answer = question_data["correct_answer"] + analysis, true_false = self.answer_verification(response, correct_answer) + return pid, analysis, true_false + + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(process_single_result, (pid, data)) + for pid, data in results.items()] + + for future in tqdm.tqdm(concurrent.futures.as_completed(futures), + total=len(futures), + desc="Scoring results"): + pid, analysis, true_false = future.result() + correct += 1 if true_false else 0 + results[pid].update({ + "stepwise_analysis": analysis, + "true_false": true_false + }) + + return results, correct + + +def load_data(data_file, result_dir, response_type): + # Load the benchmark data + with open(data_file, 'r') as f: + # convert the benchmark data to a dictionary + benchmark_data = {data["pid"]: data for data in json.load(f)} + + # Load the results + results = {} + for file in os.listdir(result_dir): + if file.endswith(".json") and "output_" in file: + with open(os.path.join(result_dir, file), 'r') as f: + result = json.load(f) + + # Get the index of the result + index = file.replace(".json", "").replace("output_", "") # "0", "1", "2", ... + pid = str(int(index)) # NOTE adjust the index to match the pid + assert result["pid"] == benchmark_data[pid]["pid"] + + # Save the results + results[pid] = benchmark_data[pid] + assert response_type in result + results[pid]["response"] = result[response_type] + results[pid]["correct_answer"] = benchmark_data[pid]["answer"] + + return results + + +def parse_args(): + parser = argparse.ArgumentParser(description="Extract and score the results from the benchmark data") + parser.add_argument("--data_file", type=str, default="data/validation.json", help="The file containing the benchmark data") + parser.add_argument("--result_dir", type=str, default=None, help="The directory containing the results") + parser.add_argument("--output_file", type=str, default="final_results.json", help="The file to save the extracted results") + parser.add_argument("--log_dir", type=str, default=None, help="The directory containing the logs") + parser.add_argument("--response_type", type=str, default="direct_output", + choices=["final_output", "direct_output", "base_response"], + help="The type of response to extract from the results") + parser.add_argument("--max_workers", type=int, default=16, help="The maximum number of workers to use") + return parser.parse_args() + +if __name__ == "__main__": + + args = parse_args() + + # Load and print the arguments + print("#"*50) + print(f"Arguments: {args}") + for arg, value in args.__dict__.items(): + print(f"# {arg}: {value}") + print("#"*50) + + scorer = ResultScorer() + analyzer = ResultAnalyzer() if ResultAnalyzer else None + + # Load the results + results = load_data(args.data_file, args.result_dir, args.response_type) + + # Score the results + results, correct = scorer.score_results(results, max_workers=args.max_workers) + + # Calculate accuracy and wrong answers + acc = round(correct / len(results) * 100, 2) + print(f"\nAccuracy: {acc}% ({correct}/{len(results)})") + + # Save detailed results + output_file = os.path.join(args.result_dir, args.output_file) + with open(output_file, 'w') as f: + json.dump(results, f, indent=4) + print(f"\nResults saved to {output_file}") + + # Calculate wrong answers + wrong_pids = [pid for pid, data in results.items() if not data["true_false"]] + wrong_pids = sorted(wrong_pids, key=lambda x: int(x)) + wrong_indices = [int(pid) for pid in wrong_pids] + print(f"Wrong PIDs: {wrong_pids}") + print(f"Wrong Indices: {wrong_indices}") + + scores = { + "correct": correct, + "total": len(results), + "accuracy": acc, + "wrong_pids": wrong_pids, + "wrong_indices": wrong_indices + } + + # Calculate additional statistics if log directory is provided + log_dir = args.log_dir or args.result_dir.replace("results", "logs") + if analyzer and os.path.exists(log_dir): + + if args.response_type == "base_response": + print("Base response is not supported for scoring.") + print("Exited.\n") + exit() + + # Calculate the average time and steps + step_stats = analyzer.calculate_time_steps(log_dir) + print(f"\nStep stats:") + for key, value in step_stats.items(): + print(f"- {key}: \t{value}") + + # Calculate the usage of tools + tool_usage = analyzer.calculate_tool_usage(args.result_dir) + print(f"\nTool usage:") + for tool, ratio in tool_usage.items(): + print(f"- {tool}: \t{ratio}") + + # Update the scores + scores.update({ + "step_stats": step_stats, + "tool_usage": tool_usage + }) + + + # Save the scores + score_file = os.path.join(args.result_dir, f"final_scores_{args.response_type}.json") + with open(score_file, 'w') as f: + json.dump(scores, f, indent=4) + print(f"Scores saved to {score_file}") diff --git a/scripts/process_tool_use_data.py b/scripts/process_tool_use_data.py new file mode 100644 index 00000000..32ff5841 --- /dev/null +++ b/scripts/process_tool_use_data.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +""" +Data Processing Script for Tool Use Environment +Converts data.json to training format compatible with OpenManus RL system +""" + +import json +import argparse +import os +from typing import List, Dict, Any +import pandas as pd +from sklearn.model_selection import train_test_split + + +def load_data(data_path: str) -> List[Dict]: + """Load data from JSON file""" + with open(data_path, 'r') as f: + data = json.load(f) + return data + + +def filter_data_by_level(data: List[Dict], level: str = None) -> List[Dict]: + """Filter data by difficulty level if specified""" + if level is None: + return data + + filtered_data = [] + for item in data: + item_level = item.get('Level', '1') + if item_level == str(level): + filtered_data.append(item) + + return filtered_data + + +def convert_to_training_format(data: List[Dict]) -> List[Dict]: + """ + Convert raw data to training format expected by the system. + Each item contains the task and expected answer. + """ + training_data = [] + + for item in data: + # Extract required fields + training_item = { + 'pid': str(item.get('pid', len(training_data))), + 'question': item.get('question', ''), + 'answer': item.get('answer', ''), + 'task_id': item.get('task_id', ''), + 'level': item.get('Level', '1'), + 'split': item.get('split', 'train') # Default to train if not specified + } + + # Add optional metadata + if 'Annotator Metadata' in item: + metadata = item['Annotator Metadata'] + training_item['metadata'] = { + 'steps': metadata.get('Steps', ''), + 'num_steps': metadata.get('Number of steps', ''), + 'tools': metadata.get('Tools', ''), + 'num_tools': metadata.get('Number of tools', ''), + 'time_taken': metadata.get('How long did this take?', '') + } + + training_data.append(training_item) + + return training_data + + +def split_data(data: List[Dict], train_ratio: float = 0.8, val_ratio: float = 0.2) -> Dict[str, List[Dict]]: + """ + Split data into train/val sets if not already split. + """ + # Check if data already has split field + has_splits = any('split' in item for item in data) + + if has_splits: + # Use existing splits + train_data = [item for item in data if item.get('split', 'train') == 'train'] + val_data = [item for item in data if item.get('split', 'train') in ['val', 'validation']] + test_data = [item for item in data if item.get('split', 'train') == 'test'] + + print(f"Using existing splits: {len(train_data)} train, {len(val_data)} val, {len(test_data)} test") + + return { + 'train': train_data, + 'val': val_data, + 'test': test_data + } + else: + # Create new splits + train_data, temp_data = train_test_split(data, test_size=(1-train_ratio), random_state=42) + val_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42) + + # Update split field + for item in train_data: + item['split'] = 'train' + for item in val_data: + item['split'] = 'val' + for item in test_data: + item['split'] = 'test' + + print(f"Created new splits: {len(train_data)} train, {len(val_data)} val, {len(test_data)} test") + + return { + 'train': train_data, + 'val': val_data, + 'test': test_data + } + + +def save_data(data_splits: Dict[str, List[Dict]], output_dir: str): + """Save processed data to output directory""" + os.makedirs(output_dir, exist_ok=True) + + for split_name, split_data in data_splits.items(): + if not split_data: # Skip empty splits + continue + + # Save as JSON + json_path = os.path.join(output_dir, f"{split_name}.json") + with open(json_path, 'w') as f: + json.dump(split_data, f, indent=2, ensure_ascii=False) + + # Save as Parquet (compatible with existing data format) + parquet_path = os.path.join(output_dir, f"{split_name}.parquet") + df = pd.DataFrame(split_data) + df.to_parquet(parquet_path, index=False) + + print(f"Saved {len(split_data)} {split_name} samples to {json_path} and {parquet_path}") + + +def analyze_data(data: List[Dict]): + """Print data analysis""" + print("\n" + "="*50) + print("DATA ANALYSIS") + print("="*50) + + print(f"Total samples: {len(data)}") + + # Analyze by level + levels = {} + for item in data: + level = item.get('level', '1') + levels[level] = levels.get(level, 0) + 1 + + print("\nBy difficulty level:") + for level in sorted(levels.keys()): + print(f" Level {level}: {levels[level]} samples") + + # Analyze answer lengths + answer_lengths = [len(str(item.get('answer', ''))) for item in data] + print(f"\nAnswer length statistics:") + print(f" Min: {min(answer_lengths)}, Max: {max(answer_lengths)}") + print(f" Average: {sum(answer_lengths) / len(answer_lengths):.1f}") + + # Check for tools mentioned in metadata + tools_mentioned = set() + for item in data: + metadata = item.get('metadata', {}) + tools = metadata.get('tools', '') + if tools: + # Simple extraction of tool names + if 'search' in tools.lower(): + tools_mentioned.add('search') + if 'browser' in tools.lower(): + tools_mentioned.add('browser') + if 'image' in tools.lower(): + tools_mentioned.add('image_processing') + + print(f"\nTools mentioned in metadata: {list(tools_mentioned)}") + + +def main(): + parser = argparse.ArgumentParser(description="Process tool use data for training") + parser.add_argument("--input", type=str, default="../data/gaia/val.json", + help="Input data file path") + parser.add_argument("--output", type=str, default="../data/gaia", + help="Output directory for processed data") + parser.add_argument("--level", type=str, default=None, + help="Filter by difficulty level (1, 2, 3, etc.)") + parser.add_argument("--train_ratio", type=float, default=0.8, + help="Training data ratio (default: 0.8)") + parser.add_argument("--val_ratio", type=float, default=0.2, + help="Validation data ratio (default: 0.2)") + parser.add_argument("--max_samples", type=int, default=None, + help="Maximum number of samples to process") + + args = parser.parse_args() + + print("Processing Tool Use Data") + print(f"Input: {args.input}") + print(f"Output: {args.output}") + + # Load data + print("Loading data...") + raw_data = load_data(args.input) + print(f"Loaded {len(raw_data)} samples") + + # Filter by level if specified + if args.level: + raw_data = filter_data_by_level(raw_data, args.level) + print(f"Filtered to {len(raw_data)} samples for level {args.level}") + + # Limit samples if specified + if args.max_samples and len(raw_data) > args.max_samples: + raw_data = raw_data[:args.max_samples] + print(f"Limited to {len(raw_data)} samples") + + # Convert to training format + print("Converting to training format...") + training_data = convert_to_training_format(raw_data) + + # Analyze data + analyze_data(training_data) + + # Split data + print("\nSplitting data...") + data_splits = split_data(training_data, args.train_ratio, args.val_ratio) + + # Save processed data + print("\nSaving processed data...") + save_data(data_splits, args.output) + + print(f"\nProcessing complete! Data saved to {args.output}") + print("\nNext steps:") + print(f"1. Update your config file to use env_name: 'tool_use'") + print(f"2. Set data_path: '{args.output}/train.json' in config") + print(f"3. Configure available_tools in your config") + + +if __name__ == "__main__": + main() diff --git a/scripts/rollout/gaia.py b/scripts/rollout/gaia.py new file mode 100644 index 00000000..a7937e50 --- /dev/null +++ b/scripts/rollout/gaia.py @@ -0,0 +1,454 @@ +import os +try: + from dotenv import load_dotenv # type: ignore + load_dotenv() +except Exception: + pass +import time +import json +import logging +import argparse +from types import SimpleNamespace +from datetime import datetime +from typing import List +from concurrent.futures import ThreadPoolExecutor, as_completed +from collections import defaultdict +import numpy as np +import random +import sys +from openmanus_rl.environments.env_manager import * +from openai import OpenAI +from together import Together + + +def build_env(env_name, tasks_data, available_tools, env_num=1, seed=1, history_length=2, max_steps=30): + group_n = 1 + if env_name == "gaia": + # Build GAIA/Tool Use environments + from openmanus_rl.environments.env_package.tool_use.projection import tool_use_projection + from openmanus_rl.environments.env_package.tool_use.envs import build_tool_use_envs + from openmanus_rl.environments.env_package.tool_use.manager import ToolUseEnvironmentManager + + envs = build_tool_use_envs( + tasks_data=tasks_data, + available_tools=available_tools, + seed=seed, + env_num=env_num, + group_n=group_n, + is_train=True + ) + + # Minimal config object with required fields + cfg = SimpleNamespace( + env=SimpleNamespace( + env_name="tool_use", + history_length=history_length, + max_steps=max_steps # Controlled by CLI + ) + ) + env_manager = ToolUseEnvironmentManager(envs, tool_use_projection, cfg) + else: + raise ValueError(f"Unsupported environment name: {env_name}") + + return env_manager + + +class Agent: + def __init__(self, model_name="gpt-4o", temperature: float = 0.4, base_url: str | None = None): + self.model_name = model_name + self.temperature = temperature + + # Check if model is a Together model (contains "/" and no base_url provided) + self.is_together = "/" in model_name and base_url is None + + if self.is_together: + self.client = Together( + api_key=os.environ.get('TOGETHER_API_KEY', ''), + ) + elif base_url: + self.client = OpenAI( + api_key=os.getenv('OPENAI_API_KEY', 'EMPTY'), + base_url=base_url, + ) + else: + self.client = OpenAI( + api_key=os.environ['OPENAI_API_KEY'], + ) + + def get_action_from_gpt(self, obs): + response = self.client.chat.completions.create( + model=self.model_name, + messages=[ + { + "role": "user", + "content": obs + } + ], + temperature=self.temperature, + n=1, + ) + action = response.choices[0].message.content.strip() + return action + + def get_actions_batch(self, prompts: List[str], concurrency: int = 4, retries: int = 3, backoff: float = 0.5) -> List[str]: + actions = [None] * len(prompts) + + def _one(idx_prompt): + idx, prompt = idx_prompt + delay = backoff + for attempt in range(retries): + try: + act = self.get_action_from_gpt(prompt) + return idx, act + except Exception as e: + if attempt == retries - 1: + return idx, "None" + time.sleep(delay) + delay *= 2 + + with ThreadPoolExecutor(max_workers=max(1, concurrency)) as ex: + futures = [ex.submit(_one, (i, p)) for i, p in enumerate(prompts)] + for fut in as_completed(futures): + i, act = fut.result() + actions[i] = act + + return actions + + +def load_gaia_tasks(data_path: str, max_tasks: int = None) -> List[dict]: + """Load GAIA tasks from JSON file""" + with open(data_path, 'r', encoding='utf-8') as f: + tasks = json.load(f) + + if max_tasks: + tasks = tasks[:max_tasks] + + return tasks + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--env_name", default="gaia") + parser.add_argument("--batch_size", type=int, default=10, help="Number of envs to process per batch") + parser.add_argument("--total_envs", type=int, default=100, help="Total number of environments to rollout") + parser.add_argument("--test_times", type=int, default=1) + parser.add_argument("--max_steps", type=int, default=30, help="Maximum steps per task") + parser.add_argument("--seed", type=int, default=1) + parser.add_argument("--history_length", type=int, default=2) + parser.add_argument("--model", default="gpt-4o-mini", help="Model name (OpenAI: gpt-4o, gpt-4o-mini; Together: Qwen/Qwen2.5-7B-Instruct-Turbo, etc.)") + parser.add_argument("--temperature", type=float, default=0.4) + parser.add_argument("--concurrency", type=int, default=4, help="Max concurrent OpenAI requests per step") + parser.add_argument("--retries", type=int, default=3, help="Retries per request on failure") + parser.add_argument("--dump_path", default=None, help="If set, write JSONL trajectory to this file") + parser.add_argument("--base_url", default=None, help="OpenAI-compatible base URL (e.g., vLLM http://127.0.0.1:8000/v1)") + parser.add_argument("--chat_root", default=None, help="If set, save per-episode chat histories under this root") + parser.add_argument("--data_path", default="data/gaia/val.json", help="Path to GAIA dataset") + parser.add_argument("--tools", nargs='+', default=['google_search', 'wikipedia_knowledge_searcher', 'python_code_generator'], + help="List of available tools") + parser.add_argument("--dry_run", action="store_true", help="仅打印任务分配，不创建环境、不调用模型") + args = parser.parse_args() + + # -------- logging ---------- + os.makedirs("logs/gaia", exist_ok=True) + log_fp = os.path.join( + "logs/gaia", f"run_log_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + ) + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(message)s", + handlers=[logging.FileHandler(log_fp, encoding="utf-8"), logging.StreamHandler()], + ) + + # -------- Parameters ---------- + max_steps = args.max_steps + batch_size = args.batch_size + total_envs = args.total_envs + test_times = args.test_times + env_name = args.env_name + + # Load GAIA tasks + logging.info(f"Loading GAIA tasks from {args.data_path}") + all_tasks = load_gaia_tasks(args.data_path) + logging.info(f"Loaded {len(all_tasks)} total tasks from {args.data_path}") + + # Ensure we have enough tasks for requested envs + if len(all_tasks) < total_envs: + logging.warning(f"Only {len(all_tasks)} tasks available, adjusting total_envs from {total_envs} to {len(all_tasks)}") + total_envs = len(all_tasks) + + # Calculate number of batches needed + num_batches = (total_envs + batch_size - 1) // batch_size + logging.info(f"Running {total_envs} envs in {num_batches} batches of {batch_size}") + + # -------- Agent setup ---------- + agent = Agent(model_name=args.model, temperature=args.temperature, base_url=args.base_url) + + # Prepare trajectory dump file if requested + dump_fp = None + if args.dump_path: + os.makedirs(os.path.dirname(args.dump_path) or ".", exist_ok=True) + dump_fp = open(args.dump_path, "a", encoding="utf-8") + + # Prepare chat history directories if requested + run_ts = datetime.now().strftime('%Y%m%d_%H%M%S') + chat_ts_root = None + chat_base_dir = None + if args.chat_root: + chat_ts_root = os.path.join(args.chat_root, 'trajectories', run_ts) + chat_base_dir = os.path.join(chat_ts_root, args.env_name, args.model) + os.makedirs(chat_base_dir, exist_ok=True) + + def _sanitize(s: str) -> str: + return ''.join(c if c.isalnum() or c in ('-', '_', '.') else '-' for c in s)[:200] + + # Accumulated statistics across all batches + all_overall_success_rates = [] + all_task_success_history = defaultdict(list) + global_env_counter = 0 + + # Shuffle tasks for random sampling + rng = random.Random(args.seed) + rng.shuffle(all_tasks) + + # Dry-run mode + if args.dry_run: + logging.info(f"[Dry-Run] total_envs={total_envs}, batch_size={batch_size}, num_batches={num_batches}") + for b in range(num_batches): + start = b * batch_size + end = start + min(batch_size, total_envs - start) + batch_tasks = all_tasks[start:end] + pids = [t.get('pid', f'task_{i}') for i, t in enumerate(batch_tasks)] + logging.info(f"[Dry-Run] Batch {b+1:02d}: {len(batch_tasks)} tasks; PIDs: {', '.join(pids[:3])}...") + sys.exit(0) + + # ======================= Main Batch Loop ======================= + for batch_idx in range(num_batches): + # Calculate actual batch size for this batch (last batch might be smaller) + current_batch_size = min(batch_size, total_envs - batch_idx * batch_size) + logging.info(f"\n========== Starting Batch {batch_idx + 1}/{num_batches} with {current_batch_size} envs ==========") + + # Select tasks for this batch + start = batch_idx * batch_size + end = start + current_batch_size + batch_tasks = all_tasks[start:end] + + # Create environment for this batch + env_manager = build_env( + env_name, + tasks_data=batch_tasks, + available_tools=args.tools, + env_num=current_batch_size, + seed=args.seed + batch_idx, + history_length=args.history_length, + max_steps=args.max_steps, + ) + + # Batch-level statistics + batch_overall_success_rates = [] + batch_task_success_history = defaultdict(list) + + try: + # ======================= Test Loop for this Batch ======================= + for test_idx in range(test_times): + logging.info(f"\n========== Start Batch {batch_idx + 1} Test {test_idx} ==========") + start_time = time.time() + + obs, infos = env_manager.reset() + env_dones = [False] * current_batch_size + # Persist PIDs from reset infos for later logging (step infos omit pid) + pids = [infos[i].get("pid", f"task_{i}") for i in range(len(infos))] + + # per-env chat buffers + chats = [[] for _ in range(current_batch_size)] + # track which envs already dumped to disk + saved_flags = [False] * current_batch_size + # keep last infos for fallback dump (failure/timeout) + last_infos = infos + + # Statistics for single round + overall_success_this_round = np.zeros(current_batch_size, dtype=bool) + task_success_cnt = defaultdict(int) + task_total_cnt = defaultdict(int) + + for step_idx in range(max_steps): + logging.info(f"Batch {batch_idx + 1} Step {step_idx}; Dones ({np.array(env_dones).sum().item()}/{current_batch_size}); SR {overall_success_this_round.mean().item()}") + + # --- Assemble actions --- + prompts = [] + idx_map = [] # map from prompts index back to env index + for i in range(current_batch_size): + if not env_dones[i]: + prompts.append(obs["text"][i]) + idx_map.append(i) + + batch_actions = agent.get_actions_batch(prompts, concurrency=args.concurrency, retries=args.retries) + actions = ["None"] * current_batch_size + for k, i in enumerate(idx_map): + actions[i] = batch_actions[k] + + # --- Environment stepping --- + prev_prompts = obs["text"] # keep for logging & chat history + # Preserve the model's raw outputs for logging/chat before any projection mutates them + raw_actions = actions.copy() + # Pass a copy into the env manager so in-place projection does not alter our raw copy + obs, rewards, dones, infos = env_manager.step(actions.copy()) + last_infos = infos + + # --- Determine endings and successes --- + for i in range(current_batch_size): + if env_dones[i]: + continue + + # Append chat turns for acted envs + if prev_prompts and i < len(prev_prompts): + chats[i].append({"role": "user", "content": prev_prompts[i]}) + # Save the model's full raw reply (not the post-projection/action-only string) + chats[i].append({"role": "assistant", "content": raw_actions[i]}) + + # Dump trajectory row (only for envs that acted this step, including final step) + if args.dump_path and (i in idx_map): + try: + row = { + "batch_idx": batch_idx, + "test_idx": test_idx, + "step": step_idx, + "env_id": global_env_counter + i, # Global env ID across all batches + "prompt": prev_prompts[i], + # Save the full raw model output for this step + "action": raw_actions[i], + # Also save the executed (post-projection) action for debugging + "action_exec": actions[i], + "reward": float(rewards[i]) if i < len(rewards) else None, + "done": bool(dones[i]) if i < len(dones) else None, + "won": bool(infos[i].get("won", False)), + "pid": pids[i] if i < len(pids) else "unknown", + "is_action_valid": bool(infos[i].get("is_action_valid", False)), + } + dump_fp.write(json.dumps(row, ensure_ascii=False) + "\n") + except Exception: + pass + + if dones[i]: + env_dones[i] = True + won = bool(infos[i].get("won", False)) + overall_success_this_round[i] = won + + # Track success for this task + pid = pids[i] if i < len(pids) else f"task_{i}" + task_total_cnt[pid] = 1 + if won: + task_success_cnt[pid] = 1 + + # If this env just finished, dump chat history if requested + if chat_base_dir and not saved_flags[i]: + try: + pid = pids[i] if i < len(pids) else f"task_{i}" + task_dir = os.path.join(chat_base_dir, _sanitize(pid)) + os.makedirs(task_dir, exist_ok=True) + unique_id = f"b{batch_idx:03d}_t{test_idx:02d}_e{i:02d}" + base = f"chat_{_sanitize(pid)}-{unique_id}" + out_path = os.path.join(task_dir, base + ".json") + meta = { + "batch_idx": batch_idx, + "env_id": global_env_counter + i, + "test_idx": test_idx, + "model": args.model, + "pid": pid, + "steps": step_idx + 1, + "won": bool(infos[i].get("won", False)), + "timestamp": run_ts, + } + with open(out_path, "w", encoding="utf-8") as f: + json.dump({"messages": chats[i], "metadata": meta}, f, ensure_ascii=False, indent=2) + saved_flags[i] = True + except Exception: + pass + + if all(env_dones): + logging.info("All environments finished early!") + break + + # After loop: dump any unfinished envs (failures/timeouts) + if chat_base_dir: + for i in range(current_batch_size): + if not saved_flags[i]: + try: + pid = pids[i] if i < len(pids) else f"task_{i}" + task_dir = os.path.join(chat_base_dir, _sanitize(pid)) + os.makedirs(task_dir, exist_ok=True) + unique_id = f"b{batch_idx:03d}_t{test_idx:02d}_e{i:02d}" + base = f"chat_{_sanitize(pid)}-{unique_id}" + out_path = os.path.join(task_dir, base + ".json") + steps_taken = max(0, len(chats[i]) // 2) + meta = { + "batch_idx": batch_idx, + "env_id": global_env_counter + i, + "test_idx": test_idx, + "model": args.model, + "pid": pid, + "steps": steps_taken, + "won": bool(last_infos[i].get("won", False)) if isinstance(last_infos, list) and i < len(last_infos) else False, + "timestamp": run_ts, + } + with open(out_path, "w", encoding="utf-8") as f: + json.dump({"messages": chats[i], "metadata": meta}, f, ensure_ascii=False, indent=2) + saved_flags[i] = True + except Exception: + pass + + # -------- Single round results -------- + round_success_rate = overall_success_this_round.mean() + batch_overall_success_rates.append(round_success_rate) + + logging.info(f"Batch {batch_idx + 1} Test {test_idx} overall success: {round_success_rate:.4f}") + + # Aggregate per-task results for this round + for pid, total in task_total_cnt.items(): + if total > 0: + rate = task_success_cnt.get(pid, 0) / total + batch_task_success_history[pid].append(rate) + + # Log individual task results (top few) + success_pids = [pid for pid, cnt in task_success_cnt.items() if cnt > 0] + if success_pids: + logging.info(f" Successful tasks: {', '.join(success_pids[:5])}{'...' if len(success_pids) > 5 else ''}") + + logging.info( + f"Batch {batch_idx + 1} Test {test_idx} time elapsed: {time.time() - start_time:.2f}s\n" + ) + + finally: + # Accumulate batch results to global results + all_overall_success_rates.extend(batch_overall_success_rates) + for task, rates in batch_task_success_history.items(): + all_task_success_history[task].extend(rates) + + # Update global env counter + global_env_counter += current_batch_size + + # Clean up resources for this batch + try: + env_manager.envs.close() + logging.info(f"Released resources for Batch {batch_idx + 1}") + except Exception as e: + logging.warning(f"Failed to release resources for Batch {batch_idx + 1}: {e}") + + logging.info(f"========== Finished Batch {batch_idx + 1}/{num_batches}, processed {global_env_counter}/{total_envs} envs ==========\n") + + # ======================= Final Summary ======================= + logging.info("=============== Final Summary ===============") + logging.info( + f"Total batches: {num_batches} | Batch size: {batch_size} | Total envs processed: {global_env_counter}" + ) + logging.info( + f"Overall success avg ± std: " + f"{np.mean(all_overall_success_rates):.4f} ± {np.std(all_overall_success_rates):.4f}" + ) + + # Summary of task-level success + successful_tasks = sum(1 for rates in all_task_success_history.values() if any(r > 0 for r in rates)) + logging.info(f"Successfully completed {successful_tasks} out of {len(all_task_success_history)} unique tasks") + + if dump_fp is not None: + dump_fp.flush() + dump_fp.close() diff --git a/scripts/rollout/openmanus_rollout.py b/scripts/rollout/openmanus_rollout.py new file mode 100644 index 00000000..f696bfe6 --- /dev/null +++ b/scripts/rollout/openmanus_rollout.py @@ -0,0 +1,726 @@ +#!/usr/bin/env python3 +""" +Unified rollout script for AlfWorld, GAIA, and WebShop environments. +Provides a single interface for running rollouts across all three environments. +""" + +import os +import time +import json +import logging +import argparse +from types import SimpleNamespace +from datetime import datetime +from typing import List, Dict, Any, Optional +from concurrent.futures import ThreadPoolExecutor, as_completed +from collections import defaultdict +import numpy as np +import random +import hashlib +import sys +from openmanus_rl.environments.env_manager import * +from openai import OpenAI +from together import Together + +try: + import dotenv + dotenv.load_dotenv() +except Exception: + pass + + +class UnifiedAgent: + """Unified agent that can work with all environments""" + + def __init__(self, model_name: str = "gpt-4o", temperature: float = 0.4, + base_url: str | None = None, env_type: str = "alfworld"): + self.model_name = model_name + self.temperature = temperature + self.env_type = env_type + + # Determine which client to use based on model name and base_url + # Use Together client only for models that explicitly look like Together models + # (e.g., meta-llama/Llama-2-7b-chat-hf, Qwen/Qwen2.5-7B-Instruct-Turbo) + together_providers = ['meta-llama/', 'Qwen/', 'mistralai/', 'NousResearch/', 'teknium/'] + self.is_together = any(model_name.startswith(provider) for provider in together_providers) and base_url is None + + if self.is_together: + self.client = Together( + api_key=os.environ.get('TOGETHER_API_KEY', ''), + ) + elif base_url: + self.client = OpenAI( + api_key=os.getenv('OPENAI_API_KEY', 'EMPTY'), + base_url=base_url, + ) + else: + self.client = OpenAI( + api_key=os.environ.get('OPENAI_API_KEY'), + ) + + # Set environment-specific system prompts + self.system_prompts = { + "webshop": ( + "You are an expert web shopping agent. Respond strictly as " + "....... The must be a single " + "admissible action exactly from the provided list, or a search[query]." + ), + "gaia": None, # GAIA uses prompt templates in the environment + "alfworld": None, # AlfWorld uses prompt templates in the environment + } + + def get_action_from_llm(self, obs: str) -> str: + """Get action from LLM for a single observation""" + messages = [] + + # Add system prompt if available for this environment + system_prompt = self.system_prompts.get(self.env_type) + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + + messages.append({"role": "user", "content": obs}) + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + temperature=self.temperature, + n=1, + ) + return response.choices[0].message.content.strip() + + def get_actions_batch(self, prompts: List[str], concurrency: int = 4, + retries: int = 3, backoff: float = 0.5) -> List[str]: + """Get actions for multiple observations in parallel""" + actions = [None] * len(prompts) + + def _one(idx_prompt): + idx, prompt = idx_prompt + delay = backoff + for attempt in range(retries): + try: + act = self.get_action_from_llm(prompt) + return idx, act + except Exception as e: + if attempt == retries - 1: + # Return a default action based on environment + default_actions = { + "webshop": "errorsearch[product]", + "gaia": "None", + "alfworld": "None" + } + return idx, default_actions.get(self.env_type, "None") + time.sleep(delay) + delay *= 2 + + with ThreadPoolExecutor(max_workers=max(1, concurrency)) as ex: + futures = [ex.submit(_one, (i, p)) for i, p in enumerate(prompts)] + for fut in as_completed(futures): + i, act = fut.result() + actions[i] = act + + return actions + + +class EnvironmentFactory: + """Factory for creating different environment types""" + + @staticmethod + def build_env(env_type: str, **kwargs) -> Any: + """Build environment based on type""" + + if env_type == "alfworld": + return EnvironmentFactory._build_alfworld(**kwargs) + elif env_type == "gaia": + return EnvironmentFactory._build_gaia(**kwargs) + elif env_type == "webshop": + return EnvironmentFactory._build_webshop(**kwargs) + else: + raise ValueError(f"Unsupported environment type: {env_type}") + + @staticmethod + def _build_alfworld(env_num: int = 1, seed: int = 1, history_length: int = 2, + alf_env_type: str = "alfworld/AlfredTWEnv", + game_files: Optional[List[str]] = None, **kwargs): + """Build AlfWorld environment""" + from openmanus_rl.environments.env_package.alfworld import alfworld_projection + from openmanus_rl.environments.env_package.alfworld import build_alfworld_envs + + alf_config_path = os.path.join( + os.path.dirname(__file__), + '../../openmanus_rl/environments/env_package/alfworld/configs/config_tw.yaml' + ) + + envs = build_alfworld_envs( + alf_config_path, + seed=seed, + env_num=env_num, + group_n=1, + is_train=True, + env_kwargs={}, + game_files=game_files + ) + + cfg = SimpleNamespace( + env=SimpleNamespace( + env_name=alf_env_type, + history_length=history_length + ) + ) + + return AlfWorldEnvironmentManager(envs, alfworld_projection, cfg) + + @staticmethod + def _build_gaia(tasks_data: List[Dict], available_tools: List[str], + env_num: int = 1, seed: int = 1, history_length: int = 2, + max_steps: int = 30, **kwargs): + """Build GAIA/Tool Use environment""" + + from openmanus_rl.environments.env_package.tool_use.projection import tool_use_projection + from openmanus_rl.environments.env_package.tool_use.envs import build_tool_use_envs + from openmanus_rl.environments.env_package.tool_use.manager import ToolUseEnvironmentManager + + envs = build_tool_use_envs( + tasks_data=tasks_data, + available_tools=available_tools, + seed=seed, + env_num=env_num, + group_n=1, + is_train=True + ) + + cfg = SimpleNamespace( + env=SimpleNamespace( + env_name="tool_use", + history_length=history_length, + max_steps=max_steps + ) + ) + + return ToolUseEnvironmentManager(envs, tool_use_projection, cfg) + + @staticmethod + def _build_webshop(env_num: int = 1, seed: int = 1, history_length: int = 2, + use_train_set: bool = False, **kwargs): + """Build WebShop environment""" + from openmanus_rl.environments.env_package.webshop import build_webshop_envs, webshop_projection + + env_kwargs = {"observation_mode": "text"} + + envs = build_webshop_envs( + seed=seed, + env_num=env_num, + group_n=1, + is_train=use_train_set, + env_kwargs=env_kwargs, + ) + + cfg = SimpleNamespace( + env=SimpleNamespace( + env_name="webshop/WebAgentTextEnv", + history_length=history_length + ) + ) + + return WebshopEnvironmentManager(envs, webshop_projection, cfg) + + +def load_gaia_tasks(data_path: str, max_tasks: Optional[int] = None) -> List[Dict]: + """Load GAIA tasks from JSON file""" + with open(data_path, 'r', encoding='utf-8') as f: + tasks = json.load(f) + + if max_tasks: + tasks = tasks[:max_tasks] + + return tasks + + +def prepare_alfworld_game_files(env_type: str, total_envs: int, seed: int) -> Optional[List[str]]: + """Prepare unique game files for AlfWorld if requested""" + if env_type != "alfworld": + return None + + from openmanus_rl.environments.env_package.alfworld.envs import load_config_file + from openmanus_rl.environments.env_package.alfworld.alfworld.agents.environment import get_environment + + alf_config_path = os.path.join( + os.path.dirname(__file__), + '../../openmanus_rl/environments/env_package/alfworld/configs/config_tw.yaml' + ) + + try: + cfg = load_config_file(alf_config_path) + env_type = cfg['env']['type'] + BaseEnvCls = get_environment(env_type) + tmp_env = BaseEnvCls(cfg, train_eval='train') + tmp_env.collect_game_files() + all_game_files = list(getattr(tmp_env, 'game_files', [])) + + if len(all_game_files) < total_envs: + logging.error(f"Not enough game files: need {total_envs}, have {len(all_game_files)}") + return None + + rng = random.Random(seed) + rng.shuffle(all_game_files) + return all_game_files[:total_envs] + + except Exception as e: + logging.error(f"Failed to collect game files: {e}") + return None + + +def main(): + parser = argparse.ArgumentParser(description="Unified rollout script for multiple environments") + + # Environment selection + parser.add_argument("--env", choices=["alfworld", "gaia", "webshop"], required=True, + help="Environment to run") + + # Common parameters + parser.add_argument("--batch_size", type=int, default=10, + help="Number of envs to process per batch") + parser.add_argument("--total_envs", type=int, default=100, + help="Total number of environments to rollout") + parser.add_argument("--test_times", type=int, default=1, + help="Number of test runs per batch") + parser.add_argument("--max_steps", type=int, default=None, + help="Maximum steps per episode (default: 50 for alfworld, 30 for gaia/webshop)") + parser.add_argument("--seed", type=int, default=1) + parser.add_argument("--history_length", type=int, default=2) + + # Model parameters + parser.add_argument("--model", default="gpt-4o-mini", + help="Model name (OpenAI: gpt-4o, gpt-4o-mini; Together: Qwen/Qwen2.5-7B-Instruct-Turbo, etc.)") + parser.add_argument("--temperature", type=float, default=0.4) + parser.add_argument("--base_url", default=None, + help="OpenAI-compatible base URL (e.g., vLLM http://127.0.0.1:8000/v1)") + + # Execution parameters + parser.add_argument("--concurrency", type=int, default=4, + help="Max concurrent LLM requests per step") + parser.add_argument("--retries", type=int, default=3, + help="Retries per request on failure") + + # Output parameters + parser.add_argument("--dump_path", default=None, + help="If set, write JSONL trajectory to this file") + parser.add_argument("--chat_root", default=None, + help="If set, save per-episode chat histories under this root") + + # Environment-specific parameters + parser.add_argument("--alf_env_type", default="alfworld/AlfredTWEnv", + help="AlfWorld environment type") + parser.add_argument("--gaia_data_path", default="data/gaia/val.json", + help="Path to GAIA dataset") + parser.add_argument("--gaia_tools", nargs='+', + default=['google_search', 'wikipedia_knowledge_searcher', 'python_code_generator'], + help="List of available tools for GAIA") + parser.add_argument("--webshop_train", action="store_true", + help="Use WebShop training set instead of test set") + + # Other options + parser.add_argument("--unique_envs", action="store_true", + help="Ensure unique tasks/games across all environments") + parser.add_argument("--dry_run", action="store_true", + help="Only print batch allocation without running") + parser.add_argument("--debug", action="store_true", + help="Enable debug logging") + + # Summary-related options + parser.add_argument("--use_summary", action="store_true", + help="Enable memory summarization instead of sliding window") + parser.add_argument("--summary_api_key", default=None, + help="API key for summary LLM (defaults to environment variables)") + parser.add_argument("--summary_endpoint", default=None, + help="API endpoint for summary LLM (defaults to environment variables)") + + args = parser.parse_args() + + # Set default max_steps based on environment + if args.max_steps is None: + args.max_steps = { + "alfworld": 50, + "gaia": 30, + "webshop": 30 + }[args.env] + + # Setup logging + os.makedirs(f"logs/{args.env}", exist_ok=True) + log_fp = os.path.join( + f"logs/{args.env}", + f"unified_run_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + ) + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO, + format="%(asctime)s - %(message)s", + handlers=[logging.FileHandler(log_fp, encoding="utf-8"), logging.StreamHandler()], + ) + + logging.info(f"Starting unified rollout for {args.env}") + logging.info(f"Model: {args.model}, Temperature: {args.temperature}") + logging.info(f"Total envs: {args.total_envs}, Batch size: {args.batch_size}, Max steps: {args.max_steps}") + + # Calculate number of batches + num_batches = (args.total_envs + args.batch_size - 1) // args.batch_size + logging.info(f"Running {args.total_envs} envs in {num_batches} batches") + + # Prepare output files + dump_fp = None + if args.dump_path: + os.makedirs(os.path.dirname(args.dump_path) or ".", exist_ok=True) + dump_fp = open(args.dump_path, "a", encoding="utf-8") + logging.info(f"Dumping trajectories to: {args.dump_path}") + + # Prepare chat history directories + run_ts = datetime.now().strftime('%Y%m%d_%H%M%S') + chat_base_dir = None + if args.chat_root: + chat_ts_root = os.path.join(args.chat_root, 'trajectories', run_ts) + chat_base_dir = os.path.join(chat_ts_root, args.env, args.model.replace('/', '_')) + os.makedirs(chat_base_dir, exist_ok=True) + logging.info(f"Saving chats to: {chat_base_dir}") + + def _sanitize(s: str) -> str: + """Sanitize string for filename""" + return ''.join(c if c.isalnum() or c in ('-', '_', '.') else '-' for c in str(s))[:200] + + # Prepare environment-specific data + gaia_tasks = None + alfworld_game_files = None + + if args.env == "gaia": + logging.info(f"Loading GAIA tasks from {args.gaia_data_path}") + gaia_tasks = load_gaia_tasks(args.gaia_data_path) + logging.info(f"Loaded {len(gaia_tasks)} tasks") + + if len(gaia_tasks) < args.total_envs: + logging.warning(f"Only {len(gaia_tasks)} tasks available, adjusting total_envs") + args.total_envs = len(gaia_tasks) + num_batches = (args.total_envs + args.batch_size - 1) // args.batch_size + + # Shuffle tasks for random sampling + rng = random.Random(args.seed) + rng.shuffle(gaia_tasks) + + elif args.env == "alfworld" and args.unique_envs: + alfworld_game_files = prepare_alfworld_game_files(args.env, args.total_envs, args.seed) + if alfworld_game_files: + logging.info(f"Prepared {len(alfworld_game_files)} unique game files") + + # Dry run mode + if args.dry_run: + logging.info(f"[Dry-Run] Environment: {args.env}") + logging.info(f"[Dry-Run] Total envs: {args.total_envs}, Batches: {num_batches}") + + for b in range(num_batches): + start = b * args.batch_size + end = min(start + args.batch_size, args.total_envs) + batch_size = end - start + + if args.env == "gaia" and gaia_tasks: + batch_tasks = gaia_tasks[start:end] + pids = [t.get('pid', f'task_{i}') for i, t in enumerate(batch_tasks)] + logging.info(f"[Dry-Run] Batch {b+1:02d}: {batch_size} tasks; PIDs: {', '.join(pids[:3])}...") + elif args.env == "alfworld" and alfworld_game_files: + batch_files = alfworld_game_files[start:end] + examples = [os.path.basename(f) for f in batch_files[:3]] + logging.info(f"[Dry-Run] Batch {b+1:02d}: {batch_size} files; Examples: {', '.join(examples)}") + else: + logging.info(f"[Dry-Run] Batch {b+1:02d}: {batch_size} environments") + + sys.exit(0) + + # Initialize agent (defer until after potential dry-run exit to avoid requiring API keys) + agent = UnifiedAgent( + model_name=args.model, + temperature=args.temperature, + base_url=args.base_url, + env_type=args.env + ) + + # Statistics tracking + all_overall_success_rates = [] + all_task_success_history = defaultdict(list) + global_env_counter = 0 + + # Main rollout loop + try: + for batch_idx in range(num_batches): + # Calculate actual batch size + current_batch_size = min(args.batch_size, args.total_envs - batch_idx * args.batch_size) + logging.info(f"\n========== Starting Batch {batch_idx + 1}/{num_batches} with {current_batch_size} envs ==========") + + # Prepare environment-specific kwargs + env_kwargs = { + "env_num": current_batch_size, + "seed": args.seed + batch_idx, + "history_length": args.history_length, + # Summary configuration + "use_summary": args.use_summary, + "summary_api_key": args.summary_api_key or os.getenv("OAI_KEY") or os.getenv("OPENAI_API_KEY"), + "summary_endpoint": args.summary_endpoint or os.getenv("OAI_ENDPOINT") or os.getenv("OPENAI_ENDPOINT"), + } + + if args.env == "gaia": + start = batch_idx * args.batch_size + end = start + current_batch_size + env_kwargs["tasks_data"] = gaia_tasks[start:end] + env_kwargs["available_tools"] = args.gaia_tools + env_kwargs["max_steps"] = args.max_steps + + elif args.env == "alfworld": + env_kwargs["alf_env_type"] = args.alf_env_type + if alfworld_game_files: + start = batch_idx * args.batch_size + end = start + current_batch_size + env_kwargs["game_files"] = alfworld_game_files[start:end] + + elif args.env == "webshop": + env_kwargs["use_train_set"] = args.webshop_train + + # Create environment + env_manager = EnvironmentFactory.build_env(args.env, **env_kwargs) + + # Batch-level statistics + batch_overall_success_rates = [] + batch_task_success_history = defaultdict(list) + + try: + # Test loop for this batch + for test_idx in range(args.test_times): + logging.info(f"\n========== Start Batch {batch_idx + 1} Test {test_idx} ==========") + start_time = time.time() + + obs, infos = env_manager.reset() + env_dones = [False] * current_batch_size + + # Per-env chat buffers + chats = [[] for _ in range(current_batch_size)] + saved_flags = [False] * current_batch_size + last_infos = infos + + # Statistics for single round + overall_success_this_round = np.zeros(current_batch_size, dtype=bool) + task_success_cnt = defaultdict(int) + task_total_cnt = defaultdict(int) + + for step_idx in range(args.max_steps): + logging.info(f"Batch {batch_idx + 1} Step {step_idx}; Dones ({np.array(env_dones).sum()}/{current_batch_size}); SR {overall_success_this_round.mean():.3f}") + + # Assemble actions + prompts = [] + idx_map = [] + for i in range(current_batch_size): + if not env_dones[i]: + prompts.append(obs["text"][i]) + idx_map.append(i) + + if not prompts: + break + + batch_actions = agent.get_actions_batch( + prompts, + concurrency=args.concurrency, + retries=args.retries + ) + + actions = ["None"] * current_batch_size + for k, i in enumerate(idx_map): + actions[i] = batch_actions[k] + + # Environment stepping + prev_prompts = obs["text"] + raw_actions = actions.copy() + obs, rewards, dones, infos = env_manager.step(actions.copy()) + last_infos = infos + + # Process results + for i in range(current_batch_size): + if env_dones[i]: + continue + + # Append chat history + if prev_prompts and i < len(prev_prompts): + chats[i].append({"role": "user", "content": prev_prompts[i]}) + chats[i].append({"role": "assistant", "content": raw_actions[i]}) + + # Dump trajectory + if args.dump_path and (i in idx_map): + try: + row = { + "batch_idx": batch_idx, + "test_idx": test_idx, + "step": step_idx, + "env_id": global_env_counter + i, + "prompt": prev_prompts[i], + "action": raw_actions[i], + "reward": float(rewards[i]) if i < len(rewards) else None, + "done": bool(dones[i]) if i < len(dones) else None, + "won": bool(infos[i].get("won", False)), + "is_action_valid": bool(infos[i].get("is_action_valid", False)), + } + + # Add environment-specific fields + if args.env == "gaia": + row["pid"] = infos[i].get("pid", "unknown") + elif args.env == "alfworld": + row["gamefile"] = infos[i].get("extra.gamefile", "") + elif args.env == "webshop": + row["task_score"] = float(infos[i].get("task_score", 0)) + + dump_fp.write(json.dumps(row, ensure_ascii=False) + "\n") + except Exception as e: + logging.debug(f"Dump error: {e}") + + # Check if done + if dones[i]: + env_dones[i] = True + won = bool(infos[i].get("won", False)) + overall_success_this_round[i] = won + + # Track task success + if args.env == "gaia": + task_id = infos[i].get("pid", f"task_{i}") + elif args.env == "alfworld": + gamefile = infos[i].get("extra.gamefile", "") + # Extract task type from gamefile + task_types = ["pick_and_place", "pick_two_obj_and_place", + "look_at_obj_in_light", "pick_heat_then_place_in_recep", + "pick_cool_then_place_in_recep", "pick_clean_then_place_in_recep"] + task_id = "other" + for t in task_types: + if t in gamefile: + task_id = t + break + else: # webshop + task_id = f"task_{i}" + + task_total_cnt[task_id] = 1 + if won: + task_success_cnt[task_id] = 1 + + # Save chat history + if chat_base_dir and not saved_flags[i]: + try: + task_hash = hashlib.sha1(str(task_id).encode()).hexdigest()[:8] + unique_id = f"b{batch_idx:03d}_t{test_idx:02d}_e{i:02d}-{task_hash}" + out_path = os.path.join(chat_base_dir, f"chat_{unique_id}.json") + + meta = { + "batch_idx": batch_idx, + "env_id": global_env_counter + i, + "test_idx": test_idx, + "model": args.model, + "steps": step_idx + 1, + "won": won, + "timestamp": run_ts, + "environment": args.env, + } + + with open(out_path, "w", encoding="utf-8") as f: + json.dump({"messages": chats[i], "metadata": meta}, f, ensure_ascii=False, indent=2) + saved_flags[i] = True + except Exception as e: + logging.debug(f"Failed to save chat: {e}") + + if all(env_dones): + logging.info("All environments finished early!") + break + + # Save any unfinished chats + if chat_base_dir: + for i in range(current_batch_size): + if not saved_flags[i]: + try: + task_hash = hashlib.sha1(f"unfinished_{i}".encode()).hexdigest()[:8] + unique_id = f"b{batch_idx:03d}_t{test_idx:02d}_e{i:02d}-{task_hash}" + out_path = os.path.join(chat_base_dir, f"chat_{unique_id}.json") + + meta = { + "batch_idx": batch_idx, + "env_id": global_env_counter + i, + "test_idx": test_idx, + "model": args.model, + "steps": len(chats[i]) // 2, + "won": False, + "timestamp": run_ts, + "environment": args.env, + } + + with open(out_path, "w", encoding="utf-8") as f: + json.dump({"messages": chats[i], "metadata": meta}, f, ensure_ascii=False, indent=2) + saved_flags[i] = True + except Exception as e: + logging.debug(f"Failed to save unfinished chat: {e}") + + # Round statistics + round_success_rate = overall_success_this_round.mean() + batch_overall_success_rates.append(round_success_rate) + + logging.info(f"Batch {batch_idx + 1} Test {test_idx} overall success: {round_success_rate:.4f}") + + # Calculate and store per-task success rates for this test + for task, total in task_total_cnt.items(): + if total > 0: + rate = task_success_cnt.get(task, 0) / total + batch_task_success_history[task].append(rate) + + # Log task-specific results for alfworld + if args.env == "alfworld": + logging.info(f" {task:<35s}: {rate:.4f} ({task_success_cnt.get(task, 0)}/{task_total_cnt[task]})") + + logging.info(f"Batch {batch_idx + 1} Test {test_idx} time elapsed: {time.time() - start_time:.2f}s\n") + + finally: + # Accumulate batch results + all_overall_success_rates.extend(batch_overall_success_rates) + for task, rates in batch_task_success_history.items(): + all_task_success_history[task].extend(rates) + + # Update global counter + global_env_counter += current_batch_size + + # Clean up resources + try: + env_manager.envs.close() + logging.info(f"Released resources for Batch {batch_idx + 1}") + except Exception as e: + logging.warning(f"Failed to release resources: {e}") + + logging.info(f"========== Finished Batch {batch_idx + 1}/{num_batches}, processed {global_env_counter}/{args.total_envs} envs ==========\n") + + finally: + if dump_fp is not None: + dump_fp.flush() + dump_fp.close() + logging.info(f"Trajectories saved to: {args.dump_path}") + + # Final summary + logging.info("=============== Final Summary ===============") + logging.info(f"Environment: {args.env}") + logging.info(f"Total batches: {num_batches} | Batch size: {args.batch_size} | Total envs processed: {global_env_counter}") + + if all_overall_success_rates: + logging.info( + f"Overall success avg ± std: " + f"{np.mean(all_overall_success_rates):.4f} ± {np.std(all_overall_success_rates):.4f}" + ) + + # Environment-specific summaries + if args.env == "alfworld": + task_types = ["pick_and_place", "pick_two_obj_and_place", "look_at_obj_in_light", + "pick_heat_then_place_in_recep", "pick_cool_then_place_in_recep", + "pick_clean_then_place_in_recep", "other"] + for task in task_types: + if task in all_task_success_history and all_task_success_history[task]: + rates = [r for r in all_task_success_history[task] if r is not None] + if rates: + logging.info(f"{task:<35s}: {np.mean(rates):.4f} ± {np.std(rates):.4f}") + + elif args.env == "gaia": + successful_tasks = sum(1 for rates in all_task_success_history.values() if any(r > 0 for r in rates)) + logging.info(f"Successfully completed {successful_tasks} out of {len(all_task_success_history)} unique tasks") + + +if __name__ == "__main__": + main() diff --git a/scripts/rollout/run_alfworld_rollout.py b/scripts/rollout/run_alfworld_rollout.py new file mode 100644 index 00000000..a4fc0700 --- /dev/null +++ b/scripts/rollout/run_alfworld_rollout.py @@ -0,0 +1,478 @@ +import os +try: + from dotenv import load_dotenv # type: ignore + load_dotenv() +except Exception: + pass +import time +import json +import logging +import argparse +from types import SimpleNamespace +from datetime import datetime +from typing import List +from concurrent.futures import ThreadPoolExecutor, as_completed +from collections import defaultdict +import numpy as np +import random +import sys +from openmanus_rl.environments.env_manager import * +from openai import OpenAI +from together import Together +from openmanus_rl.environments.env_package.alfworld.envs import load_config_file +from openmanus_rl.environments.env_package.alfworld.alfworld.agents.environment import get_environment + +def build_env(env_name, env_num=1, seed=1, history_length=2, alf_env_type="alfworld/AlfredTWEnv", game_files=None): + group_n = 1 + if env_name == "alfworld": + # Test AlfWorldEnvironmentManager + from openmanus_rl.environments.env_package.alfworld import alfworld_projection + from openmanus_rl.environments.env_package.alfworld import build_alfworld_envs + alf_config_path = os.path.join(os.path.dirname(__file__), '../../openmanus_rl/environments/env_package/alfworld/configs/config_tw.yaml') + # Now with game_files support! + envs = build_alfworld_envs(alf_config_path, seed=seed, env_num=env_num, group_n=group_n, is_train=True, env_kwargs={}, game_files=game_files) + # Minimal config object with required fields + cfg = SimpleNamespace(env=SimpleNamespace(env_name=alf_env_type, history_length=history_length)) + env_manager = AlfWorldEnvironmentManager(envs, alfworld_projection, cfg) + else: + raise ValueError(f"Unsupported environment name: {env_name}") + + return env_manager + +class Agent: + def __init__(self, model_name="gpt-4o", temperature: float = 0.4, base_url: str | None = None): + self.model_name = model_name + self.temperature = temperature + + # Check if model is a Together model (contains "/" and no base_url provided) + self.is_together = "/" in model_name and base_url is None + + if self.is_together: + self.client = Together( + api_key=os.environ.get('TOGETHER_API_KEY', ''), + ) + elif base_url: + self.client = OpenAI( + api_key=os.getenv('OPENAI_API_KEY', 'EMPTY'), + base_url=base_url, + ) + else: + self.client = OpenAI( + api_key=os.environ['OPENAI_API_KEY'], + ) + + def get_action_from_gpt(self, obs): + response = self.client.chat.completions.create( + model=self.model_name, + messages=[ + { + "role": "user", + "content": obs + } + ], + temperature=self.temperature, + n=1, + ) + action = response.choices[0].message.content.strip() + return action + + def get_actions_batch(self, prompts: List[str], concurrency: int = 4, retries: int = 3, backoff: float = 0.5) -> List[str]: + actions = [None] * len(prompts) + + def _one(idx_prompt): + idx, prompt = idx_prompt + delay = backoff + for attempt in range(retries): + try: + act = self.get_action_from_gpt(prompt) + return idx, act + except Exception as e: + if attempt == retries - 1: + return idx, "None" + time.sleep(delay) + delay *= 2 + + with ThreadPoolExecutor(max_workers=max(1, concurrency)) as ex: + futures = [ex.submit(_one, (i, p)) for i, p in enumerate(prompts)] + for fut in as_completed(futures): + i, act = fut.result() + actions[i] = act + + return actions + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--env_name", default="alfworld") + parser.add_argument("--batch_size", type=int, default=10, help="Number of envs to process per batch") + parser.add_argument("--total_envs", type=int, default=1000, help="Total number of environments to rollout") + parser.add_argument("--test_times", type=int, default=1) + parser.add_argument("--max_steps", type=int, default=50) + parser.add_argument("--seed", type=int, default=1) + parser.add_argument("--history_length", type=int, default=2) + parser.add_argument("--model", default="gpt-4o-mini", help="Model name (OpenAI: gpt-4o, gpt-4o-mini; Together: Qwen/Qwen2.5-7B-Instruct-Turbo, etc.)") + parser.add_argument("--temperature", type=float, default=0.4) + parser.add_argument("--concurrency", type=int, default=4, help="Max concurrent OpenAI requests per step") + parser.add_argument("--retries", type=int, default=3, help="Retries per request on failure") + parser.add_argument("--dump_path", default=None, help="If set, write JSONL trajectory to this file") + parser.add_argument("--base_url", default=None, help="OpenAI-compatible base URL (e.g., vLLM http://127.0.0.1:8000/v1)") + parser.add_argument("--chat_root", default=None, help="If set, save per-episode chat histories under this root: trajectories/react///chat_histories") + parser.add_argument("--alf_env_type", default="alfworld/AlfredTWEnv", help="alfworld/AlfredTWEnv or alfworld/AlfredThorEnv") + parser.add_argument("--unique_envs", action="store_true", help="确保每个环境使用唯一的游戏文件（无重复采样）") + parser.add_argument("--dry_run", action="store_true", help="仅打印唯一任务的批次分配，不创建环境、不调用模型") + args = parser.parse_args() + + # -------- logging ---------- + os.makedirs("logs/alfworld", exist_ok=True) + log_fp = os.path.join( + "logs/alfworld", f"run_log_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + ) + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(message)s", + handlers=[logging.FileHandler(log_fp, encoding="utf-8"), logging.StreamHandler()], + ) + + # -------- Parameters ---------- + max_steps = args.max_steps + batch_size = args.batch_size + total_envs = args.total_envs + test_times = args.test_times + env_name = args.env_name + + # Calculate number of batches needed + num_batches = (total_envs + batch_size - 1) // batch_size + logging.info(f"Running {total_envs} envs in {num_batches} batches of {batch_size}") + + # Keywords for 6 subtasks + TASKS = [ + "pick_and_place", + "pick_two_obj_and_place", + "look_at_obj_in_light", + "pick_heat_then_place_in_recep", + "pick_cool_then_place_in_recep", + "pick_clean_then_place_in_recep", + ] + + # -------- Agent setup ---------- + agent = None + if not args.dry_run: + agent = Agent(model_name=args.model, temperature=args.temperature, base_url=args.base_url) + + # Prepare trajectory dump file if requested + dump_fp = None + if args.dump_path: + os.makedirs(os.path.dirname(args.dump_path) or ".", exist_ok=True) + dump_fp = open(args.dump_path, "a", encoding="utf-8") + + # Prepare chat history directories if requested + run_ts = datetime.now().strftime('%Y%m%d_%H%M%S') + chat_ts_root = None + chat_base_dir = None + if args.chat_root: + # /trajectories//// + chat_ts_root = os.path.join(args.chat_root, 'trajectories', run_ts) + chat_base_dir = os.path.join(chat_ts_root, args.env_name, args.model) + os.makedirs(chat_base_dir, exist_ok=True) + + def _sanitize(s: str) -> str: + return ''.join(c if c.isalnum() or c in ('-', '_', '.') else '-' for c in s)[:200] + + # Accumulated statistics across all batches + all_overall_success_rates = [] + all_task_success_history = defaultdict(list) + global_env_counter = 0 + + # Helper: collect all train game files + def collect_all_game_files(alf_config_path, is_train=True, eval_dataset='eval_in_distribution'): + cfg = load_config_file(alf_config_path) + env_type = cfg['env']['type'] + BaseEnvCls = get_environment(env_type) + tmp_env = BaseEnvCls(cfg, train_eval='train' if is_train else eval_dataset) + tmp_env.collect_game_files() + return list(getattr(tmp_env, 'game_files', [])) + + # Pre-assign unique game files when requested + alf_config_path = os.path.join(os.path.dirname(__file__), '../../openmanus_rl/environments/env_package/alfworld/configs/config_tw.yaml') + preassigned_game_files = None + if args.unique_envs: + try: + all_game_files = collect_all_game_files(alf_config_path, is_train=True) + except Exception as e: + logging.error(f"Failed to collect game files for unique_envs: {e}") + sys.exit(1) + rng = random.Random(args.seed) + rng.shuffle(all_game_files) + if len(all_game_files) < total_envs: + logging.error(f"游戏文件不足：需要{total_envs}个，只有{len(all_game_files)}个") + sys.exit(1) + preassigned_game_files = all_game_files[:total_envs] + logging.info(f"Unique envs enabled: using {len(preassigned_game_files)} distinct game files from {len(all_game_files)} available") + + # Dry-run: only print allocation then exit + if args.dry_run: + logging.info(f"[Dry-Run] total_envs={total_envs}, batch_size={batch_size}, num_batches={num_batches}") + for b in range(num_batches): + start = b * batch_size + end = start + min(batch_size, total_envs - start) + batch_slice = preassigned_game_files[start:end] + examples = ", ".join(os.path.basename(p) for p in batch_slice[:3]) + logging.info(f"[Dry-Run] Batch {b+1:02d}: {len(batch_slice)} files; examples: {examples}") + sys.exit(0) + else: + if args.dry_run: + logging.warning("--dry_run 需要配合 --unique_envs 使用；当前未启用 unique_envs，直接退出。") + sys.exit(0) + + # ======================= Main Batch Loop ======================= + for batch_idx in range(num_batches): + # Calculate actual batch size for this batch (last batch might be smaller) + current_batch_size = min(batch_size, total_envs - batch_idx * batch_size) + logging.info(f"\n========== Starting Batch {batch_idx + 1}/{num_batches} with {current_batch_size} envs ==========") + + # Select per-batch game files if unique_envs is on + batch_game_files = None + if preassigned_game_files is not None: + start = batch_idx * batch_size + end = start + current_batch_size + batch_game_files = preassigned_game_files[start:end] + + # Create environment for this batch + env_manager = build_env( + env_name, + env_num=current_batch_size, + seed=args.seed + batch_idx, + history_length=args.history_length, + alf_env_type=args.alf_env_type, + game_files=batch_game_files, + ) + + # Batch-level statistics + batch_overall_success_rates = [] + batch_task_success_history = defaultdict(list) + try: + # ======================= Test Loop for this Batch ======================= + for test_idx in range(test_times): + logging.info(f"\n========== Start Batch {batch_idx + 1} Test {test_idx} ==========") + start_time = time.time() + + obs, infos = env_manager.reset() + env_dones = [False] * current_batch_size + + # per-env chat buffers + chats = [[] for _ in range(current_batch_size)] + # track which envs already dumped to disk + saved_flags = [False] * current_batch_size + # keep last infos for fallback dump (failure/timeout) + last_infos = infos + + # Statistics for single round + overall_success_this_round = np.zeros(current_batch_size, dtype=bool) + task_success_cnt = defaultdict(int) + task_total_cnt = defaultdict(int) + + for step_idx in range(max_steps): + logging.info(f"Batch {batch_idx + 1} Step {step_idx}; Dones ({np.array(env_dones).sum().item()}/{current_batch_size}); SR {overall_success_this_round.mean().item()}") + + # --- Assemble actions --- + prompts = [] + idx_map = [] # map from prompts index back to env index + for i in range(current_batch_size): + if not env_dones[i]: + prompts.append(obs["text"][i]) + idx_map.append(i) + + batch_actions = agent.get_actions_batch(prompts, concurrency=args.concurrency, retries=args.retries) + actions = ["None"] * current_batch_size + for k, i in enumerate(idx_map): + actions[i] = batch_actions[k] + + # --- Environment stepping --- + prev_prompts = obs["text"] # keep for logging & chat history + # Preserve the model's raw outputs for logging/chat before any projection mutates them + raw_actions = actions.copy() + # Pass a copy into the env manager so in-place projection does not alter our raw copy + obs, rewards, dones, infos = env_manager.step(actions.copy()) + last_infos = infos + + # --- Determine endings and successes --- + for i in range(current_batch_size): + if env_dones[i]: + continue + + # Append chat turns for acted envs + if prev_prompts and i < len(prev_prompts): + chats[i].append({"role": "user", "content": prev_prompts[i]}) + # Save the model's full raw reply (not the post-projection/action-only string) + chats[i].append({"role": "assistant", "content": raw_actions[i]}) + + # Dump trajectory row (only for envs that acted this step, including final step) + if args.dump_path and (i in idx_map): + try: + row = { + "batch_idx": batch_idx, + "test_idx": test_idx, + "step": step_idx, + "env_id": global_env_counter + i, # Global env ID across all batches + "prompt": prev_prompts[i], + # Save the full raw model output for this step + "action": raw_actions[i], + # Also save the executed (post-projection) action for debugging + "action_exec": actions[i], + "reward": float(rewards[i]) if i < len(rewards) else None, + "done": bool(dones[i]) if i < len(dones) else None, + "won": bool(infos[i].get("won", False)), + "gamefile": infos[i].get("extra.gamefile"), + "is_action_valid": bool(infos[i].get("is_action_valid", False)), + } + dump_fp.write(json.dumps(row, ensure_ascii=False) + "\n") + except Exception: + pass + + if dones[i]: + env_dones[i] = True + won = bool(infos[i].get("won", False)) + overall_success_this_round[i] = won + + # Parse task type + gamefile = infos[i].get("extra.gamefile", "") + matched = False + for task in TASKS: + if task in gamefile: + task_total_cnt[task] += 1 + if won: + task_success_cnt[task] += 1 + matched = True + break + if not matched: + # Unrecognized tasks are also counted in total + task_total_cnt["other"] += 1 + if won: + task_success_cnt["other"] += 1 + + # If this env just finished, dump chat history if requested + if chat_base_dir and not saved_flags[i]: + try: + task = None + try: + task = env_manager.tasks[i] + except Exception: + task = "unknown" + gamefile = infos[i].get("extra.gamefile", "") + task_dir = os.path.join(chat_base_dir, _sanitize(task)) + os.makedirs(task_dir, exist_ok=True) + unique_id = f"b{batch_idx:03d}_t{test_idx:02d}_e{i:02d}" + base = f"chat_{_sanitize(task)}-{_sanitize(gamefile) or f'env{i}'}-{unique_id}" + out_path = os.path.join(task_dir, base + ".json") + meta = { + "batch_idx": batch_idx, + "env_id": global_env_counter + i, + "test_idx": test_idx, + "model": args.model, + "task": task, + "gamefile": gamefile, + "steps": step_idx + 1, + "won": bool(infos[i].get("won", False)), + "timestamp": run_ts, + } + with open(out_path, "w", encoding="utf-8") as f: + json.dump({"messages": chats[i], "metadata": meta}, f, ensure_ascii=False, indent=2) + saved_flags[i] = True + except Exception: + pass + + if all(env_dones): + logging.info("All environments finished early!") + break + + # After loop: dump any unfinished envs (failures/timeouts) + if chat_base_dir: + for i in range(current_batch_size): + if not saved_flags[i]: + try: + task = None + try: + task = env_manager.tasks[i] + except Exception: + task = "unknown" + gamefile = last_infos[i].get("extra.gamefile", "") if isinstance(last_infos, list) and i < len(last_infos) else "" + task_dir = os.path.join(chat_base_dir, _sanitize(task)) + os.makedirs(task_dir, exist_ok=True) + unique_id = f"b{batch_idx:03d}_t{test_idx:02d}_e{i:02d}" + base = f"chat_{_sanitize(task)}-{_sanitize(gamefile) or f'env{i}'}-{unique_id}" + out_path = os.path.join(task_dir, base + ".json") + steps_taken = max(0, len(chats[i]) // 2) + meta = { + "batch_idx": batch_idx, + "env_id": global_env_counter + i, + "test_idx": test_idx, + "model": args.model, + "task": task, + "gamefile": gamefile, + "steps": steps_taken, + "won": bool(last_infos[i].get("won", False)) if isinstance(last_infos, list) and i < len(last_infos) else False, + "timestamp": run_ts, + } + with open(out_path, "w", encoding="utf-8") as f: + json.dump({"messages": chats[i], "metadata": meta}, f, ensure_ascii=False, indent=2) + saved_flags[i] = True + except Exception: + pass + + # -------- Single round results -------- + round_success_rate = overall_success_this_round.mean() + batch_overall_success_rates.append(round_success_rate) + + logging.info(f"Batch {batch_idx + 1} Test {test_idx} overall success: {round_success_rate:.4f}") + + for task in TASKS + ["other"]: + if task_total_cnt.get(task, 0) > 0: + rate = task_success_cnt[task] / task_total_cnt[task] + batch_task_success_history[task].append(rate) + logging.info( + f" {task:<35s}: {rate:.4f} " + f"({task_success_cnt[task]}/{task_total_cnt[task]})" + ) + + logging.info( + f"Batch {batch_idx + 1} Test {test_idx} time elapsed: {time.time() - start_time:.2f}s\n" + ) + + finally: + # Accumulate batch results to global results + all_overall_success_rates.extend(batch_overall_success_rates) + for task, rates in batch_task_success_history.items(): + all_task_success_history[task].extend(rates) + + # Update global env counter + global_env_counter += current_batch_size + + # Clean up Ray actors for this batch to free resources + try: + env_manager.envs.close() + logging.info(f"Released resources for Batch {batch_idx + 1}") + except Exception as e: + logging.warning(f"Failed to release resources for Batch {batch_idx + 1}: {e}") + + logging.info(f"========== Finished Batch {batch_idx + 1}/{num_batches}, processed {global_env_counter}/{total_envs} envs ==========\n") + + # ======================= Final Summary ======================= + logging.info("=============== Final Summary ===============") + logging.info( + f"Total batches: {num_batches} | Batch size: {batch_size} | Total envs processed: {global_env_counter}" + ) + logging.info( + f"Overall success avg ± std: " + f"{np.mean(all_overall_success_rates):.4f} ± {np.std(all_overall_success_rates):.4f}" + ) + + for task in TASKS + ["other"]: + if all_task_success_history.get(task): + logging.info( + f"{task:<35s}: " + f"{np.mean(all_task_success_history[task]):.4f} ± " + f"{np.std(all_task_success_history[task]):.4f}" + ) + + if dump_fp is not None: + dump_fp.flush() + dump_fp.close() diff --git a/scripts/rollout/webshop.py b/scripts/rollout/webshop.py new file mode 100644 index 00000000..0ddfd3ff --- /dev/null +++ b/scripts/rollout/webshop.py @@ -0,0 +1,352 @@ +import os +import time +import json +import logging +import argparse +from types import SimpleNamespace +from datetime import datetime +from typing import List +from concurrent.futures import ThreadPoolExecutor, as_completed +import numpy as np +import hashlib +import sys + +from openmanus_rl.environments.env_manager import WebshopEnvironmentManager +from openmanus_rl.environments.env_package.webshop import build_webshop_envs, webshop_projection +from openai import OpenAI +try: + import dotenv + dotenv.load_dotenv() +except Exception: + pass + +def build_env(env_name: str, env_num: int = 1, seed: int = 1, history_length: int = 2, use_train_set: bool = False) -> WebshopEnvironmentManager: + if env_name != "webshop": + raise ValueError(f"Unsupported environment name: {env_name}") + + env_kwargs = {"observation_mode": "text"} + + envs = build_webshop_envs( + seed=seed, + env_num=env_num, + group_n=1, + is_train=use_train_set, + env_kwargs=env_kwargs, + ) + + cfg = SimpleNamespace(env=SimpleNamespace(env_name="webshop/WebAgentTextEnv", history_length=history_length)) + return WebshopEnvironmentManager(envs, webshop_projection, cfg) + + +class Agent: + def __init__(self, model_name: str = "gpt-4o", temperature: float = 0.4, base_url: str | None = None): + self.model_name = model_name + self.temperature = temperature + # vLLM/OpenAI-compatible: when base_url provided, point to e.g. http://127.0.0.1:8000/v1 + if base_url: + self.client = OpenAI(api_key=os.getenv("OPENAI_API_KEY", "EMPTY"), base_url=base_url) + else: + self.client = OpenAI(api_key=os.environ["OPENAI_API_KEY"]) # will raise if missing + + self.system_prompt = ( + "You are an expert web shopping agent. Respond strictly as \n" + "....... The must be a single \n" + "admissible action exactly from the provided list, or a search[query]." + ) + + def get_action(self, obs_text: str) -> str: + resp = self.client.chat.completions.create( + model=self.model_name, + messages=[ + {"role": "system", "content": self.system_prompt}, + {"role": "user", "content": obs_text}, + ], + temperature=self.temperature, + n=1, + ) + return resp.choices[0].message.content.strip() + + def get_actions_batch(self, prompts: List[str], concurrency: int = 4, retries: int = 3, backoff: float = 0.5) -> List[str]: + actions = [None] * len(prompts) + + def _one(idx_prompt): + idx, prompt = idx_prompt + delay = backoff + for attempt in range(retries): + try: + act = self.get_action(prompt) + return idx, act + except Exception: + if attempt == retries - 1: + return idx, "errorsearch[product]" + time.sleep(delay) + delay *= 2 + + with ThreadPoolExecutor(max_workers=max(1, concurrency)) as ex: + futures = [ex.submit(_one, (i, p)) for i, p in enumerate(prompts)] + for fut in as_completed(futures): + i, act = fut.result() + actions[i] = act + return actions + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--env_name", default="webshop") + parser.add_argument("--batch_size", type=int, default=4) + parser.add_argument("--total_envs", type=int, default=8) + parser.add_argument("--test_times", type=int, default=1) + parser.add_argument("--max_steps", type=int, default=30) + parser.add_argument("--seed", type=int, default=1) + parser.add_argument("--history_length", type=int, default=2) + parser.add_argument("--model", default="gpt-4o") + parser.add_argument("--temperature", type=float, default=0.4) + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--retries", type=int, default=3) + parser.add_argument("--dump_path", default=None, help="Write JSONL trajectory to this file if set") + parser.add_argument("--base_url", default=None, help="OpenAI-compatible base URL (e.g., vLLM http://127.0.0.1:8000/v1)") + parser.add_argument("--chat_root", default=None, help="Optional chat history root dir") + parser.add_argument("--use_train_set", action="store_true", help="Use training set goals instead of test set") + parser.add_argument("--unique_envs", action="store_true", help="Ensure unique goal indices across total_envs (no repeats)") + parser.add_argument("--dry_run", action="store_true", help="Only print planned batches when --unique_envs is set, then exit") + parser.add_argument("--debug", action="store_true") + args = parser.parse_args() + + # logging + os.makedirs("logs/webshop", exist_ok=True) + log_fp = os.path.join("logs/webshop", f"run2_log_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log") + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO, + format="%(asctime)s - %(message)s", + handlers=[logging.FileHandler(log_fp, encoding="utf-8"), logging.StreamHandler()], + ) + + batch_size = args.batch_size + total_envs = args.total_envs + num_batches = (total_envs + batch_size - 1) // batch_size + logging.info(f"Running {total_envs} envs in {num_batches} batches of {batch_size}") + logging.info(f"Model={args.model}, base_url={args.base_url}, temp={args.temperature}") + + agent = Agent(model_name=args.model, temperature=args.temperature, base_url=args.base_url) + + dump_fp = None + if args.dump_path: + os.makedirs(os.path.dirname(args.dump_path) or ".", exist_ok=True) + dump_fp = open(args.dump_path, "a", encoding="utf-8") + logging.info(f"Dumping trajectories to: {args.dump_path}") + + run_ts = datetime.now().strftime('%Y%m%d_%H%M%S') + chat_base_dir = None + if args.chat_root: + chat_ts_root = os.path.join(args.chat_root, 'trajectories', run_ts) + chat_base_dir = os.path.join(chat_ts_root, args.env_name, args.model.replace('/', '_')) + os.makedirs(chat_base_dir, exist_ok=True) + logging.info(f"Saving chats to: {chat_base_dir}") + + def _sanitize(s: str) -> str: + return ''.join(c if c.isalnum() or c in ('-', '_', '.') else '-' for c in s)[:200] + + all_succ = [] + all_reward = [] + global_env_counter = 0 + + # Pre-assign unique goal indices across all envs when requested + preassigned_goal_indices = None + if args.unique_envs: + # Build a temporary env to fetch the available goal index pool (train or test) + try: + tmp_env = build_env(args.env_name, env_num=1, seed=args.seed, history_length=args.history_length, use_train_set=args.use_train_set) + pool = list(tmp_env.envs.goal_idxs) + try: + tmp_env.envs.close() + except Exception: + pass + except Exception as e: + logging.error(f"Failed to probe goal pool: {e}") + raise + + if len(pool) < args.total_envs: + raise ValueError(f"Not enough unique goals in the {'train' if args.use_train_set else 'test'} set: need {args.total_envs}, available {len(pool)}") + + rng = np.random.RandomState(args.seed) + preassigned_goal_indices = rng.choice(pool, size=args.total_envs, replace=False).tolist() + logging.info(f"Unique envs enabled: sampled {len(preassigned_goal_indices)} unique goal indices from pool size {len(pool)}") + + if args.dry_run: + logging.info(f"[Dry-Run] total_envs={args.total_envs}, batch_size={batch_size}, num_batches={num_batches}") + for b in range(num_batches): + start = b * batch_size + end = min(start + min(batch_size, args.total_envs - start), args.total_envs) + batch_slice = preassigned_goal_indices[start:end] + logging.info(f"[Dry-Run] Batch {b+1:02d}: {len(batch_slice)} goals; examples: {batch_slice[:5]}") + sys.exit(0) + + try: + for b in range(num_batches): + cur_n = min(batch_size, total_envs - b * batch_size) + logging.info(f"\n== Batch {b+1}/{num_batches} with {cur_n} envs ==") + env = build_env(args.env_name, env_num=cur_n, seed=args.seed + b, history_length=args.history_length, use_train_set=args.use_train_set) + + # If unique envs requested, force this batch to use the pre-assigned, non-overlapping goal indices + if preassigned_goal_indices is not None: + start = b * batch_size + end = start + cur_n + batch_slice = preassigned_goal_indices[start:end] + env.envs.goal_idxs = list(batch_slice) + logging.info(f"Batch {b+1}: using preassigned goals (len={len(batch_slice)}) from {'train' if args.use_train_set else 'test'} set") + + for t in range(args.test_times): + obs, infos = env.reset() + dones = [False] * cur_n + chats = [[] for _ in range(cur_n)] + saved = [False] * cur_n + last_infos = infos + succ = np.zeros(cur_n, dtype=bool) + rew = np.zeros(cur_n, dtype=float) + + for step in range(args.max_steps): + logging.info(f"Batch {b+1} Step {step}; dones {np.sum(dones)}/{cur_n}") + + prompts = [] + idx_map = [] + for i in range(cur_n): + if not dones[i]: + prompts.append(obs["text"][i]) + idx_map.append(i) + + actions = ["None"] * cur_n + raw = [None] * cur_n + if prompts: + batch_actions = agent.get_actions_batch(prompts, concurrency=args.concurrency, retries=args.retries) + for k, i in enumerate(idx_map): + actions[i] = batch_actions[k] + raw[i] = batch_actions[k] + + prev_prompts = obs["text"] + obs, rewards, dones_vec, infos = env.step(actions) + last_infos = infos + + for i in range(cur_n): + if not dones[i]: + chats[i].append({"role": "user", "content": prev_prompts[i]}) + chats[i].append({"role": "assistant", "content": raw[i]}) + + rew[i] += rewards[i] + if dones_vec[i] and not dones[i]: + dones[i] = True + won = bool(infos[i].get("won", False)) + succ[i] = won + logging.info(f"Env {i} finished @step {step}: won={won}, task_score={infos[i].get('task_score', 0):.3f}, total_reward={rew[i]:.3f}") + + # Save per-episode chat history when an env finishes (short-hash filename) + if chat_base_dir and not saved[i]: + try: + # Task text -> short hash for compact naming + try: + task_text = env.tasks[i] + except Exception: + task_text = "unknown" + task_hash = hashlib.sha1(task_text.encode("utf-8")).hexdigest()[:8] + + unique_id = f"g{global_env_counter + i}-b{b:03d}_t{t:02d}_e{i:02d}-{task_hash}" + out_path = os.path.join(chat_base_dir, f"chat_{unique_id}.json") + + meta = { + "batch_idx": b, + "test_idx": t, + "env_id": global_env_counter + i, + "model": args.model, + "task": task_text, + "steps": max(0, len(chats[i]) // 2), + "won": bool(infos[i].get("won", False)), + "task_score": float(infos[i].get("task_score", 0)), + "timestamp": run_ts, + "task_hash": task_hash, + } + with open(out_path, "w", encoding="utf-8") as f: + json.dump({"messages": chats[i], "metadata": meta}, f, ensure_ascii=False, indent=2) + saved[i] = True + logging.info(f"Saved chat: {out_path}") + except Exception as e: + logging.warning(f"Failed to save chat for env {i}: {e}") + + # dump JSONL per acted env + if args.dump_path and i in idx_map: + try: + exec_act, valid = webshop_projection([raw[i]]) + row = { + "batch_idx": b, + "test_idx": t, + "step": step, + "env_id": global_env_counter + i, + "prompt": prev_prompts[i], + "action": raw[i], + "action_exec": exec_act[0] if exec_act else None, + "is_action_valid": bool(valid[0]) if valid else None, + "reward": float(rewards[i]), + "done": bool(dones_vec[i]), + "won": bool(infos[i].get("won", False)), + "task_score": float(infos[i].get("task_score", 0)), + "available_actions": infos[i].get("available_actions"), + } + dump_fp.write(json.dumps(row, ensure_ascii=False) + "\n") + except Exception as e: + logging.warning(f"Dump error: {e}") + + if np.all(dones): + break + + all_succ.append(succ.mean()) + all_reward.append(rew.mean()) + logging.info(f"Batch {b+1} Test {t}: SR={succ.mean():.4f}, Reward={rew.mean():.4f}") + + # Save unfinished env chats (timeouts/failures) using short-hash filename + if chat_base_dir: + for i in range(cur_n): + if not saved[i]: + try: + try: + task_text = env.tasks[i] + except Exception: + task_text = "unknown" + task_hash = hashlib.sha1(task_text.encode("utf-8")).hexdigest()[:8] + + unique_id = f"g{global_env_counter + i}-b{b:03d}_t{t:02d}_e{i:02d}-{task_hash}" + out_path = os.path.join(chat_base_dir, f"chat_{unique_id}.json") + + meta = { + "batch_idx": b, + "test_idx": t, + "env_id": global_env_counter + i, + "model": args.model, + "task": task_text, + "steps": max(0, len(chats[i]) // 2), + "won": bool(last_infos[i].get("won", False)) if isinstance(last_infos, list) and i < len(last_infos) else False, + "task_score": float(last_infos[i].get("task_score", 0)) if isinstance(last_infos, list) and i < len(last_infos) else 0.0, + "timestamp": run_ts, + "task_hash": task_hash, + } + with open(out_path, "w", encoding="utf-8") as f: + json.dump({"messages": chats[i], "metadata": meta}, f, ensure_ascii=False, indent=2) + saved[i] = True + logging.info(f"Saved unfinished chat: {out_path}") + except Exception as e: + logging.warning(f"Failed to save unfinished chat for env {i}: {e}") + + global_env_counter += cur_n + try: + env.envs.close() + except Exception: + pass + logging.info(f"== Finished Batch {b+1}/{num_batches}, processed {global_env_counter}/{total_envs} envs ==\n") + + finally: + if dump_fp is not None: + dump_fp.flush() + dump_fp.close() + logging.info(f"Trajectories saved to: {args.dump_path}") + + if all_succ: + logging.info(f"Overall SR: {np.mean(all_succ):.4f} ± {np.std(all_succ):.4f}") + if all_reward: + logging.info(f"Overall Reward: {np.mean(all_reward):.4f} ± {np.std(all_reward):.4f}") diff --git a/scripts/serve_model.sh b/scripts/serve_model.sh new file mode 100755 index 00000000..5e039e6b --- /dev/null +++ b/scripts/serve_model.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Script to serve model with vLLM in OpenManus-RL Docker container + +echo "=========================================" +echo "Starting vLLM Model Server for OpenManus-RL" +echo "=========================================" + +# Enter the container and start vLLM +docker exec -it openmanus-rl bash -c ' +# Set up environment variables for ROCm +export TRANSFORMERS_NO_TORCHVISION=1 +export HF_HUB_DISABLE_TORCHVISION_IMPORT=1 +export VLLM_USE_ROCM=1 +export VLLM_PLATFORM=rocm +export LD_LIBRARY_PATH="$(python - <<'PY' +import os, torch +print(os.path.join(os.path.dirname(torch.__file__), 'lib')) +PY +):${LD_LIBRARY_PATH:-}:/opt/rocm/lib:/opt/rocm/lib64" + +# Start vLLM server +echo "Starting vLLM server on port 8000..." +vllm serve /root/models/GiGPO-Qwen2.5-7B-Instruct-ALFWorld \ + --served-model-name qwen2.5-7b-alfworld \ + --dtype bfloat16 \ + --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.55 \ + --max-model-len 16384 \ + --enforce-eager \ + --device cuda \ + --host 0.0.0.0 --port 8000 +' + diff --git a/openmanus_rl/memory/rag_memory.py b/test/test_setup.py similarity index 100% rename from openmanus_rl/memory/rag_memory.py rename to test/test_setup.py