diff --git a/submissions/scribe_actioneer_opus47.json b/submissions/scribe_actioneer_opus47.json new file mode 100644 index 000000000..86f9154d4 --- /dev/null +++ b/submissions/scribe_actioneer_opus47.json @@ -0,0 +1,1622 @@ +[ + { + "dataset": "deps_dev_v1", + "query": "1", + "run": "0", + "answer": "[('@dylanvann/svelte', '3.25.4', 73499), ('@dumc11/tailwindcss', '0.4.0', 73464), ('@dreampie/semantic-ui', '2.2.11', 51069), ('@dongls/pdfjs-dist', '3.2.72', 44231), ('@dman777/shadow-dom-quill-temp', '1.0.0', 42407)]" + }, + { + "dataset": "deps_dev_v1", + "query": "2", + "run": "0", + "answer": "['mui-org/material-ui', 'moment/moment', 'semantic-org/semantic-ui', 'react-native-elements/react-native-elements', 'sveltejs/svelte']" + }, + { + "dataset": "github_repos", + "query": "1", + "run": "0", + "answer": "0.0005" + }, + { + "dataset": "github_repos", + "query": "2", + "run": "0", + "answer": "uacaps/PageMenu" + }, + { + "dataset": "github_repos", + "query": "3", + "run": "0", + "answer": "1077" + }, + { + "dataset": "github_repos", + "query": "4", + "run": "0", + "answer": "['apple/swift', 'twbs/bootstrap', 'Microsoft/vscode', 'facebook/react', 'tensorflow/tensorflow']" + }, + { + "dataset": "music_brainz_20k", + "query": "1", + "run": "0", + "answer": "1059.46" + }, + { + "dataset": "music_brainz_20k", + "query": "2", + "run": "0", + "answer": "Amazon Music" + }, + { + "dataset": "music_brainz_20k", + "query": "3", + "run": "0", + "answer": "Zo gaat het leven aan je voor" + }, + { + "dataset": "stockindex", + "query": "1", + "run": "0", + "answer": "399001.SZ" + }, + { + "dataset": "stockindex", + "query": "2", + "run": "0", + "answer": "['IXIC']" + }, + { + "dataset": "stockindex", + "query": "3", + "run": "0", + "answer": "[('IXIC', 'United States'), ('NSEI', 'India'), ('399001.SZ', 'China'), ('GDAXI', 'Germany'), ('TWII', 'Taiwan')]" + }, + { + "dataset": "stockmarket", + "query": "1", + "run": "0", + "answer": "18.440000534057617" + }, + { + "dataset": "stockmarket", + "query": "2", + "run": "0", + "answer": "['BOIL', 'BZQ', 'COM', 'DUST', 'EDZ', 'ERX', 'FAZ', 'FXP', 'GFIN', 'GUSH', 'HYUP', 'JDST', 'JNUG', 'JPN', 'LABD', 'LABU', 'LBJ', 'MDY', 'PTIN', 'RTL', 'SDOW', 'SOXS', 'SSG', 'TECS', 'TZA', 'UVXY', 'VIXY', 'VPC', 'XES', 'XOP', 'YANG'], 31" + }, + { + "dataset": "stockmarket", + "query": "3", + "run": "0", + "answer": "[('Apex Global Brands Inc.', 23781.42), ('BIO-key International, Inc.', 10988.14), ('CBAK Energy Technology, Inc.', 86223.32), ('China Ceramics Co., Ltd.', 4366.80), ('Correvio Pharma Corp.', 145247.83), ('CounterPath Corporation', 375.49), ('DASAN Zhone Solutions, Inc.', 15578.66), ('Future FinTech Group Inc.', 9.85), ('Frontier Communications Corporation', 254397.63), ('Ideanomics, Inc.', 10.28), ('Ocean Power Technologies, Inc.', 254.15), ('Pacific Ethanol, Inc.', 10706.72), ('Synthesis Energy Systems, Inc.', 2390.51), ('Sunesis Pharmaceuticals, Inc.', 781.82), ('Sypris Solutions, Inc.', 36836.36)]" + }, + { + "dataset": "stockmarket", + "query": "4", + "run": "0", + "answer": "['MFA Financial, Inc.', 'Argo Group International Holdings, Ltd.', 'HDFC Bank Limited', 'Albany International Corporation', 'DTE Energy Company']" + }, + { + "dataset": "stockmarket", + "query": "5", + "run": "0", + "answer": "['Synthesis Energy Systems, Inc.', 'TD Holdings, Inc.', 'TMSR Holding Company Limited', 'Verb Technology Company, Inc.', 'Sunesis Pharmaceuticals, Inc.']" + }, + { + "dataset": "agnews", + "query": "1", + "run": "0", + "answer": "The Rundown" + }, + { + "dataset": "agnews", + "query": "2", + "run": "0", + "answer": "0.1892" + }, + { + "dataset": "agnews", + "query": "3", + "run": "0", + "answer": "353.64" + }, + { + "dataset": "agnews", + "query": "4", + "run": "0", + "answer": "Europe" + }, + { + "dataset": "bookreview", + "query": "1", + "run": "0", + "answer": "2020s" + }, + { + "dataset": "bookreview", + "query": "2", + "run": "0", + "answer": "['Reunion: The Children of Lauderdale Park', 'The Prophet: With Original 1923 Illustrations by the Author', 'The Melancholy Strumpet Master', 'Child Of The King A Journey of Hope Book 1: Earthly Story With A Heavenly Message', 'Fire Cracker', 'Local Honey', 'Hollywood Confessions: Hollywood Headlines Book #3 (Hollywood Headlines Mysteries)', 'Knowing When To Die: Uncollected Stories', 'Childe Harold of Dysna', 'Forged in Blood (Freehold)', 'Exits, Desires, & Slow Fires', 'Kennebago Moments', 'The Sludge', 'Liza of Lambeth', 'Something That Feels Like Truth (Switchgrass Books)']" + }, + { + "dataset": "bookreview", + "query": "3", + "run": "0", + "answer": "['The Library Book', \"Benny Goes To The Moon: The great new book from Top Children's entertainer Gerry Ogilvie (1)\", 'The Old Man and the Pirate Princess', 'LunaLu the Llamacorn', 'Around the World Mazes', 'Pok\u00e9mon: Sun & Moon, Vol. 8 (8)', 'Monstrous Stories #4: The Day the Mice Stood Still', 'Egypt (Enchantment of the World)', 'Favorite Thorton W. Burgess Stories: 6 Books', 'Behind the Wheel (Choose Your Own Adventure #35)(Paperback/Revised)', 'Cheer Up, Ben Franklin! (Young Historians)', 'Trouble in the CTC!: The Terra Prime Adventures Book 2', 'Clark the Shark: Tooth Trouble, No. 1', 'Cleo Porter and the Body Electric']" + }, + { + "dataset": "crmarenapro", + "query": "1", + "run": "0", + "answer": "Authority, Timeline" + }, + { + "dataset": "crmarenapro", + "query": "2", + "run": "0", + "answer": "ka0Wt000000Ens5IAC" + }, + { + "dataset": "crmarenapro", + "query": "3", + "run": "0", + "answer": "Negotiation" + }, + { + "dataset": "crmarenapro", + "query": "4", + "run": "0", + "answer": "November" + }, + { + "dataset": "crmarenapro", + "query": "5", + "run": "0", + "answer": "a03Wt00000JqnHwIAJ" + }, + { + "dataset": "crmarenapro", + "query": "6", + "run": "0", + "answer": "ka0Wt000000EnwvIAC" + }, + { + "dataset": "crmarenapro", + "query": "7", + "run": "0", + "answer": "ka0Wt000000EoD3IAK" + }, + { + "dataset": "crmarenapro", + "query": "8", + "run": "0", + "answer": "005Wt000003NIliIAG" + }, + { + "dataset": "crmarenapro", + "query": "9", + "run": "0", + "answer": "MI" + }, + { + "dataset": "crmarenapro", + "query": "10", + "run": "0", + "answer": "005Wt000003NDqDIAW" + }, + { + "dataset": "crmarenapro", + "query": "11", + "run": "0", + "answer": "01tWt000006hV8LIAU" + }, + { + "dataset": "crmarenapro", + "query": "12", + "run": "0", + "answer": "005Wt000003NDEBIA4" + }, + { + "dataset": "crmarenapro", + "query": "13", + "run": "0", + "answer": "005Wt000003NHw5IAG" + }, + { + "dataset": "googlelocal", + "query": "1", + "run": "0", + "answer": "['Widows Peak Salon', 'City Textile', 'Nobel Textile Co', 'San Soo Dang', 'Nova Fabrics']" + }, + { + "dataset": "googlelocal", + "query": "2", + "run": "0", + "answer": "[('Elite Massage', 5.0), ('Angel-A Massage', 4.33), ('Aurora Massage', 4.18), ('J B Oriental Inc', 4.17), ('Orient Massage', 3.93), ('Good Massage', 3.88), ('SUSY massage', 3.88), ('Happy Spa & Massage (Formerly Hawaii Massage & Spa)', 2.12)]" + }, + { + "dataset": "googlelocal", + "query": "3", + "run": "0", + "answer": "[{'name': 'Taba Rug Gallery', 'hours': [['Thursday', '10AM\u20137PM'], ['Friday', '10AM\u20137PM'], ['Saturday', '10AM\u20137PM'], ['Sunday', '11AM\u20136PM'], ['Monday', '10AM\u20137PM'], ['Tuesday', '10AM\u20137PM'], ['Wednesday', '10AM\u20137PM']], 'avg_rating': 5.0}, {'name': 'Beauty Divine Artistry', 'hours': [['Thursday', '9AM\u20138PM'], ['Friday', '9AM\u20138PM'], ['Saturday', '10AM\u20137PM'], ['Sunday', '11AM\u20136PM'], ['Monday', '9AM\u20138PM'], ['Tuesday', '9AM\u20138PM'], ['Wednesday', '9AM\u20138PM']], 'avg_rating': 5.0}, {'name': 'White Barn Candle Co', 'hours': [['Thursday', '10AM\u20139PM'], ['Friday', '10AM\u20139PM'], ['Saturday', '10AM\u20139PM'], ['Sunday', '11AM\u20137PM'], ['Monday', '10AM\u20139PM'], ['Tuesday', '10AM\u20139PM'], ['Wednesday', '10AM\u20139PM']], 'avg_rating': 5.0}, {'name': 'TACOS LA CABANA', 'hours': [['Thursday', 'Closed'], ['Friday', '5\u201311PM'], ['Saturday', '5\u201311PM'], ['Sunday', '5\u201311PM'], ['Monday', '5\u201311PM'], ['Tuesday', 'Closed'], ['Wednesday', 'Closed']], 'avg_rating': 5.0}, {'name': 'Mariscos el poblano', 'hours': [['Thursday', 'Open 24 hours'], ['Friday', '8AM\u20133:30PM'], ['Saturday', '8AM\u20133:30PM'], ['Sunday', '8AM\u20133:30PM'], ['Monday', '9AM\u20133:30AM'], ['Tuesday', '8AM\u20133:30PM'], ['Wednesday', '8AM\u20133:30PM']], 'avg_rating': 5.0}]" + }, + { + "dataset": "googlelocal", + "query": "4", + "run": "0", + "answer": "[('Encino Dermatology & Laser: Alex Khadavi MD', 19), ('The Boochyard @ Local Roots', 17), ('Aurora Massage', 14)]" + }, + { + "dataset": "pancancer_atlas", + "query": "1", + "run": "0", + "answer": "[('Astrocytoma', 2.5713), ('Oligoastrocytoma', 2.7136), ('Oligodendroglioma', 2.6825)]" + }, + { + "dataset": "pancancer_atlas", + "query": "2", + "run": "0", + "answer": "['Infiltrating Lobular Carcinoma', 'Mixed Histology (please specify)', 'Other specify']" + }, + { + "dataset": "pancancer_atlas", + "query": "3", + "run": "0", + "answer": "305.12" + }, + { + "dataset": "yelp", + "query": "1", + "run": "0", + "answer": "3.55" + }, + { + "dataset": "yelp", + "query": "2", + "run": "0", + "answer": "PA, 3.70" + }, + { + "dataset": "yelp", + "query": "3", + "run": "0", + "answer": "35" + }, + { + "dataset": "yelp", + "query": "4", + "run": "0", + "answer": "Restaurants, 3.63" + }, + { + "dataset": "yelp", + "query": "5", + "run": "0", + "answer": "PA, average rating: 3.48" + }, + { + "dataset": "yelp", + "query": "6", + "run": "0", + "answer": "Coffee House Too Cafe \u2014 Restaurants, Breakfast & Brunch, American (New), Cafes" + }, + { + "dataset": "yelp", + "query": "7", + "run": "0", + "answer": "['Restaurants', 'Food', 'American (New)', 'Shopping', 'Breakfast & Brunch']" + }, + { + "dataset": "deps_dev_v1", + "query": "1", + "run": "1", + "answer": "[('@dylanvann/svelte', '3.25.4', 73499), ('@dumc11/tailwindcss', '0.4.0', 73464), ('@dreampie/semantic-ui', '2.2.11', 51069), ('@dongls/pdfjs-dist', '3.2.72', 44231), ('@dman777/shadow-dom-quill-temp', '1.0.0', 42407)]" + }, + { + "dataset": "deps_dev_v1", + "query": "2", + "run": "1", + "answer": "['mui-org/material-ui', 'moment/moment', 'semantic-org/semantic-ui', 'react-native-elements/react-native-elements', 'sveltejs/svelte']" + }, + { + "dataset": "github_repos", + "query": "1", + "run": "1", + "answer": "0.0005" + }, + { + "dataset": "github_repos", + "query": "2", + "run": "1", + "answer": "uacaps/PageMenu" + }, + { + "dataset": "github_repos", + "query": "3", + "run": "1", + "answer": "1077" + }, + { + "dataset": "github_repos", + "query": "4", + "run": "1", + "answer": "['apple/swift', 'twbs/bootstrap', 'Microsoft/vscode', 'facebook/react', 'tensorflow/tensorflow']" + }, + { + "dataset": "music_brainz_20k", + "query": "1", + "run": "1", + "answer": "1059.46" + }, + { + "dataset": "music_brainz_20k", + "query": "2", + "run": "1", + "answer": "Amazon Music" + }, + { + "dataset": "music_brainz_20k", + "query": "3", + "run": "1", + "answer": "Zo gaat het leven aan je voor" + }, + { + "dataset": "stockindex", + "query": "1", + "run": "1", + "answer": "399001.SZ" + }, + { + "dataset": "stockindex", + "query": "2", + "run": "1", + "answer": "['IXIC']" + }, + { + "dataset": "stockindex", + "query": "3", + "run": "1", + "answer": "[('IXIC', 'United States'), ('NSEI', 'India'), ('399001.SZ', 'China'), ('GDAXI', 'Germany'), ('TWII', 'Taiwan')]" + }, + { + "dataset": "stockmarket", + "query": "1", + "run": "1", + "answer": "18.440000534057617" + }, + { + "dataset": "stockmarket", + "query": "2", + "run": "1", + "answer": "['UVXY', 'GUSH', 'COM', 'RTL', 'JDST', 'HYUP', 'SOXS', 'VPC', 'DUST', 'LBJ', 'PTIN', 'GFIN', 'ERX', 'LABD', 'VIXY', 'BZQ', 'JNUG', 'SDOW', 'BOIL', 'YANG', 'EDZ', 'FAZ', 'XES', 'TZA', 'SSG', 'MDY', 'TECS', 'LABU', 'XOP', 'FXP', 'JPN'], 31" + }, + { + "dataset": "stockmarket", + "query": "3", + "run": "1", + "answer": "{'Apex Global Brands Inc.': 23781.42, 'BIO-key International, Inc.': 10988.14, 'CBAK Energy Technology, Inc.': 86223.32, 'China Ceramics Co., Ltd.': 4366.80, 'Correvio Pharma Corp.': 145247.83, 'CounterPath Corporation': 375.49, 'DASAN Zhone Solutions, Inc.': 15578.66, 'Future FinTech Group Inc.': 9.85, 'Frontier Communications Corporation': 254397.63, 'Ideanomics, Inc.': 10.28, 'Ocean Power Technologies, Inc.': 254.15, 'Pacific Ethanol, Inc.': 10706.72, 'Synthesis Energy Systems, Inc.': 2390.51, 'Sunesis Pharmaceuticals, Inc.': 781.82, 'Sypris Solutions, Inc.': 36836.36}" + }, + { + "dataset": "stockmarket", + "query": "4", + "run": "1", + "answer": "['MFA Financial, Inc.', 'Argo Group International Holdings, Ltd.', 'HDFC Bank Limited', 'Albany International Corporation', 'DTE Energy Company']" + }, + { + "dataset": "stockmarket", + "query": "5", + "run": "1", + "answer": "['Synthesis Energy Systems, Inc.', 'TD Holdings, Inc.', 'TMSR Holding Company Limited', 'Verb Technology Company, Inc.', 'Sunesis Pharmaceuticals, Inc.']" + }, + { + "dataset": "agnews", + "query": "1", + "run": "1", + "answer": "The Rundown" + }, + { + "dataset": "agnews", + "query": "2", + "run": "1", + "answer": "0.1532" + }, + { + "dataset": "agnews", + "query": "3", + "run": "1", + "answer": "353.64" + }, + { + "dataset": "agnews", + "query": "4", + "run": "1", + "answer": "Europe" + }, + { + "dataset": "bookreview", + "query": "1", + "run": "1", + "answer": "2020s" + }, + { + "dataset": "bookreview", + "query": "2", + "run": "1", + "answer": "['Knowing When To Die: Uncollected Stories', 'Childe Harold of Dysna', 'Forged in Blood (Freehold)', 'Exits, Desires, & Slow Fires', 'Kennebago Moments', 'The Sludge', 'Liza of Lambeth', 'Something That Feels Like Truth (Switchgrass Books)', 'The Prophet: With Original 1923 Illustrations by the Author', 'The Melancholy Strumpet Master', 'Child Of The King A Journey of Hope Book 1: Earthly Story With A Heavenly Message', 'Fire Cracker', 'Local Honey', 'Reunion: The Children of Lauderdale Park', 'Hollywood Confessions: Hollywood Headlines Book #3 (Hollywood Headlines Mysteries)']" + }, + { + "dataset": "bookreview", + "query": "3", + "run": "1", + "answer": "['The Library Book', \"Benny Goes To The Moon: The great new book from Top Children's entertainer Gerry Ogilvie (1)\", 'The Old Man and the Pirate Princess', 'LunaLu the Llamacorn', 'Trouble in the CTC!: The Terra Prime Adventures Book 2', 'Around the World Mazes', 'Cleo Porter and the Body Electric', 'Pok\u00e9mon: Sun & Moon, Vol. 8 (8)', 'Monstrous Stories #4: The Day the Mice Stood Still', 'Egypt (Enchantment of the World)', 'Clark the Shark: Tooth Trouble, No. 1', 'Favorite Thorton W. Burgess Stories: 6 Books', 'Behind the Wheel (Choose Your Own Adventure #35)(Paperback/Revised)', 'Cheer Up, Ben Franklin! (Young Historians)']" + }, + { + "dataset": "crmarenapro", + "query": "1", + "run": "1", + "answer": "Authority, Timeline" + }, + { + "dataset": "crmarenapro", + "query": "2", + "run": "1", + "answer": "ka0Wt000000Ens5IAC" + }, + { + "dataset": "crmarenapro", + "query": "3", + "run": "1", + "answer": "Negotiation" + }, + { + "dataset": "crmarenapro", + "query": "4", + "run": "1", + "answer": "November" + }, + { + "dataset": "crmarenapro", + "query": "5", + "run": "1", + "answer": "a03Wt00000JqnHwIAJ" + }, + { + "dataset": "crmarenapro", + "query": "6", + "run": "1", + "answer": "ka0Wt000000EnwvIAC" + }, + { + "dataset": "crmarenapro", + "query": "7", + "run": "1", + "answer": "ka0Wt000000EoD3IAK" + }, + { + "dataset": "crmarenapro", + "query": "8", + "run": "1", + "answer": "005Wt000003NIliIAG" + }, + { + "dataset": "crmarenapro", + "query": "9", + "run": "1", + "answer": "MI" + }, + { + "dataset": "crmarenapro", + "query": "10", + "run": "1", + "answer": "005Wt000003NDqDIAW" + }, + { + "dataset": "crmarenapro", + "query": "11", + "run": "1", + "answer": "01tWt000006hV8LIAU" + }, + { + "dataset": "crmarenapro", + "query": "12", + "run": "1", + "answer": "005Wt000003NDEBIA4" + }, + { + "dataset": "crmarenapro", + "query": "13", + "run": "1", + "answer": "005Wt000003NHw5IAG" + }, + { + "dataset": "googlelocal", + "query": "1", + "run": "1", + "answer": "['Widows Peak Salon', 'City Textile', 'Nobel Textile Co', 'San Soo Dang', 'Nova Fabrics']" + }, + { + "dataset": "googlelocal", + "query": "2", + "run": "1", + "answer": "[('Elite Massage', 5.0), ('Angel-A Massage', 4.33), ('Aurora Massage', 4.18), ('J B Oriental Inc', 4.17), ('Orient Massage', 3.93), ('Good Massage', 3.88), ('SUSY massage', 3.88), ('Happy Spa & Massage (Formerly Hawaii Massage & Spa)', 2.12)]" + }, + { + "dataset": "googlelocal", + "query": "3", + "run": "1", + "answer": "[{'name': 'Taba Rug Gallery', 'hours': [['Thursday', '10AM\u20137PM'], ['Friday', '10AM\u20137PM'], ['Saturday', '10AM\u20137PM'], ['Sunday', '11AM\u20136PM'], ['Monday', '10AM\u20137PM'], ['Tuesday', '10AM\u20137PM'], ['Wednesday', '10AM\u20137PM']], 'avg_rating': 5.0}, {'name': 'Beauty Divine Artistry', 'hours': [['Thursday', '9AM\u20138PM'], ['Friday', '9AM\u20138PM'], ['Saturday', '10AM\u20137PM'], ['Sunday', '11AM\u20136PM'], ['Monday', '9AM\u20138PM'], ['Tuesday', '9AM\u20138PM'], ['Wednesday', '9AM\u20138PM']], 'avg_rating': 5.0}, {'name': 'Mariscos el poblano', 'hours': [['Thursday', 'Open 24 hours'], ['Friday', '8AM\u20133:30PM'], ['Saturday', '8AM\u20133:30PM'], ['Sunday', '8AM\u20133:30PM'], ['Monday', '9AM\u20133:30AM'], ['Tuesday', '8AM\u20133:30PM'], ['Wednesday', '8AM\u20133:30PM']], 'avg_rating': 5.0}, {'name': 'White Barn Candle Co', 'hours': [['Thursday', '10AM\u20139PM'], ['Friday', '10AM\u20139PM'], ['Saturday', '10AM\u20139PM'], ['Sunday', '11AM\u20137PM'], ['Monday', '10AM\u20139PM'], ['Tuesday', '10AM\u20139PM'], ['Wednesday', '10AM\u20139PM']], 'avg_rating': 5.0}, {'name': 'TACOS LA CABANA', 'hours': [['Thursday', 'Closed'], ['Friday', '5\u201311PM'], ['Saturday', '5\u201311PM'], ['Sunday', '5\u201311PM'], ['Monday', '5\u201311PM'], ['Tuesday', 'Closed'], ['Wednesday', 'Closed']], 'avg_rating': 5.0}]" + }, + { + "dataset": "googlelocal", + "query": "4", + "run": "1", + "answer": "[('Encino Dermatology & Laser: Alex Khadavi MD', 19), ('The Boochyard @ Local Roots', 17), ('Aurora Massage', 14)]" + }, + { + "dataset": "pancancer_atlas", + "query": "1", + "run": "1", + "answer": "[('Astrocytoma', 2.5713), ('Oligoastrocytoma', 2.7136), ('Oligodendroglioma', 2.6825)]" + }, + { + "dataset": "pancancer_atlas", + "query": "2", + "run": "1", + "answer": "['Infiltrating Lobular Carcinoma', 'Mixed Histology (please specify)', 'Other specify']" + }, + { + "dataset": "pancancer_atlas", + "query": "3", + "run": "1", + "answer": "305.12" + }, + { + "dataset": "yelp", + "query": "1", + "run": "1", + "answer": "3.55" + }, + { + "dataset": "yelp", + "query": "2", + "run": "1", + "answer": "PA, 3.70" + }, + { + "dataset": "yelp", + "query": "3", + "run": "1", + "answer": "35" + }, + { + "dataset": "yelp", + "query": "4", + "run": "1", + "answer": "Restaurants, 3.63" + }, + { + "dataset": "yelp", + "query": "5", + "run": "1", + "answer": "PA, average rating: 3.48" + }, + { + "dataset": "yelp", + "query": "6", + "run": "1", + "answer": "Coffee House Too Cafe \u2014 Restaurants, Breakfast & Brunch, American (New), Cafes" + }, + { + "dataset": "yelp", + "query": "7", + "run": "1", + "answer": "['Restaurants', 'Food', 'Shopping', 'American (New)', 'Grocery']" + }, + { + "dataset": "deps_dev_v1", + "query": "1", + "run": "2", + "answer": "[('@dylanvann/svelte', '3.25.4', 73499), ('@dumc11/tailwindcss', '0.4.0', 73464), ('@dreampie/semantic-ui', '2.2.11', 51069), ('@dongls/pdfjs-dist', '3.2.72', 44231), ('@dman777/shadow-dom-quill-temp', '1.0.0', 42407)]" + }, + { + "dataset": "deps_dev_v1", + "query": "2", + "run": "2", + "answer": "['mui-org/material-ui', 'moment/moment', 'semantic-org/semantic-ui', 'react-native-elements/react-native-elements', 'sveltejs/svelte']" + }, + { + "dataset": "github_repos", + "query": "1", + "run": "2", + "answer": "0.0005" + }, + { + "dataset": "github_repos", + "query": "2", + "run": "2", + "answer": "uacaps/PageMenu" + }, + { + "dataset": "github_repos", + "query": "3", + "run": "2", + "answer": "1077" + }, + { + "dataset": "github_repos", + "query": "4", + "run": "2", + "answer": "['apple/swift', 'twbs/bootstrap', 'Microsoft/vscode', 'facebook/react', 'tensorflow/tensorflow']" + }, + { + "dataset": "music_brainz_20k", + "query": "1", + "run": "2", + "answer": "1059.46" + }, + { + "dataset": "music_brainz_20k", + "query": "2", + "run": "2", + "answer": "Amazon Music" + }, + { + "dataset": "music_brainz_20k", + "query": "3", + "run": "2", + "answer": "Zo gaat het leven aan je voor" + }, + { + "dataset": "stockindex", + "query": "1", + "run": "2", + "answer": "399001.SZ" + }, + { + "dataset": "stockindex", + "query": "2", + "run": "2", + "answer": "['IXIC']" + }, + { + "dataset": "stockindex", + "query": "3", + "run": "2", + "answer": "[('IXIC', 'United States'), ('GDAXI', 'Germany'), ('NSEI', 'India'), ('399001.SZ', 'China'), ('TWII', 'Taiwan')]" + }, + { + "dataset": "stockmarket", + "query": "1", + "run": "2", + "answer": "18.440000534057617" + }, + { + "dataset": "stockmarket", + "query": "2", + "run": "2", + "answer": "['BOIL', 'BZQ', 'COM', 'DUST', 'EDZ', 'ERX', 'FAZ', 'FXP', 'GFIN', 'GUSH', 'HYUP', 'JDST', 'JNUG', 'JPN', 'LABD', 'LABU', 'LBJ', 'MDY', 'PTIN', 'RTL', 'SDOW', 'SOXS', 'SSG', 'TECS', 'TZA', 'UVXY', 'VIXY', 'VPC', 'XES', 'XOP', 'YANG'], 31" + }, + { + "dataset": "stockmarket", + "query": "3", + "run": "2", + "answer": "[('Apex Global Brands Inc.', 23781.4229), ('BIO-key International, Inc.', 10988.1423), ('CBAK Energy Technology, Inc.', 86223.3202), ('China Ceramics Co., Ltd.', 4366.7984), ('Correvio Pharma Corp.', 145247.8261), ('CounterPath Corporation', 375.4941), ('DASAN Zhone Solutions, Inc.', 15578.6561), ('Future FinTech Group Inc.', 9.8452), ('Frontier Communications Corporation', 254397.6285), ('Ideanomics, Inc.', 10.2767), ('Ocean Power Technologies, Inc.', 254.1502), ('Pacific Ethanol, Inc.', 10706.7194), ('Synthesis Energy Systems, Inc.', 2390.5138), ('Sunesis Pharmaceuticals, Inc.', 781.8182), ('Sypris Solutions, Inc.', 36836.3636)]" + }, + { + "dataset": "stockmarket", + "query": "4", + "run": "2", + "answer": "['MFA Financial, Inc.', 'Argo Group International Holdings, Ltd.', 'HDFC Bank Limited', 'Albany International Corporation', 'DTE Energy Company']" + }, + { + "dataset": "stockmarket", + "query": "5", + "run": "2", + "answer": "['Synthesis Energy Systems, Inc.', 'TD Holdings, Inc.', 'TMSR Holding Company Limited', 'Verb Technology Company, Inc.', 'Sunesis Pharmaceuticals, Inc.']" + }, + { + "dataset": "agnews", + "query": "1", + "run": "2", + "answer": "The Rundown" + }, + { + "dataset": "agnews", + "query": "2", + "run": "2", + "answer": "0.1532" + }, + { + "dataset": "agnews", + "query": "3", + "run": "2", + "answer": "353.64" + }, + { + "dataset": "agnews", + "query": "4", + "run": "2", + "answer": "Europe" + }, + { + "dataset": "bookreview", + "query": "1", + "run": "2", + "answer": "2020s" + }, + { + "dataset": "bookreview", + "query": "2", + "run": "2", + "answer": "['Reunion: The Children of Lauderdale Park', 'The Prophet: With Original 1923 Illustrations by the Author', 'The Melancholy Strumpet Master', 'Child Of The King A Journey of Hope Book 1: Earthly Story With A Heavenly Message', 'Fire Cracker', 'Local Honey', 'Hollywood Confessions: Hollywood Headlines Book #3 (Hollywood Headlines Mysteries)', 'Knowing When To Die: Uncollected Stories', 'Childe Harold of Dysna', 'Forged in Blood (Freehold)', 'Exits, Desires, & Slow Fires', 'Kennebago Moments', 'The Sludge', 'Liza of Lambeth', 'Something That Feels Like Truth (Switchgrass Books)']" + }, + { + "dataset": "bookreview", + "query": "3", + "run": "2", + "answer": "['The Library Book', \"Benny Goes To The Moon: The great new book from Top Children's entertainer Gerry Ogilvie (1)\", 'The Old Man and the Pirate Princess', 'LunaLu the Llamacorn', 'Trouble in the CTC!: The Terra Prime Adventures Book 2', 'Around the World Mazes', 'Cleo Porter and the Body Electric', 'Pok\u00e9mon: Sun & Moon, Vol. 8 (8)', 'Monstrous Stories #4: The Day the Mice Stood Still', 'Egypt (Enchantment of the World)', 'Clark the Shark: Tooth Trouble, No. 1', 'Favorite Thorton W. Burgess Stories: 6 Books', 'Behind the Wheel (Choose Your Own Adventure #35)(Paperback/Revised)', 'Cheer Up, Ben Franklin! (Young Historians)']" + }, + { + "dataset": "crmarenapro", + "query": "1", + "run": "2", + "answer": "Authority, Timeline" + }, + { + "dataset": "crmarenapro", + "query": "2", + "run": "2", + "answer": "ka0Wt000000Ens5IAC" + }, + { + "dataset": "crmarenapro", + "query": "3", + "run": "2", + "answer": "Negotiation" + }, + { + "dataset": "crmarenapro", + "query": "4", + "run": "2", + "answer": "November" + }, + { + "dataset": "crmarenapro", + "query": "5", + "run": "2", + "answer": "a03Wt00000JqnHwIAJ" + }, + { + "dataset": "crmarenapro", + "query": "6", + "run": "2", + "answer": "ka0Wt000000EnwvIAC" + }, + { + "dataset": "crmarenapro", + "query": "7", + "run": "2", + "answer": "ka0Wt000000EoD3IAK" + }, + { + "dataset": "crmarenapro", + "query": "8", + "run": "2", + "answer": "005Wt000003NIliIAG" + }, + { + "dataset": "crmarenapro", + "query": "9", + "run": "2", + "answer": "MI" + }, + { + "dataset": "crmarenapro", + "query": "10", + "run": "2", + "answer": "005Wt000003NDqDIAW" + }, + { + "dataset": "crmarenapro", + "query": "11", + "run": "2", + "answer": "01tWt000006hV8LIAU" + }, + { + "dataset": "crmarenapro", + "query": "12", + "run": "2", + "answer": "005Wt000003NDEBIA4" + }, + { + "dataset": "crmarenapro", + "query": "13", + "run": "2", + "answer": "005Wt000003NHw5IAG" + }, + { + "dataset": "googlelocal", + "query": "1", + "run": "2", + "answer": "['Widows Peak Salon', 'City Textile', 'Nobel Textile Co', 'San Soo Dang', 'Nova Fabrics']" + }, + { + "dataset": "googlelocal", + "query": "2", + "run": "2", + "answer": "[('Elite Massage', 5.0), ('Angel-A Massage', 4.33), ('Aurora Massage', 4.18), ('J B Oriental Inc', 4.17), ('Orient Massage', 3.93), ('Good Massage', 3.88), ('SUSY massage', 3.88), ('Happy Spa & Massage (Formerly Hawaii Massage & Spa)', 2.12)]" + }, + { + "dataset": "googlelocal", + "query": "3", + "run": "2", + "answer": "[{'name': 'Taba Rug Gallery', 'hours': 'Mon-Sat 10AM-7PM; Sun 11AM-6PM', 'avg_rating': 5.0}, {'name': 'Beauty Divine Artistry', 'hours': 'Mon-Fri 9AM-8PM; Sat 10AM-7PM; Sun 11AM-6PM', 'avg_rating': 5.0}, {'name': 'Mariscos el poblano', 'hours': 'Thu Open 24 hours; Mon 9AM-3:30AM; Tue/Wed/Fri/Sat/Sun 8AM-3:30PM', 'avg_rating': 5.0}, {'name': 'TACOS LA CABANA', 'hours': 'Fri/Sat/Sun/Mon 5-11PM; Tue/Wed/Thu Closed', 'avg_rating': 5.0}, {'name': 'White Barn Candle Co', 'hours': 'Mon-Sat 10AM-9PM; Sun 11AM-7PM', 'avg_rating': 5.0}]" + }, + { + "dataset": "googlelocal", + "query": "4", + "run": "2", + "answer": "[('Encino Dermatology & Laser: Alex Khadavi MD', 19), ('The Boochyard @ Local Roots', 17), ('Aurora Massage', 14)]" + }, + { + "dataset": "pancancer_atlas", + "query": "1", + "run": "2", + "answer": "[('Astrocytoma', 2.5713), ('Oligoastrocytoma', 2.7136), ('Oligodendroglioma', 2.6825)]" + }, + { + "dataset": "pancancer_atlas", + "query": "2", + "run": "2", + "answer": "['Infiltrating Lobular Carcinoma', 'Mixed Histology (please specify)', 'Other specify']" + }, + { + "dataset": "pancancer_atlas", + "query": "3", + "run": "2", + "answer": "305.12" + }, + { + "dataset": "yelp", + "query": "1", + "run": "2", + "answer": "3.55" + }, + { + "dataset": "yelp", + "query": "2", + "run": "2", + "answer": "PA, 3.70" + }, + { + "dataset": "yelp", + "query": "3", + "run": "2", + "answer": "35" + }, + { + "dataset": "yelp", + "query": "4", + "run": "2", + "answer": "Restaurants, 3.64" + }, + { + "dataset": "yelp", + "query": "5", + "run": "2", + "answer": "PA, average rating: 3.48" + }, + { + "dataset": "yelp", + "query": "6", + "run": "2", + "answer": "Coffee House Too Cafe \u2014 Restaurants, Breakfast & Brunch, American (New), Cafes" + }, + { + "dataset": "yelp", + "query": "7", + "run": "2", + "answer": "['Restaurants', 'Food', 'American (New)', 'Shopping', 'Breakfast & Brunch']" + }, + { + "dataset": "deps_dev_v1", + "query": "1", + "run": "3", + "answer": "[('@dylanvann/svelte', '3.25.4', 73499), ('@dumc11/tailwindcss', '0.4.0', 73464), ('@dreampie/semantic-ui', '2.2.11', 51069), ('@dongls/pdfjs-dist', '3.2.72', 44231), ('@dman777/shadow-dom-quill-temp', '1.0.0', 42407)]" + }, + { + "dataset": "deps_dev_v1", + "query": "2", + "run": "3", + "answer": "['mui-org/material-ui', 'moment/moment', 'semantic-org/semantic-ui', 'react-native-elements/react-native-elements', 'sveltejs/svelte']" + }, + { + "dataset": "github_repos", + "query": "1", + "run": "3", + "answer": "0.0005" + }, + { + "dataset": "github_repos", + "query": "2", + "run": "3", + "answer": "uacaps/PageMenu" + }, + { + "dataset": "github_repos", + "query": "3", + "run": "3", + "answer": "1077" + }, + { + "dataset": "github_repos", + "query": "4", + "run": "3", + "answer": "['apple/swift', 'twbs/bootstrap', 'Microsoft/vscode', 'facebook/react', 'tensorflow/tensorflow']" + }, + { + "dataset": "music_brainz_20k", + "query": "1", + "run": "3", + "answer": "1059.46" + }, + { + "dataset": "music_brainz_20k", + "query": "2", + "run": "3", + "answer": "Amazon Music" + }, + { + "dataset": "music_brainz_20k", + "query": "3", + "run": "3", + "answer": "Groovey" + }, + { + "dataset": "stockindex", + "query": "1", + "run": "3", + "answer": "399001.SZ" + }, + { + "dataset": "stockindex", + "query": "2", + "run": "3", + "answer": "['IXIC']" + }, + { + "dataset": "stockindex", + "query": "3", + "run": "3", + "answer": "[('IXIC', 'United States'), ('399001.SZ', 'China'), ('TWII', 'Taiwan'), ('GDAXI', 'Germany'), ('NSEI', 'India')]" + }, + { + "dataset": "stockmarket", + "query": "1", + "run": "3", + "answer": "18.440000534057617" + }, + { + "dataset": "stockmarket", + "query": "2", + "run": "3", + "answer": "['BOIL', 'BZQ', 'COM', 'DUST', 'EDZ', 'ERX', 'FAZ', 'FXP', 'GFIN', 'GUSH', 'HYUP', 'JDST', 'JNUG', 'JPN', 'LABD', 'LABU', 'LBJ', 'MDY', 'PTIN', 'RTL', 'SDOW', 'SOXS', 'SSG', 'TECS', 'TZA', 'UVXY', 'VIXY', 'VPC', 'XES', 'XOP', 'YANG'], 31" + }, + { + "dataset": "stockmarket", + "query": "3", + "run": "3", + "answer": "{'Apex Global Brands Inc.': 23781.42, 'BIO-key International, Inc.': 10988.14, 'CBAK Energy Technology, Inc.': 86223.32, 'China Ceramics Co., Ltd.': 4366.80, 'Correvio Pharma Corp.': 145247.83, 'CounterPath Corporation': 375.49, 'DASAN Zhone Solutions, Inc.': 15578.66, 'Future FinTech Group Inc.': 9.85, 'Frontier Communications Corporation': 254397.63, 'Ideanomics, Inc.': 10.28, 'Ocean Power Technologies, Inc.': 254.15, 'Pacific Ethanol, Inc.': 10706.72, 'Synthesis Energy Systems, Inc.': 2390.51, 'Sunesis Pharmaceuticals, Inc.': 781.82, 'Sypris Solutions, Inc.': 36836.36}" + }, + { + "dataset": "stockmarket", + "query": "4", + "run": "3", + "answer": "['MFA Financial, Inc.', 'Argo Group International Holdings, Ltd.', 'HDFC Bank Limited', 'Albany International Corporation', 'DTE Energy Company']" + }, + { + "dataset": "stockmarket", + "query": "5", + "run": "3", + "answer": "['Synthesis Energy Systems, Inc.', 'TD Holdings, Inc.', 'TMSR Holding Company Limited', 'Verb Technology Company, Inc.', 'Sunesis Pharmaceuticals, Inc.']" + }, + { + "dataset": "agnews", + "query": "1", + "run": "3", + "answer": "The Rundown" + }, + { + "dataset": "agnews", + "query": "2", + "run": "3", + "answer": "0.1712" + }, + { + "dataset": "agnews", + "query": "3", + "run": "3", + "answer": "353.64" + }, + { + "dataset": "agnews", + "query": "4", + "run": "3", + "answer": "Europe" + }, + { + "dataset": "bookreview", + "query": "1", + "run": "3", + "answer": "2020s" + }, + { + "dataset": "bookreview", + "query": "2", + "run": "3", + "answer": "['Reunion: The Children of Lauderdale Park', 'The Prophet: With Original 1923 Illustrations by the Author', 'The Melancholy Strumpet Master', 'Child Of The King A Journey of Hope Book 1: Earthly Story With A Heavenly Message', 'Fire Cracker', 'Local Honey', 'Hollywood Confessions: Hollywood Headlines Book #3 (Hollywood Headlines Mysteries)', 'Knowing When To Die: Uncollected Stories', 'Childe Harold of Dysna', 'Forged in Blood (Freehold)', 'Exits, Desires, & Slow Fires', 'Kennebago Moments', 'The Sludge', 'Liza of Lambeth', 'Something That Feels Like Truth (Switchgrass Books)']" + }, + { + "dataset": "bookreview", + "query": "3", + "run": "3", + "answer": "['Monstrous Stories #4: The Day the Mice Stood Still', 'The Old Man and the Pirate Princess', 'Egypt (Enchantment of the World)', 'Clark the Shark: Tooth Trouble, No. 1', 'Favorite Thorton W. Burgess Stories: 6 Books', 'Behind the Wheel (Choose Your Own Adventure #35)(Paperback/Revised)', 'Cheer Up, Ben Franklin! (Young Historians)', 'The Library Book', \"Benny Goes To The Moon: The great new book from Top Children's entertainer Gerry Ogilvie (1)\", 'LunaLu the Llamacorn', 'Trouble in the CTC!: The Terra Prime Adventures Book 2', 'Around the World Mazes', 'Cleo Porter and the Body Electric', 'Pok\u00e9mon: Sun & Moon, Vol. 8 (8)']" + }, + { + "dataset": "crmarenapro", + "query": "1", + "run": "3", + "answer": "Authority, Timeline" + }, + { + "dataset": "crmarenapro", + "query": "2", + "run": "3", + "answer": "ka0Wt000000Ens5IAC" + }, + { + "dataset": "crmarenapro", + "query": "3", + "run": "3", + "answer": "Negotiation" + }, + { + "dataset": "crmarenapro", + "query": "4", + "run": "3", + "answer": "November" + }, + { + "dataset": "crmarenapro", + "query": "5", + "run": "3", + "answer": "a03Wt00000JqnHwIAJ" + }, + { + "dataset": "crmarenapro", + "query": "6", + "run": "3", + "answer": "ka0Wt000000EnwvIAC" + }, + { + "dataset": "crmarenapro", + "query": "7", + "run": "3", + "answer": "ka0Wt000000EoD3IAK" + }, + { + "dataset": "crmarenapro", + "query": "8", + "run": "3", + "answer": "005Wt000003NIliIAG" + }, + { + "dataset": "crmarenapro", + "query": "9", + "run": "3", + "answer": "MI" + }, + { + "dataset": "crmarenapro", + "query": "10", + "run": "3", + "answer": "005Wt000003NDqDIAW" + }, + { + "dataset": "crmarenapro", + "query": "11", + "run": "3", + "answer": "01tWt000006hV8LIAU" + }, + { + "dataset": "crmarenapro", + "query": "12", + "run": "3", + "answer": "005Wt000003NDEBIA4" + }, + { + "dataset": "crmarenapro", + "query": "13", + "run": "3", + "answer": "005Wt000003NHw5IAG" + }, + { + "dataset": "googlelocal", + "query": "1", + "run": "3", + "answer": "['Widows Peak Salon', 'City Textile', 'Nobel Textile Co', 'San Soo Dang', 'Nova Fabrics']" + }, + { + "dataset": "googlelocal", + "query": "2", + "run": "3", + "answer": "[('Elite Massage', 5.0), ('Angel-A Massage', 4.33), ('Aurora Massage', 4.18), ('J B Oriental Inc', 4.17), ('Orient Massage', 3.93), ('Good Massage', 3.88), ('SUSY massage', 3.88), ('Happy Spa & Massage (Formerly Hawaii Massage & Spa)', 2.12)]" + }, + { + "dataset": "googlelocal", + "query": "3", + "run": "3", + "answer": "[('Taba Rug Gallery', 'Mon-Sat 10AM-7PM, Sun 11AM-6PM', 5.0), ('Beauty Divine Artistry', 'Mon-Fri 9AM-8PM, Sat 10AM-7PM, Sun 11AM-6PM', 5.0), ('White Barn Candle Co', 'Mon-Sat 10AM-9PM, Sun 11AM-7PM', 5.0), ('TACOS LA CABANA', 'Fri-Mon 5-11PM, Tue/Wed/Thu Closed', 5.0), ('Mariscos el poblano', 'Thu Open 24 hours, Mon 9AM-3:30AM, Fri-Wed 8AM-3:30PM', 5.0)]" + }, + { + "dataset": "googlelocal", + "query": "4", + "run": "3", + "answer": "[('Encino Dermatology & Laser: Alex Khadavi MD', 19), ('The Boochyard @ Local Roots', 17), ('Aurora Massage', 14)]" + }, + { + "dataset": "pancancer_atlas", + "query": "1", + "run": "3", + "answer": "[('Astrocytoma', 2.5713), ('Oligoastrocytoma', 2.7136), ('Oligodendroglioma', 2.6825)]" + }, + { + "dataset": "pancancer_atlas", + "query": "2", + "run": "3", + "answer": "['Infiltrating Lobular Carcinoma', 'Mixed Histology (please specify)', 'Other specify']" + }, + { + "dataset": "pancancer_atlas", + "query": "3", + "run": "3", + "answer": "305.1239" + }, + { + "dataset": "yelp", + "query": "1", + "run": "3", + "answer": "3.55" + }, + { + "dataset": "yelp", + "query": "2", + "run": "3", + "answer": "PA, 3.70" + }, + { + "dataset": "yelp", + "query": "3", + "run": "3", + "answer": "35" + }, + { + "dataset": "yelp", + "query": "4", + "run": "3", + "answer": "Restaurants, 3.46" + }, + { + "dataset": "yelp", + "query": "5", + "run": "3", + "answer": "PA, average rating: 3.48" + }, + { + "dataset": "yelp", + "query": "6", + "run": "3", + "answer": "Coffee House Too Cafe \u2014 Restaurants, Breakfast & Brunch, American (New), Cafes" + }, + { + "dataset": "yelp", + "query": "7", + "run": "3", + "answer": "['Restaurants', 'Food', 'American (New)', 'Shopping', 'Breakfast & Brunch']" + }, + { + "dataset": "deps_dev_v1", + "query": "1", + "run": "4", + "answer": "[('@dylanvann/svelte', '3.25.4', 73499), ('@dumc11/tailwindcss', '0.4.0', 73464), ('@dreampie/semantic-ui', '2.2.11', 51069), ('@dongls/pdfjs-dist', '3.2.72', 44231), ('@dman777/shadow-dom-quill-temp', '1.0.0', 42407)]" + }, + { + "dataset": "deps_dev_v1", + "query": "2", + "run": "4", + "answer": "['mui-org/material-ui', 'moment/moment', 'semantic-org/semantic-ui', 'react-native-elements/react-native-elements', 'sveltejs/svelte']" + }, + { + "dataset": "github_repos", + "query": "1", + "run": "4", + "answer": "0.0005" + }, + { + "dataset": "github_repos", + "query": "2", + "run": "4", + "answer": "uacaps/PageMenu" + }, + { + "dataset": "github_repos", + "query": "3", + "run": "4", + "answer": "1077" + }, + { + "dataset": "github_repos", + "query": "4", + "run": "4", + "answer": "['apple/swift', 'twbs/bootstrap', 'Microsoft/vscode', 'facebook/react', 'tensorflow/tensorflow']" + }, + { + "dataset": "music_brainz_20k", + "query": "1", + "run": "4", + "answer": "1059.46" + }, + { + "dataset": "music_brainz_20k", + "query": "2", + "run": "4", + "answer": "Amazon Music" + }, + { + "dataset": "music_brainz_20k", + "query": "3", + "run": "4", + "answer": "Zo gaat het leven aan je voor" + }, + { + "dataset": "stockindex", + "query": "1", + "run": "4", + "answer": "399001.SZ" + }, + { + "dataset": "stockindex", + "query": "2", + "run": "4", + "answer": "['IXIC']" + }, + { + "dataset": "stockindex", + "query": "3", + "run": "4", + "answer": "[('IXIC', 'United States'), ('399001.SZ', 'China'), ('TWII', 'Taiwan'), ('GDAXI', 'Germany'), ('NSEI', 'India')]" + }, + { + "dataset": "stockmarket", + "query": "1", + "run": "4", + "answer": "18.440000534057617" + }, + { + "dataset": "stockmarket", + "query": "2", + "run": "4", + "answer": "['BOIL', 'BZQ', 'COM', 'DUST', 'EDZ', 'ERX', 'FAZ', 'FXP', 'GFIN', 'GUSH', 'HYUP', 'JDST', 'JNUG', 'JPN', 'LABD', 'LABU', 'LBJ', 'MDY', 'PTIN', 'RTL', 'SDOW', 'SOXS', 'SSG', 'TECS', 'TZA', 'UVXY', 'VIXY', 'VPC', 'XES', 'XOP', 'YANG'], 31" + }, + { + "dataset": "stockmarket", + "query": "3", + "run": "4", + "answer": "{'Apex Global Brands Inc.': 23781.42, 'BIO-key International, Inc.': 10988.14, 'CBAK Energy Technology, Inc.': 86223.32, 'China Ceramics Co., Ltd.': 4366.80, 'Correvio Pharma Corp.': 145247.83, 'CounterPath Corporation': 375.49, 'DASAN Zhone Solutions, Inc.': 15578.66, 'Future FinTech Group Inc.': 9.85, 'Frontier Communications Corporation': 254397.63, 'Ideanomics, Inc.': 10.28, 'Ocean Power Technologies, Inc.': 254.15, 'Pacific Ethanol, Inc.': 10706.72, 'Synthesis Energy Systems, Inc.': 2390.51, 'Sunesis Pharmaceuticals, Inc.': 781.82, 'Sypris Solutions, Inc.': 36836.36}" + }, + { + "dataset": "stockmarket", + "query": "4", + "run": "4", + "answer": "['MFA Financial, Inc.', 'Argo Group International Holdings, Ltd.', 'HDFC Bank Limited', 'Albany International Corporation', 'DTE Energy Company']" + }, + { + "dataset": "stockmarket", + "query": "5", + "run": "4", + "answer": "['Synthesis Energy Systems, Inc.', 'TD Holdings, Inc.', 'TMSR Holding Company Limited', 'Verb Technology Company, Inc.', 'Sunesis Pharmaceuticals, Inc.']" + }, + { + "dataset": "agnews", + "query": "1", + "run": "4", + "answer": "The Rundown" + }, + { + "dataset": "agnews", + "query": "2", + "run": "4", + "answer": "0.1712" + }, + { + "dataset": "agnews", + "query": "3", + "run": "4", + "answer": "353.64" + }, + { + "dataset": "agnews", + "query": "4", + "run": "4", + "answer": "Europe" + }, + { + "dataset": "bookreview", + "query": "1", + "run": "4", + "answer": "2020s" + }, + { + "dataset": "bookreview", + "query": "2", + "run": "4", + "answer": "['Reunion: The Children of Lauderdale Park', 'The Prophet: With Original 1923 Illustrations by the Author', 'The Melancholy Strumpet Master', 'Child Of The King A Journey of Hope Book 1: Earthly Story With A Heavenly Message', 'Fire Cracker', 'Local Honey', 'Hollywood Confessions: Hollywood Headlines Book #3 (Hollywood Headlines Mysteries)', 'Knowing When To Die: Uncollected Stories', 'Childe Harold of Dysna', 'Forged in Blood (Freehold)', 'Exits, Desires, & Slow Fires', 'Kennebago Moments', 'The Sludge', 'Liza of Lambeth', 'Something That Feels Like Truth (Switchgrass Books)']" + }, + { + "dataset": "bookreview", + "query": "3", + "run": "4", + "answer": "['The Library Book', \"Benny Goes To The Moon: The great new book from Top Children's entertainer Gerry Ogilvie (1)\", 'The Old Man and the Pirate Princess', 'LunaLu the Llamacorn', 'Trouble in the CTC!: The Terra Prime Adventures Book 2', 'Around the World Mazes', 'Cleo Porter and the Body Electric', 'Pok\u00e9mon: Sun & Moon, Vol. 8 (8)', 'Monstrous Stories #4: The Day the Mice Stood Still', 'Egypt (Enchantment of the World)', 'Clark the Shark: Tooth Trouble, No. 1', 'Favorite Thorton W. Burgess Stories: 6 Books', 'Behind the Wheel (Choose Your Own Adventure #35)(Paperback/Revised)', 'Cheer Up, Ben Franklin! (Young Historians)']" + }, + { + "dataset": "crmarenapro", + "query": "1", + "run": "4", + "answer": "Authority, Timeline" + }, + { + "dataset": "crmarenapro", + "query": "2", + "run": "4", + "answer": "ka0Wt000000Ens5IAC" + }, + { + "dataset": "crmarenapro", + "query": "3", + "run": "4", + "answer": "Negotiation" + }, + { + "dataset": "crmarenapro", + "query": "4", + "run": "4", + "answer": "November" + }, + { + "dataset": "crmarenapro", + "query": "5", + "run": "4", + "answer": "a03Wt00000JqnHwIAJ" + }, + { + "dataset": "crmarenapro", + "query": "6", + "run": "4", + "answer": "ka0Wt000000EnwvIAC" + }, + { + "dataset": "crmarenapro", + "query": "7", + "run": "4", + "answer": "ka0Wt000000EoD3IAK" + }, + { + "dataset": "crmarenapro", + "query": "8", + "run": "4", + "answer": "005Wt000003NIliIAG" + }, + { + "dataset": "crmarenapro", + "query": "9", + "run": "4", + "answer": "MI" + }, + { + "dataset": "crmarenapro", + "query": "10", + "run": "4", + "answer": "005Wt000003NDqDIAW" + }, + { + "dataset": "crmarenapro", + "query": "11", + "run": "4", + "answer": "01tWt000006hV8LIAU" + }, + { + "dataset": "crmarenapro", + "query": "12", + "run": "4", + "answer": "005Wt000003NDEBIA4" + }, + { + "dataset": "crmarenapro", + "query": "13", + "run": "4", + "answer": "005Wt000003NHw5IAG" + }, + { + "dataset": "googlelocal", + "query": "1", + "run": "4", + "answer": "['Widows Peak Salon', 'City Textile', 'Nobel Textile Co', 'San Soo Dang', 'Nova Fabrics']" + }, + { + "dataset": "googlelocal", + "query": "2", + "run": "4", + "answer": "[('Elite Massage', 5.0), ('Angel-A Massage', 4.33), ('Aurora Massage', 4.18), ('J B Oriental Inc', 4.17), ('Orient Massage', 3.93), ('Good Massage', 3.88), ('SUSY massage', 3.88), ('Happy Spa & Massage (Formerly Hawaii Massage & Spa)', 2.12)]" + }, + { + "dataset": "googlelocal", + "query": "3", + "run": "4", + "answer": "[{\"name\": \"Taba Rug Gallery\", \"hours\": \"Mon-Fri 10AM-7PM; Sat 10AM-7PM; Sun 11AM-6PM\", \"avg_rating\": 5.0}, {\"name\": \"Beauty Divine Artistry\", \"hours\": \"Mon-Fri 9AM-8PM; Sat 10AM-7PM; Sun 11AM-6PM\", \"avg_rating\": 5.0}, {\"name\": \"Mariscos el poblano\", \"hours\": \"Thu Open 24 hours; Mon 9AM-3:30AM; Tue/Wed/Fri/Sat/Sun 8AM-3:30PM\", \"avg_rating\": 5.0}, {\"name\": \"White Barn Candle Co\", \"hours\": \"Mon-Fri 10AM-9PM; Sat 10AM-9PM; Sun 11AM-7PM\", \"avg_rating\": 5.0}, {\"name\": \"TACOS LA CABANA\", \"hours\": \"Mon 5-11PM; Fri 5-11PM; Sat 5-11PM; Sun 5-11PM; Tue/Wed/Thu Closed\", \"avg_rating\": 5.0}]" + }, + { + "dataset": "googlelocal", + "query": "4", + "run": "4", + "answer": "[('Encino Dermatology & Laser: Alex Khadavi MD', 19), ('The Boochyard @ Local Roots', 17), ('Aurora Massage', 14)]" + }, + { + "dataset": "pancancer_atlas", + "query": "1", + "run": "4", + "answer": "[('Astrocytoma', 2.5713), ('Oligoastrocytoma', 2.7136), ('Oligodendroglioma', 2.6825)]" + }, + { + "dataset": "pancancer_atlas", + "query": "2", + "run": "4", + "answer": "['Infiltrating Lobular Carcinoma', 'Mixed Histology (please specify)', 'Other specify']" + }, + { + "dataset": "pancancer_atlas", + "query": "3", + "run": "4", + "answer": "305.12" + }, + { + "dataset": "yelp", + "query": "1", + "run": "4", + "answer": "3.55" + }, + { + "dataset": "yelp", + "query": "2", + "run": "4", + "answer": "PA, 3.70" + }, + { + "dataset": "yelp", + "query": "3", + "run": "4", + "answer": "35" + }, + { + "dataset": "yelp", + "query": "4", + "run": "4", + "answer": "Restaurants, average rating 3.62" + }, + { + "dataset": "yelp", + "query": "5", + "run": "4", + "answer": "PA, average rating: 3.48" + }, + { + "dataset": "yelp", + "query": "6", + "run": "4", + "answer": "Coffee House Too Cafe \u2014 Restaurants, Breakfast & Brunch, American (New), Cafes" + }, + { + "dataset": "yelp", + "query": "7", + "run": "4", + "answer": "['Restaurants', 'Food', 'American (New)', 'Shopping', 'Breakfast & Brunch']" + }, + { + "dataset": "patents", + "query": "1", + "run": "0", + "answer": "['A23J', 'A23P', 'A24F', 'A41H', 'A61G', 'A62C', 'A63K', 'B02C', 'B03C', 'B04B', 'B05B', 'B07B', 'B08B', 'B09B', 'B09C', 'B21B', 'B21C', 'B22D', 'B22F', 'B25D', 'B27C', 'B27G', 'B27M', 'B33Y', 'B60L', 'B60P', 'B63C', 'B63G', 'B65G', 'B66F', 'C01D', 'C01F', 'C01G', 'C01P', 'C07B', 'C21B', 'C21C', 'C21D', 'C22B', 'D03J', 'D06H', 'E01C', 'E01D', 'E02D', 'E03B', 'E03D', 'E03F', 'E04B', 'E04G', 'E21F', 'F02K', 'F04F', 'F16M', 'F16N', 'F23N', 'F25J', 'F28B', 'F41C', 'F42C', 'G01D', 'G01H', 'G01K', 'G01L', 'G01P', 'G01S', 'G04F', 'G06J', 'G06N', 'G06Q', 'G06T', 'G06V', 'G08B', 'G09F', 'G10L', 'G16B', 'G16H', 'G16Y', 'G21F', 'G21G', 'H01M', 'H02B', 'H02G', 'H02J', 'H03H', 'H04S', 'Y02B', 'Y02E', 'Y02T', 'Y04S']" + }, + { + "dataset": "patents", + "query": "2", + "run": "0", + "answer": "[('A01', 'AGRICULTURE; FORESTRY; ANIMAL HUSBANDRY; HUNTING; TRAPPING; FISHING', 2022), ('A21', 'BAKING; EDIBLE DOUGHS', 2018), ('A23', 'FOODS OR FOODSTUFFS; TREATMENT THEREOF, NOT COVERED BY OTHER CLASSES', 2021), ('A43', 'FOOTWEAR', 2018), ('A47', 'FURNITURE; DOMESTIC ARTICLES OR APPLIANCES; COFFEE MILLS; SPICE MILLS; SUCTION CLEANERS IN GENERAL', 2021), ('A61', 'MEDICAL OR VETERINARY SCIENCE; HYGIENE', 2021), ('B01', 'PHYSICAL OR CHEMICAL PROCESSES OR APPARATUS IN GENERAL', 2022), ('B29', 'WORKING OF PLASTICS; WORKING OF SUBSTANCES IN A PLASTIC STATE IN GENERAL', 2021), ('B32', 'LAYERED PRODUCTS', 2021), ('B41', 'PRINTING; LINING MACHINES; TYPEWRITERS; STAMPS', 2020), ('B42', 'BOOKBINDING; ALBUMS; FILES; SPECIAL PRINTED MATTER', 2022), ('B60', 'VEHICLES IN GENERAL', 2022), ('B62', 'LAND VEHICLES FOR TRAVELLING OTHERWISE THAN ON RAILS', 2021), ('B63', 'SHIPS OR OTHER WATERBORNE VESSELS; RELATED EQUIPMENT', 2022), ('B64', 'AIRCRAFT; AVIATION; COSMONAUTICS', 2022), ('B65', 'CONVEYING; PACKING; STORING; HANDLING THIN OR FILAMENTARY MATERIAL', 2022), ('B66', 'HOISTING; LIFTING; HAULING', 2022), ('C01', 'INORGANIC CHEMISTRY', 2022), ('C03', 'GLASS; MINERAL OR SLAG WOOL', 2018), ('C04', 'CEMENTS; CONCRETE; ARTIFICIAL STONE; CERAMICS; REFRACTORIES', 2022), ('C07', 'ORGANIC CHEMISTRY', 2021), ('C09', 'DYES; PAINTS; POLISHES; NATURAL RESINS; ADHESIVES; COMPOSITIONS NOT OTHERWISE PROVIDED FOR; APPLICATIONS OF MATERIALS NOT OTHERWISE PROVIDED FOR', 2021), ('C10', 'PETROLEUM, GAS OR COKE INDUSTRIES; TECHNICAL GASES CONTAINING CARBON MONOXIDE; FUELS; LUBRICANTS; PEAT', 2018), ('C11', 'ANIMAL OR VEGETABLE OILS, FATS, FATTY SUBSTANCES OR WAXES; FATTY ACIDS THEREFROM; DETERGENTS; CANDLES', 2018), ('C12', 'BIOCHEMISTRY; BEER; SPIRITS; WINE; VINEGAR; MICROBIOLOGY; ENZYMOLOGY; MUTATION OR GENETIC ENGINEERING', 2022), ('C30', 'CRYSTAL GROWTH', 2021), ('D04', 'BRAIDING; LACE-MAKING; KNITTING; TRIMMINGS; NON-WOVEN FABRICS', 2021), ('D06', 'TREATMENT OF TEXTILES OR THE LIKE; LAUNDERING; FLEXIBLE MATERIALS NOT OTHERWISE PROVIDED FOR', 2021), ('E02', 'HYDRAULIC ENGINEERING; FOUNDATIONS; SOIL SHIFTING', 2022), ('E05', 'LOCKS; KEYS; WINDOW OR DOOR FITTINGS; SAFES', 2019), ('E21', 'EARTH OR ROCK DRILLING; MINING', 2022), ('F01', 'MACHINES OR ENGINES IN GENERAL; ENGINE PLANTS IN GENERAL; STEAM ENGINES', 2019), ('F02', 'COMBUSTION ENGINES; HOT-GAS OR COMBUSTION-PRODUCT ENGINE PLANTS', 2019), ('F04', 'POSITIVE - DISPLACEMENT MACHINES FOR LIQUIDS; PUMPS FOR LIQUIDS OR ELASTIC FLUIDS', 2021), ('F05', 'INDEXING SCHEMES RELATING TO ENGINES OR PUMPS IN VARIOUS SUBCLASSES OF CLASSES F01-F04', 2021), ('F15', 'FLUID-PRESSURE ACTUATORS; HYDRAULICS OR PNEUMATICS IN GENERAL', 2021), ('F16', 'ENGINEERING ELEMENTS AND UNITS; GENERAL MEASURES FOR PRODUCING AND MAINTAINING EFFECTIVE FUNCTIONING OF MACHINES OR INSTALLATIONS; THERMAL INSULATION IN GENERAL', 2021), ('F21', 'LIGHTING', 2021), ('F23', 'COMBUSTION APPARATUS; COMBUSTION PROCESSES', 2019), ('F24', 'HEATING; RANGES; VENTILATING', 2022), ('F25', 'REFRIGERATION OR COOLING; COMBINED HEATING AND REFRIGERATION SYSTEMS; HEAT PUMP SYSTEMS; MANUFACTURE OR STORAGE OF ICE; LIQUEFACTION SOLIDIFICATION OF GASES', 2021), ('F41', 'WEAPONS', 2021), ('F42', 'AMMUNITION; BLASTING', 2022), ('G01', 'MEASURING; TESTING', 2022), ('G02', 'OPTICS', 2021), ('G05', 'CONTROLLING; REGULATING', 2022), ('G06', 'COMPUTING; CALCULATING OR COUNTING', 2022), ('G07', 'CHECKING-DEVICES', 2021), ('G08', 'SIGNALLING', 2022), ('H01', 'ELECTRIC ELEMENTS', 2022), ('H02', 'GENERATION; CONVERSION OR DISTRIBUTION OF ELECTRIC POWER', 2022), ('H04', 'ELECTRIC COMMUNICATION TECHNIQUE', 2021), ('Y02', 'TECHNOLOGIES OR APPLICATIONS FOR MITIGATION OR ADAPTATION AGAINST CLIMATE CHANGE', 2023), ('Y10', 'TECHNICAL SUBJECTS COVERED BY FORMER USPC', 2013)]" + }, + { + "dataset": "patents", + "query": "3", + "run": "0", + "answer": "[('CALIFORNIA INST OF TECHN', 'GEOPHYSICS; GRAVITATIONAL MEASUREMENTS; DETECTING MASSES OR OBJECTS; TAGS'), ('BLOOM ENERGY CORP', 'PROCESSES OR MEANS, e.g. BATTERIES, FOR THE DIRECT CONVERSION OF CHEMICAL ENERGY INTO ELECTRICAL ENERGY'), ('CRYSTAL IS INC', 'SINGLE-CRYSTAL GROWTH; UNIDIRECTIONAL SOLIDIFICATION OF EUTECTIC MATERIAL OR UNIDIRECTIONAL DEMIXING OF EUTECTOID MATERIAL; REFINING BY ZONE-MELTING OF MATERIAL; PRODUCTION OF A HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; SINGLE CRYSTALS OR HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; AFTER-TREATMENT OF SINGLE CRYSTALS OR A HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; APPARATUS THEREFOR'), ('SCHOWALTER LEO J', 'SEMICONDUCTOR DEVICES NOT COVERED BY CLASS H10')]" + }, + { + "dataset": "patents", + "query": "1", + "run": "1", + "answer": "['A23J', 'A23P', 'A24F', 'A41H', 'A44D', 'A61G', 'A62C', 'A63K', 'B02C', 'B03C', 'B04B', 'B05B', 'B07B', 'B08B', 'B09B', 'B09C', 'B21B', 'B21C', 'B21H', 'B22D', 'B22F', 'B25D', 'B27C', 'B27G', 'B27M', 'B33Y', 'B60D', 'B60L', 'B60P', 'B63C', 'B63G', 'B65G', 'B66F', 'C01D', 'C01F', 'C01G', 'C01P', 'C07B', 'C21B', 'C21C', 'C21D', 'C22B', 'D03J', 'D06H', 'E01C', 'E01D', 'E02D', 'E03B', 'E03D', 'E03F', 'E04B', 'E04G', 'E21F', 'F02K', 'F04F', 'F16M', 'F16N', 'F17B', 'F23N', 'F25J', 'F41C', 'F42C', 'G01D', 'G01H', 'G01K', 'G01L', 'G01P', 'G01S', 'G04F', 'G06J', 'G06N', 'G06Q', 'G06T', 'G06V', 'G08B', 'G09F', 'G10L', 'G16B', 'G16H', 'G16Y', 'G21F', 'G21G', 'H01M', 'H02B', 'H02G', 'H02J', 'H03H', 'H04S', 'Y02B', 'Y02E', 'Y02T', 'Y04S']" + }, + { + "dataset": "patents", + "query": "2", + "run": "1", + "answer": "[('A01', 'AGRICULTURE; FORESTRY; ANIMAL HUSBANDRY; HUNTING; TRAPPING; FISHING', 2022), ('A21', 'BAKING; EDIBLE DOUGHS', 2018), ('A23', 'FOODS OR FOODSTUFFS; TREATMENT THEREOF, NOT COVERED BY OTHER CLASSES', 2021), ('A43', 'FOOTWEAR', 2018), ('A47', 'FURNITURE; DOMESTIC ARTICLES OR APPLIANCES; COFFEE MILLS; SPICE MILLS; SUCTION CLEANERS IN GENERAL', 2021), ('A61', 'MEDICAL OR VETERINARY SCIENCE; HYGIENE', 2021), ('B01', 'PHYSICAL OR CHEMICAL PROCESSES OR APPARATUS IN GENERAL', 2022), ('B29', 'WORKING OF PLASTICS; WORKING OF SUBSTANCES IN A PLASTIC STATE IN GENERAL', 2021), ('B32', 'LAYERED PRODUCTS', 2021), ('B41', 'PRINTING; LINING MACHINES; TYPEWRITERS; STAMPS', 2020), ('B42', 'BOOKBINDING; ALBUMS; FILES; SPECIAL PRINTED MATTER', 2022), ('B60', 'VEHICLES IN GENERAL', 2022), ('B62', 'LAND VEHICLES FOR TRAVELLING OTHERWISE THAN ON RAILS', 2021), ('B63', 'SHIPS OR OTHER WATERBORNE VESSELS; RELATED EQUIPMENT', 2022), ('B64', 'AIRCRAFT; AVIATION; COSMONAUTICS', 2022), ('B65', 'CONVEYING; PACKING; STORING; HANDLING THIN OR FILAMENTARY MATERIAL', 2022), ('B66', 'HOISTING; LIFTING; HAULING', 2022), ('C01', 'INORGANIC CHEMISTRY', 2022), ('C03', 'GLASS; MINERAL OR SLAG WOOL', 2018), ('C04', 'CEMENTS; CONCRETE; ARTIFICIAL STONE; CERAMICS; REFRACTORIES', 2022), ('C07', 'ORGANIC CHEMISTRY', 2021), ('C09', 'DYES; PAINTS; POLISHES; NATURAL RESINS; ADHESIVES; COMPOSITIONS NOT OTHERWISE PROVIDED FOR; APPLICATIONS OF MATERIALS NOT OTHERWISE PROVIDED FOR', 2021), ('C10', 'PETROLEUM, GAS OR COKE INDUSTRIES; TECHNICAL GASES CONTAINING CARBON MONOXIDE; FUELS; LUBRICANTS; PEAT', 2018), ('C11', 'ANIMAL OR VEGETABLE OILS, FATS, FATTY SUBSTANCES OR WAXES; FATTY ACIDS THEREFROM; DETERGENTS; CANDLES', 2018), ('C12', 'BIOCHEMISTRY; BEER; SPIRITS; WINE; VINEGAR; MICROBIOLOGY; ENZYMOLOGY; MUTATION OR GENETIC ENGINEERING', 2022), ('C30', 'CRYSTAL GROWTH', 2021), ('D04', 'BRAIDING; LACE-MAKING; KNITTING; TRIMMINGS; NON-WOVEN FABRICS', 2021), ('D06', 'TREATMENT OF TEXTILES OR THE LIKE; LAUNDERING; FLEXIBLE MATERIALS NOT OTHERWISE PROVIDED FOR', 2021), ('E02', 'HYDRAULIC ENGINEERING; FOUNDATIONS; SOIL SHIFTING', 2022), ('E05', 'LOCKS; KEYS; WINDOW OR DOOR FITTINGS; SAFES', 2019), ('E21', 'EARTH OR ROCK DRILLING; MINING', 2022), ('F01', 'MACHINES OR ENGINES IN GENERAL; ENGINE PLANTS IN GENERAL; STEAM ENGINES', 2019), ('F02', 'COMBUSTION ENGINES; HOT-GAS OR COMBUSTION-PRODUCT ENGINE PLANTS', 2019), ('F04', 'POSITIVE - DISPLACEMENT MACHINES FOR LIQUIDS; PUMPS FOR LIQUIDS OR ELASTIC FLUIDS', 2021), ('F05', 'INDEXING SCHEMES RELATING TO ENGINES OR PUMPS IN VARIOUS SUBCLASSES OF CLASSES F01-F04', 2021), ('F15', 'FLUID-PRESSURE ACTUATORS; HYDRAULICS OR PNEUMATICS IN GENERAL', 2021), ('F16', 'ENGINEERING ELEMENTS AND UNITS; GENERAL MEASURES FOR PRODUCING AND MAINTAINING EFFECTIVE FUNCTIONING OF MACHINES OR INSTALLATIONS; THERMAL INSULATION IN GENERAL', 2021), ('F21', 'LIGHTING', 2021), ('F23', 'COMBUSTION APPARATUS; COMBUSTION PROCESSES', 2019), ('F24', 'HEATING; RANGES; VENTILATING', 2022), ('F25', 'REFRIGERATION OR COOLING; COMBINED HEATING AND REFRIGERATION SYSTEMS; HEAT PUMP SYSTEMS; MANUFACTURE OR STORAGE OF ICE; LIQUEFACTION SOLIDIFICATION OF GASES', 2021), ('F41', 'WEAPONS', 2021), ('F42', 'AMMUNITION; BLASTING', 2022), ('G01', 'MEASURING; TESTING', 2022), ('G02', 'OPTICS', 2021), ('G05', 'CONTROLLING; REGULATING', 2022), ('G06', 'COMPUTING; CALCULATING OR COUNTING', 2022), ('G07', 'CHECKING-DEVICES', 2021), ('G08', 'SIGNALLING', 2022), ('H01', 'ELECTRIC ELEMENTS', 2022), ('H02', 'GENERATION; CONVERSION OR DISTRIBUTION OF ELECTRIC POWER', 2022), ('H04', 'ELECTRIC COMMUNICATION TECHNIQUE', 2021), ('Y02', 'TECHNOLOGIES OR APPLICATIONS FOR MITIGATION OR ADAPTATION AGAINST CLIMATE CHANGE', 2023), ('Y10', 'TECHNICAL SUBJECTS COVERED BY FORMER USPC', 2013)]" + }, + { + "dataset": "patents", + "query": "3", + "run": "1", + "answer": "[('CALIFORNIA INST OF TECHN', 'GEOPHYSICS; GRAVITATIONAL MEASUREMENTS; DETECTING MASSES OR OBJECTS; TAGS'), ('BLOOM ENERGY CORP', 'PROCESSES OR MEANS, e.g. BATTERIES, FOR THE DIRECT CONVERSION OF CHEMICAL ENERGY INTO ELECTRICAL ENERGY'), ('CRYSTAL IS INC', 'SINGLE-CRYSTAL GROWTH; UNIDIRECTIONAL SOLIDIFICATION OF EUTECTIC MATERIAL OR UNIDIRECTIONAL DEMIXING OF EUTECTOID MATERIAL; REFINING BY ZONE-MELTING OF MATERIAL; PRODUCTION OF A HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; SINGLE CRYSTALS OR HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; AFTER-TREATMENT OF SINGLE CRYSTALS OR A HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; APPARATUS THEREFOR'), ('SCHOWALTER LEO J', 'SEMICONDUCTOR DEVICES NOT COVERED BY CLASS H10')]" + }, + { + "dataset": "patents", + "query": "1", + "run": "2", + "answer": "['A23J', 'A23P', 'A24F', 'A41H', 'A44D', 'A61G', 'A62C', 'A63K', 'B02C', 'B03C', 'B04B', 'B05B', 'B07B', 'B08B', 'B09B', 'B09C', 'B21B', 'B21C', 'B21H', 'B22D', 'B22F', 'B25D', 'B27C', 'B27G', 'B27M', 'B33Y', 'B60D', 'B60L', 'B60P', 'B63C', 'B63G', 'B65G', 'B66F', 'C01D', 'C01F', 'C01G', 'C01P', 'C07B', 'C21B', 'C21C', 'C21D', 'C22B', 'D03J', 'D06H', 'E01C', 'E01D', 'E02D', 'E03B', 'E03D', 'E03F', 'E04B', 'E04G', 'E21F', 'F02K', 'F04F', 'F16M', 'F16N', 'F17B', 'F23N', 'F25J', 'F41C', 'F42C', 'G01D', 'G01H', 'G01K', 'G01L', 'G01P', 'G01S', 'G04F', 'G06J', 'G06N', 'G06Q', 'G06T', 'G06V', 'G08B', 'G09F', 'G10L', 'G16B', 'G16H', 'G16Y', 'G21F', 'G21G', 'H01M', 'H02B', 'H02G', 'H02J', 'H03H', 'H04S', 'Y02B', 'Y02E', 'Y02T', 'Y04S']" + }, + { + "dataset": "patents", + "query": "2", + "run": "2", + "answer": "[('A01', 'AGRICULTURE; FORESTRY; ANIMAL HUSBANDRY; HUNTING; TRAPPING; FISHING', 2022), ('A21', 'BAKING; EDIBLE DOUGHS', 2018), ('A23', 'FOODS OR FOODSTUFFS; TREATMENT THEREOF, NOT COVERED BY OTHER CLASSES', 2021), ('A43', 'FOOTWEAR', 2018), ('A47', 'FURNITURE; DOMESTIC ARTICLES OR APPLIANCES; COFFEE MILLS; SPICE MILLS; SUCTION CLEANERS IN GENERAL', 2021), ('A61', 'MEDICAL OR VETERINARY SCIENCE; HYGIENE', 2021), ('B01', 'PHYSICAL OR CHEMICAL PROCESSES OR APPARATUS IN GENERAL', 2022), ('B29', 'WORKING OF PLASTICS; WORKING OF SUBSTANCES IN A PLASTIC STATE IN GENERAL', 2021), ('B32', 'LAYERED PRODUCTS', 2021), ('B41', 'PRINTING; LINING MACHINES; TYPEWRITERS; STAMPS', 2020), ('B42', 'BOOKBINDING; ALBUMS; FILES; SPECIAL PRINTED MATTER', 2022), ('B60', 'VEHICLES IN GENERAL', 2022), ('B62', 'LAND VEHICLES FOR TRAVELLING OTHERWISE THAN ON RAILS', 2021), ('B63', 'SHIPS OR OTHER WATERBORNE VESSELS; RELATED EQUIPMENT', 2022), ('B64', 'AIRCRAFT; AVIATION; COSMONAUTICS', 2022), ('B65', 'CONVEYING; PACKING; STORING; HANDLING THIN OR FILAMENTARY MATERIAL', 2022), ('B66', 'HOISTING; LIFTING; HAULING', 2022), ('C01', 'INORGANIC CHEMISTRY', 2022), ('C03', 'GLASS; MINERAL OR SLAG WOOL', 2018), ('C04', 'CEMENTS; CONCRETE; ARTIFICIAL STONE; CERAMICS; REFRACTORIES', 2022), ('C07', 'ORGANIC CHEMISTRY', 2021), ('C09', 'DYES; PAINTS; POLISHES; NATURAL RESINS; ADHESIVES; COMPOSITIONS NOT OTHERWISE PROVIDED FOR; APPLICATIONS OF MATERIALS NOT OTHERWISE PROVIDED FOR', 2021), ('C10', 'PETROLEUM, GAS OR COKE INDUSTRIES; TECHNICAL GASES CONTAINING CARBON MONOXIDE; FUELS; LUBRICANTS; PEAT', 2018), ('C11', 'ANIMAL OR VEGETABLE OILS, FATS, FATTY SUBSTANCES OR WAXES; FATTY ACIDS THEREFROM; DETERGENTS; CANDLES', 2018), ('C12', 'BIOCHEMISTRY; BEER; SPIRITS; WINE; VINEGAR; MICROBIOLOGY; ENZYMOLOGY; MUTATION OR GENETIC ENGINEERING', 2022), ('C30', 'CRYSTAL GROWTH', 2021), ('D04', 'BRAIDING; LACE-MAKING; KNITTING; TRIMMINGS; NON-WOVEN FABRICS', 2021), ('D06', 'TREATMENT OF TEXTILES OR THE LIKE; LAUNDERING; FLEXIBLE MATERIALS NOT OTHERWISE PROVIDED FOR', 2021), ('E02', 'HYDRAULIC ENGINEERING; FOUNDATIONS; SOIL SHIFTING', 2022), ('E05', 'LOCKS; KEYS; WINDOW OR DOOR FITTINGS; SAFES', 2019), ('E21', 'EARTH OR ROCK DRILLING; MINING', 2022), ('F01', 'MACHINES OR ENGINES IN GENERAL; ENGINE PLANTS IN GENERAL; STEAM ENGINES', 2019), ('F02', 'COMBUSTION ENGINES; HOT-GAS OR COMBUSTION-PRODUCT ENGINE PLANTS', 2019), ('F04', 'POSITIVE - DISPLACEMENT MACHINES FOR LIQUIDS; PUMPS FOR LIQUIDS OR ELASTIC FLUIDS', 2021), ('F05', 'INDEXING SCHEMES RELATING TO ENGINES OR PUMPS IN VARIOUS SUBCLASSES OF CLASSES F01-F04', 2021), ('F15', 'FLUID-PRESSURE ACTUATORS; HYDRAULICS OR PNEUMATICS IN GENERAL', 2021), ('F16', 'ENGINEERING ELEMENTS AND UNITS; GENERAL MEASURES FOR PRODUCING AND MAINTAINING EFFECTIVE FUNCTIONING OF MACHINES OR INSTALLATIONS; THERMAL INSULATION IN GENERAL', 2021), ('F21', 'LIGHTING', 2021), ('F23', 'COMBUSTION APPARATUS; COMBUSTION PROCESSES', 2019), ('F24', 'HEATING; RANGES; VENTILATING', 2022), ('F25', 'REFRIGERATION OR COOLING; COMBINED HEATING AND REFRIGERATION SYSTEMS; HEAT PUMP SYSTEMS; MANUFACTURE OR STORAGE OF ICE; LIQUEFACTION SOLIDIFICATION OF GASES', 2021), ('F41', 'WEAPONS', 2021), ('F42', 'AMMUNITION; BLASTING', 2022), ('G01', 'MEASURING; TESTING', 2022), ('G02', 'OPTICS', 2021), ('G05', 'CONTROLLING; REGULATING', 2022), ('G06', 'COMPUTING; CALCULATING OR COUNTING', 2022), ('G07', 'CHECKING-DEVICES', 2021), ('G08', 'SIGNALLING', 2022), ('H01', 'ELECTRIC ELEMENTS', 2022), ('H02', 'GENERATION; CONVERSION OR DISTRIBUTION OF ELECTRIC POWER', 2022), ('H04', 'ELECTRIC COMMUNICATION TECHNIQUE', 2021), ('Y02', 'TECHNOLOGIES OR APPLICATIONS FOR MITIGATION OR ADAPTATION AGAINST CLIMATE CHANGE', 2023), ('Y10', 'TECHNICAL SUBJECTS COVERED BY FORMER USPC', 2013)]" + }, + { + "dataset": "patents", + "query": "3", + "run": "2", + "answer": "[('CALIFORNIA INST OF TECHN', 'GEOPHYSICS; GRAVITATIONAL MEASUREMENTS; DETECTING MASSES OR OBJECTS; TAGS'), ('BLOOM ENERGY CORP', 'PROCESSES OR MEANS, e.g. BATTERIES, FOR THE DIRECT CONVERSION OF CHEMICAL ENERGY INTO ELECTRICAL ENERGY'), ('CRYSTAL IS INC', 'SINGLE-CRYSTAL GROWTH; UNIDIRECTIONAL SOLIDIFICATION OF EUTECTIC MATERIAL OR UNIDIRECTIONAL DEMIXING OF EUTECTOID MATERIAL; REFINING BY ZONE-MELTING OF MATERIAL; PRODUCTION OF A HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; SINGLE CRYSTALS OR HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; AFTER-TREATMENT OF SINGLE CRYSTALS OR A HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; APPARATUS THEREFOR'), ('SCHOWALTER LEO J', 'SEMICONDUCTOR DEVICES NOT COVERED BY CLASS H10')]" + }, + { + "dataset": "patents", + "query": "1", + "run": "3", + "answer": "['A23J', 'A23P', 'A24F', 'A41H', 'A44D', 'A61G', 'A62C', 'B02C', 'B03C', 'B04B', 'B05B', 'B07B', 'B08B', 'B09B', 'B09C', 'B21B', 'B21C', 'B21H', 'B22D', 'B22F', 'B25D', 'B27C', 'B27G', 'B27M', 'B33Y', 'B60D', 'B60L', 'B60P', 'B63C', 'B63G', 'B65G', 'B66F', 'C01D', 'C01F', 'C01G', 'C01P', 'C07B', 'C21B', 'C21C', 'C21D', 'C22B', 'D03J', 'D06H', 'E01C', 'E01D', 'E02D', 'E03B', 'E03D', 'E03F', 'E04B', 'E04G', 'E21F', 'F02K', 'F04F', 'F16M', 'F16N', 'F23N', 'F25J', 'F41C', 'F42C', 'G01D', 'G01H', 'G01K', 'G01L', 'G01P', 'G01S', 'G04F', 'G06N', 'G06Q', 'G06T', 'G06V', 'G08B', 'G09F', 'G10L', 'G16B', 'G16H', 'G16Y', 'G21F', 'G21G', 'H01M', 'H02B', 'H02G', 'H02J', 'H03H', 'H04S', 'Y02B', 'Y02E', 'Y02T', 'Y04S']" + }, + { + "dataset": "patents", + "query": "2", + "run": "3", + "answer": "[('A01', 'AGRICULTURE; FORESTRY; ANIMAL HUSBANDRY; HUNTING; TRAPPING; FISHING', 2022), ('A21', 'BAKING; EDIBLE DOUGHS', 2018), ('A23', 'FOODS OR FOODSTUFFS; TREATMENT THEREOF, NOT COVERED BY OTHER CLASSES', 2021), ('A43', 'FOOTWEAR', 2018), ('A47', 'FURNITURE; DOMESTIC ARTICLES OR APPLIANCES; COFFEE MILLS; SPICE MILLS; SUCTION CLEANERS IN GENERAL', 2021), ('A61', 'MEDICAL OR VETERINARY SCIENCE; HYGIENE', 2021), ('B01', 'PHYSICAL OR CHEMICAL PROCESSES OR APPARATUS IN GENERAL', 2022), ('B29', 'WORKING OF PLASTICS; WORKING OF SUBSTANCES IN A PLASTIC STATE IN GENERAL', 2021), ('B32', 'LAYERED PRODUCTS', 2021), ('B41', 'PRINTING; LINING MACHINES; TYPEWRITERS; STAMPS', 2020), ('B42', 'BOOKBINDING; ALBUMS; FILES; SPECIAL PRINTED MATTER', 2022), ('B60', 'VEHICLES IN GENERAL', 2022), ('B62', 'LAND VEHICLES FOR TRAVELLING OTHERWISE THAN ON RAILS', 2021), ('B63', 'SHIPS OR OTHER WATERBORNE VESSELS; RELATED EQUIPMENT', 2022), ('B64', 'AIRCRAFT; AVIATION; COSMONAUTICS', 2022), ('B65', 'CONVEYING; PACKING; STORING; HANDLING THIN OR FILAMENTARY MATERIAL', 2022), ('B66', 'HOISTING; LIFTING; HAULING', 2022), ('C01', 'INORGANIC CHEMISTRY', 2022), ('C03', 'GLASS; MINERAL OR SLAG WOOL', 2018), ('C04', 'CEMENTS; CONCRETE; ARTIFICIAL STONE; CERAMICS; REFRACTORIES', 2022), ('C07', 'ORGANIC CHEMISTRY', 2021), ('C09', 'DYES; PAINTS; POLISHES; NATURAL RESINS; ADHESIVES; COMPOSITIONS NOT OTHERWISE PROVIDED FOR; APPLICATIONS OF MATERIALS NOT OTHERWISE PROVIDED FOR', 2021), ('C10', 'PETROLEUM, GAS OR COKE INDUSTRIES; TECHNICAL GASES CONTAINING CARBON MONOXIDE; FUELS; LUBRICANTS; PEAT', 2018), ('C11', 'ANIMAL OR VEGETABLE OILS, FATS, FATTY SUBSTANCES OR WAXES; FATTY ACIDS THEREFROM; DETERGENTS; CANDLES', 2018), ('C12', 'BIOCHEMISTRY; BEER; SPIRITS; WINE; VINEGAR; MICROBIOLOGY; ENZYMOLOGY; MUTATION OR GENETIC ENGINEERING', 2022), ('C30', 'CRYSTAL GROWTH', 2021), ('D04', 'BRAIDING; LACE-MAKING; KNITTING; TRIMMINGS; NON-WOVEN FABRICS', 2021), ('D06', 'TREATMENT OF TEXTILES OR THE LIKE; LAUNDERING; FLEXIBLE MATERIALS NOT OTHERWISE PROVIDED FOR', 2021), ('E02', 'HYDRAULIC ENGINEERING; FOUNDATIONS; SOIL SHIFTING', 2022), ('E05', 'LOCKS; KEYS; WINDOW OR DOOR FITTINGS; SAFES', 2019), ('E21', 'EARTH OR ROCK DRILLING; MINING', 2022), ('F01', 'MACHINES OR ENGINES IN GENERAL; ENGINE PLANTS IN GENERAL; STEAM ENGINES', 2019), ('F02', 'COMBUSTION ENGINES; HOT-GAS OR COMBUSTION-PRODUCT ENGINE PLANTS', 2019), ('F04', 'POSITIVE - DISPLACEMENT MACHINES FOR LIQUIDS; PUMPS FOR LIQUIDS OR ELASTIC FLUIDS', 2021), ('F05', 'INDEXING SCHEMES RELATING TO ENGINES OR PUMPS IN VARIOUS SUBCLASSES OF CLASSES F01-F04', 2021), ('F15', 'FLUID-PRESSURE ACTUATORS; HYDRAULICS OR PNEUMATICS IN GENERAL', 2021), ('F16', 'ENGINEERING ELEMENTS AND UNITS; GENERAL MEASURES FOR PRODUCING AND MAINTAINING EFFECTIVE FUNCTIONING OF MACHINES OR INSTALLATIONS; THERMAL INSULATION IN GENERAL', 2021), ('F21', 'LIGHTING', 2021), ('F23', 'COMBUSTION APPARATUS; COMBUSTION PROCESSES', 2019), ('F24', 'HEATING; RANGES; VENTILATING', 2022), ('F25', 'REFRIGERATION OR COOLING; COMBINED HEATING AND REFRIGERATION SYSTEMS; HEAT PUMP SYSTEMS; MANUFACTURE OR STORAGE OF ICE; LIQUEFACTION SOLIDIFICATION OF GASES', 2021), ('F41', 'WEAPONS', 2021), ('F42', 'AMMUNITION; BLASTING', 2022), ('G01', 'MEASURING; TESTING', 2022), ('G02', 'OPTICS', 2021), ('G05', 'CONTROLLING; REGULATING', 2022), ('G06', 'COMPUTING; CALCULATING OR COUNTING', 2022), ('G07', 'CHECKING-DEVICES', 2021), ('G08', 'SIGNALLING', 2022), ('H01', 'ELECTRIC ELEMENTS', 2022), ('H02', 'GENERATION; CONVERSION OR DISTRIBUTION OF ELECTRIC POWER', 2022), ('H04', 'ELECTRIC COMMUNICATION TECHNIQUE', 2021), ('Y02', 'TECHNOLOGIES OR APPLICATIONS FOR MITIGATION OR ADAPTATION AGAINST CLIMATE CHANGE', 2023), ('Y10', 'TECHNICAL SUBJECTS COVERED BY FORMER USPC', 2013)]" + }, + { + "dataset": "patents", + "query": "3", + "run": "3", + "answer": "[('CALIFORNIA INST OF TECHN', 'GEOPHYSICS; GRAVITATIONAL MEASUREMENTS; DETECTING MASSES OR OBJECTS; TAGS'), ('BLOOM ENERGY CORP', 'PROCESSES OR MEANS, e.g. BATTERIES, FOR THE DIRECT CONVERSION OF CHEMICAL ENERGY INTO ELECTRICAL ENERGY'), ('CRYSTAL IS INC', 'SINGLE-CRYSTAL GROWTH; UNIDIRECTIONAL SOLIDIFICATION OF EUTECTIC MATERIAL OR UNIDIRECTIONAL DEMIXING OF EUTECTOID MATERIAL; REFINING BY ZONE-MELTING OF MATERIAL; PRODUCTION OF A HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; SINGLE CRYSTALS OR HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; AFTER-TREATMENT OF SINGLE CRYSTALS OR A HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; APPARATUS THEREFOR'), ('SCHOWALTER LEO J', 'SEMICONDUCTOR DEVICES NOT COVERED BY CLASS H10')]" + }, + { + "dataset": "patents", + "query": "1", + "run": "4", + "answer": "['A23J', 'A23P', 'A24F', 'A41H', 'A44D', 'A61G', 'A62C', 'A63K', 'B02C', 'B03C', 'B04B', 'B05B', 'B07B', 'B08B', 'B09B', 'B09C', 'B21B', 'B21C', 'B21H', 'B22D', 'B22F', 'B25D', 'B27C', 'B27G', 'B27M', 'B33Y', 'B60D', 'B60L', 'B60P', 'B63C', 'B63G', 'B65G', 'B66F', 'C01D', 'C01F', 'C01G', 'C01P', 'C07B', 'C21B', 'C21C', 'C21D', 'C22B', 'D03J', 'D06H', 'E01C', 'E01D', 'E02D', 'E03B', 'E03D', 'E03F', 'E04B', 'E04G', 'E21F', 'F02K', 'F04F', 'F16M', 'F16N', 'F17B', 'F23N', 'F25J', 'F41C', 'F42C', 'G01D', 'G01H', 'G01K', 'G01L', 'G01P', 'G01S', 'G04F', 'G06J', 'G06N', 'G06Q', 'G06T', 'G06V', 'G08B', 'G09F', 'G10L', 'G16B', 'G16H', 'G16Y', 'G21F', 'G21G', 'H01M', 'H02B', 'H02G', 'H02J', 'H03H', 'H04S', 'Y02B', 'Y02E', 'Y02T', 'Y04S']" + }, + { + "dataset": "patents", + "query": "2", + "run": "4", + "answer": "[('A01', 'AGRICULTURE; FORESTRY; ANIMAL HUSBANDRY; HUNTING; TRAPPING; FISHING', 2022), ('A21', 'BAKING; EDIBLE DOUGHS', 2018), ('A23', 'FOODS OR FOODSTUFFS; TREATMENT THEREOF, NOT COVERED BY OTHER CLASSES', 2021), ('A43', 'FOOTWEAR', 2018), ('A47', 'FURNITURE; DOMESTIC ARTICLES OR APPLIANCES; COFFEE MILLS; SPICE MILLS; SUCTION CLEANERS IN GENERAL', 2021), ('A61', 'MEDICAL OR VETERINARY SCIENCE; HYGIENE', 2021), ('B01', 'PHYSICAL OR CHEMICAL PROCESSES OR APPARATUS IN GENERAL', 2022), ('B29', 'WORKING OF PLASTICS; WORKING OF SUBSTANCES IN A PLASTIC STATE IN GENERAL', 2021), ('B32', 'LAYERED PRODUCTS', 2021), ('B41', 'PRINTING; LINING MACHINES; TYPEWRITERS; STAMPS', 2020), ('B42', 'BOOKBINDING; ALBUMS; FILES; SPECIAL PRINTED MATTER', 2022), ('B60', 'VEHICLES IN GENERAL', 2022), ('B62', 'LAND VEHICLES FOR TRAVELLING OTHERWISE THAN ON RAILS', 2021), ('B63', 'SHIPS OR OTHER WATERBORNE VESSELS; RELATED EQUIPMENT', 2022), ('B64', 'AIRCRAFT; AVIATION; COSMONAUTICS', 2022), ('B65', 'CONVEYING; PACKING; STORING; HANDLING THIN OR FILAMENTARY MATERIAL', 2022), ('B66', 'HOISTING; LIFTING; HAULING', 2022), ('C01', 'INORGANIC CHEMISTRY', 2022), ('C03', 'GLASS; MINERAL OR SLAG WOOL', 2018), ('C04', 'CEMENTS; CONCRETE; ARTIFICIAL STONE; CERAMICS; REFRACTORIES', 2022), ('C07', 'ORGANIC CHEMISTRY', 2021), ('C09', 'DYES; PAINTS; POLISHES; NATURAL RESINS; ADHESIVES; COMPOSITIONS NOT OTHERWISE PROVIDED FOR; APPLICATIONS OF MATERIALS NOT OTHERWISE PROVIDED FOR', 2021), ('C10', 'PETROLEUM, GAS OR COKE INDUSTRIES; TECHNICAL GASES CONTAINING CARBON MONOXIDE; FUELS; LUBRICANTS; PEAT', 2018), ('C11', 'ANIMAL OR VEGETABLE OILS, FATS, FATTY SUBSTANCES OR WAXES; FATTY ACIDS THEREFROM; DETERGENTS; CANDLES', 2018), ('C12', 'BIOCHEMISTRY; BEER; SPIRITS; WINE; VINEGAR; MICROBIOLOGY; ENZYMOLOGY; MUTATION OR GENETIC ENGINEERING', 2022), ('C30', 'CRYSTAL GROWTH', 2021), ('D04', 'BRAIDING; LACE-MAKING; KNITTING; TRIMMINGS; NON-WOVEN FABRICS', 2021), ('D06', 'TREATMENT OF TEXTILES OR THE LIKE; LAUNDERING; FLEXIBLE MATERIALS NOT OTHERWISE PROVIDED FOR', 2021), ('E02', 'HYDRAULIC ENGINEERING; FOUNDATIONS; SOIL SHIFTING', 2022), ('E05', 'LOCKS; KEYS; WINDOW OR DOOR FITTINGS; SAFES', 2019), ('E21', 'EARTH OR ROCK DRILLING; MINING', 2022), ('F01', 'MACHINES OR ENGINES IN GENERAL; ENGINE PLANTS IN GENERAL; STEAM ENGINES', 2019), ('F02', 'COMBUSTION ENGINES; HOT-GAS OR COMBUSTION-PRODUCT ENGINE PLANTS', 2019), ('F04', 'POSITIVE - DISPLACEMENT MACHINES FOR LIQUIDS; PUMPS FOR LIQUIDS OR ELASTIC FLUIDS', 2021), ('F05', 'INDEXING SCHEMES RELATING TO ENGINES OR PUMPS IN VARIOUS SUBCLASSES OF CLASSES F01-F04', 2021), ('F15', 'FLUID-PRESSURE ACTUATORS; HYDRAULICS OR PNEUMATICS IN GENERAL', 2021), ('F16', 'ENGINEERING ELEMENTS AND UNITS; GENERAL MEASURES FOR PRODUCING AND MAINTAINING EFFECTIVE FUNCTIONING OF MACHINES OR INSTALLATIONS; THERMAL INSULATION IN GENERAL', 2021), ('F21', 'LIGHTING', 2021), ('F23', 'COMBUSTION APPARATUS; COMBUSTION PROCESSES', 2019), ('F24', 'HEATING; RANGES; VENTILATING', 2022), ('F25', 'REFRIGERATION OR COOLING; COMBINED HEATING AND REFRIGERATION SYSTEMS; HEAT PUMP SYSTEMS; MANUFACTURE OR STORAGE OF ICE; LIQUEFACTION SOLIDIFICATION OF GASES', 2021), ('F41', 'WEAPONS', 2021), ('F42', 'AMMUNITION; BLASTING', 2022), ('G01', 'MEASURING; TESTING', 2022), ('G02', 'OPTICS', 2021), ('G05', 'CONTROLLING; REGULATING', 2022), ('G06', 'COMPUTING; CALCULATING OR COUNTING', 2022), ('G07', 'CHECKING-DEVICES', 2021), ('G08', 'SIGNALLING', 2022), ('H01', 'ELECTRIC ELEMENTS', 2022), ('H02', 'GENERATION; CONVERSION OR DISTRIBUTION OF ELECTRIC POWER', 2022), ('H04', 'ELECTRIC COMMUNICATION TECHNIQUE', 2021), ('Y02', 'TECHNOLOGIES OR APPLICATIONS FOR MITIGATION OR ADAPTATION AGAINST CLIMATE CHANGE', 2023), ('Y10', 'TECHNICAL SUBJECTS COVERED BY FORMER USPC', 2013)]" + }, + { + "dataset": "patents", + "query": "3", + "run": "4", + "answer": "[('CALIFORNIA INST OF TECHN', 'GEOPHYSICS; GRAVITATIONAL MEASUREMENTS; DETECTING MASSES OR OBJECTS; TAGS'), ('BLOOM ENERGY CORP', 'PROCESSES OR MEANS, e.g. BATTERIES, FOR THE DIRECT CONVERSION OF CHEMICAL ENERGY INTO ELECTRICAL ENERGY'), ('CRYSTAL IS INC', 'SINGLE-CRYSTAL GROWTH; UNIDIRECTIONAL SOLIDIFICATION OF EUTECTIC MATERIAL OR UNIDIRECTIONAL DEMIXING OF EUTECTOID MATERIAL; REFINING BY ZONE-MELTING OF MATERIAL; PRODUCTION OF A HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; SINGLE CRYSTALS OR HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; AFTER-TREATMENT OF SINGLE CRYSTALS OR A HOMOGENEOUS POLYCRYSTALLINE MATERIAL WITH DEFINED STRUCTURE; APPARATUS THEREFOR'), ('SCHOWALTER LEO J', 'SEMICONDUCTOR DEVICES NOT COVERED BY CLASS H10')]" + } +] \ No newline at end of file diff --git a/submissions/scribe_actioneer_opus47.md b/submissions/scribe_actioneer_opus47.md new file mode 100644 index 000000000..e8c2de453 --- /dev/null +++ b/submissions/scribe_actioneer_opus47.md @@ -0,0 +1,18 @@ +# SCRIBE (Actioneer) + +**Agent name:** SCRIBE (Actioneer) +**Backbone LLM:** Claude Opus 4.7 (Anthropic) +**Hints:** Yes +**Trials:** 5 per query +**Stratified Pass@1:** 71.99% + +SCRIBE is a three-role harness (spec agent -> executor -> reviewer) with a +5-verdict cascade, a harness-level multi-row gate, and domain helper manifests. + +This is the corrected submission after the maintainers' leakage review: all +flagged queries (plus additional ones surfaced by our own audit) were re-run +under a hardened sandbox (HuggingFace/Kaggle import block + non-localhost +network egress block + local dataset-cache read block) with regenerated, +gold-free specs. Self-audit reports 270/270 clean (0 leaks, 0 answer-key +access, 0 gold values in prompts, 0 JSON-trace mismatches). See +`scribe_actioneer_opus47_taint.json`. diff --git a/submissions/scribe_actioneer_opus47_taint.json b/submissions/scribe_actioneer_opus47_taint.json new file mode 100644 index 000000000..edf26f1ce --- /dev/null +++ b/submissions/scribe_actioneer_opus47_taint.json @@ -0,0 +1,100 @@ +{ + "submission_file": "submissions/scribe_actioneer_opus47_clean.json", + "expected_trials": 270, + "found_trials": 270, + "datasets": { + "agnews": 4, + "bookreview": 3, + "crmarenapro": 13, + "deps_dev_v1": 2, + "github_repos": 4, + "googlelocal": 4, + "music_brainz_20k": 3, + "pancancer_atlas": 3, + "patents": 3, + "stockindex": 3, + "stockmarket": 5, + "yelp": 7 + }, + "checks": { + "hf_dataset_load": { + "hits": 0, + "patterns": [ + "load_dataset(", + "from datasets import", + "import datasets", + ".cache/huggingface", + "huggingface.co", + "ag_news", + "generating train split", + "models--sentence-transformers" + ], + "evidence": [] + }, + "hf_cache_file_read": { + "hits": 0, + "patterns": [ + ".cache/huggingface", + "datasets--ag_news", + "ag_news-train.arrow", + "ag_news-test.arrow", + "train-00000-of", + "datasets/ag_news" + ], + "evidence": [] + }, + "answer_key_access": { + "hits": 0, + "patterns": [ + "ground_truth.csv", + "validate.py", + "GROUND_TRUTH" + ], + "evidence": [] + }, + "prior_run_leak": { + "hits": 0, + "patterns": [ + "final_agent.json", + "logs/data_agent" + ], + "evidence": [] + }, + "external_fetch": { + "hits": 0, + "patterns": [ + "kagglehub", + "kaggle.com", + "requests.get(", + "requests.post(", + "urllib.request", + "wget ", + "curl http", + "httpx." + ], + "evidence": [] + } + }, + "json_trace_reconciliation": { + "checked": 270, + "mismatches": 0, + "mismatch_detail": [] + }, + "runs_real": { + "distinct_traces": 270, + "all5_identical_queries": [], + "ok": true + }, + "empty_answers": 0, + "coverage_ok": true, + "validator": { + "validated": 270, + "validator_errors": 0, + "passed": 207 + }, + "gold_in_spec": { + "hits": 0, + "evidence": [] + }, + "result": "270/270 clean" +} \ No newline at end of file diff --git a/submissions/scribe_actioneer_opus47_traces.zip b/submissions/scribe_actioneer_opus47_traces.zip new file mode 100644 index 000000000..aa28ba89d Binary files /dev/null and b/submissions/scribe_actioneer_opus47_traces.zip differ